Chanomic Blog

Nixのいくつかの処理をPythonで実装してみる

(last modified:
)

tags:

categories:

Nixのパッケージ・derivationの探り方まとめにて色々なコマンドを紹介したが、それらがNix内部でどう処理されているのかを知りたくなり、その過程でPython実装を書いた。

目的は、以下の2つの処理をPythonで実装することである。

なお、公式ではどちらもnix-store --query --requisitesないしnix-store --query --treeで出力可能である。

(parse) drvファイルを読み込み、パースする

この先の処理を実装するにあたって、drvから情報を取り出す必要があるので、ここでパーサーを実装する。

まずdrvのファイル形式は以下のようなものであった。見やすいように改行を挟んでいるが、実際には無い。

bombrary@nixos:~$ nix derivation show `which ls` | jq -r 'to_entries[].key' | xargs cat
Derive(
 [("debug","/nix/store/b073nwng2fy24zaqbdx6zbimxkad7dyk-coreutils-full-9.3-debug","",""),
  ("info","/nix/store/1pd076gkjwh0wdv8cnxy6p7kl141jnk2-coreutils-full-9.3-info","",""),
  ("out","/nix/store/03167shkax5dxclnv6r3sd8waa6lq7ny-coreutils-full-9.3","","")],
 [("/nix/store/5q67fxm276bdp87jpmckvz3n81akw6a5-perl-5.38.2.drv",["out"]),
  ("/nix/store/98sv0g544bqmks49d6vgylbkh9sccdvm-attr-2.5.1.drv",["dev"]),
  ("/nix/store/9jcfzyyb0h86mvc31s9qmxs6lncqrwhc-acl-2.3.1.drv",["dev"]),
  ("/nix/store/akpwym6q116hivciyq2vqj9n5jk9f5i6-xz-5.4.4.drv",["bin"]),
  ("/nix/store/d1qldhg6iix84bqncbzml2a1nw8p95bg-gmp-with-cxx-6.3.0.drv",["dev"]),
  ("/nix/store/ks5ivc59k57kwii93qlsfgcx2a7xma1k-autoreconf-hook.drv",["out"]),
  ("/nix/store/mnrjvk62d35v8514kc5w31fg3py0smr8-coreutils-9.3.tar.xz.drv",["out"]),
  ("/nix/store/mvvhw7jrrr8wnjihpalw4s3y3g7jihgw-stdenv-linux.drv",["out"]),
  ("/nix/store/szciaprmwb7kdj7zv1b56midf7jfkjnw-bash-5.2-p15.drv",["out"]),
  ("/nix/store/wd100hlzyh5w9zkfljkaagp87b7h7733-openssl-3.0.12.drv",["dev"])],
 ["/nix/store/1cwqp9msvi5z8517czfl88dd42yhrdwg-separate-debug-info.sh",
  "/nix/store/v6x3cs394jgqfbi0a42pam708flxaphh-default-builder.sh"],
 "x86_64-linux",
 "/nix/store/7dpxg7ki7g8ynkdwcqf493p2x8divb4i-bash-5.2-p15/bin/bash",
 ["-e","/nix/store/v6x3cs394jgqfbi0a42pam708flxaphh-default-builder.sh"],
 [("FORCE_UNSAFE_CONFIGURE",""),
  ("NIX_CFLAGS_COMPILE",""),
  ("NIX_LDFLAGS",""),
  ("__structuredAttrs",""),
  ("buildInputs","/nix/store/hwb08pf2byl2a1rnmaxq56f389h6b6yn-acl-2.3.1-dev /nix/store/djciacxl96yr2wd02lcxyn8z046fzrqr-attr-2.5.1-dev /nix/store/1fszsmhmlhbi4yzl2wgi08cfw0dng7pq-gmp-with-cxx-6.3.0-dev /nix/store/2d8yhfx7f2crn8scyzdk6dg3lw7y1ifh-openssl-3.0.12-dev"),
  ("builder","/nix/store/7dpxg7ki7g8ynkdwcqf493p2x8divb4i-bash-5.2-p15/bin/bash"),
  ("cmakeFlags",""),
  ("configureFlags","--with-packager=https://nixos.org --enable-single-binary=symlinks --with-openssl gl_cv_have_proc_uptime=yes"),
  ("debug","/nix/store/b073nwng2fy24zaqbdx6zbimxkad7dyk-coreutils-full-9.3-debug"),
  ("depsBuildBuild",""),
  ("depsBuildBuildPropagated",""),
  ("depsBuildTarget",""),
  ("depsBuildTargetPropagated",""),
  ("depsHostHost",""),
  ("depsHostHostPropagated",""),
  ("depsTargetTarget",""),
  ("depsTargetTargetPropagated",""),
  ("doCheck","1"),
  ("doInstallCheck",""),
  ("enableParallelBuilding","1"),
  ("enableParallelChecking","1"),
  ("enableParallelInstalling","1"),
  ("info","/nix/store/1pd076gkjwh0wdv8cnxy6p7kl141jnk2-coreutils-full-9.3-info"),
  ("mesonFlags",""),
  ("name","coreutils-full-9.3"),
  ("nativeBuildInputs","/nix/store/nsl35d8x8jp0vy8n4xy8sx9v68gdh444-autoreconf-hook /nix/store/rza0ib08brnkwx75n7rncyjq97j76ris-perl-5.38.2 /nix/store/3q6fnwcm677l1q60vkhcf9m1gxhv83jm-xz-5.4.4-bin /nix/store/1cwqp9msvi5z8517czfl88dd42yhrdwg-separate-debug-info.sh"),
  ("out","/nix/store/03167shkax5dxclnv6r3sd8waa6lq7ny-coreutils-full-9.3"),
  ("outputs","out info debug"),
  ("patches",""),
  ("pname","coreutils-full"),
  ("postInstall",""),
  ("postPatch","# The test tends to fail on btrfs, f2fs and maybe other unusual filesystems.\nsed '2i echo Skipping dd sparse test && exit 77' -i ./tests/dd/sparse.sh\nsed '2i echo Skipping du threshold test && exit 77' -i ./tests/du/threshold.sh\nsed '2i echo Skipping cp reflink-auto test && exit 77' -i ./tests/cp/reflink-auto.sh\nsed '2i echo Skipping cp sparse test && exit 77' -i ./tests/cp/sparse.sh\nsed '2i echo Skipping rm deep-2 test && exit 77' -i ./tests/rm/deep-2.sh\nsed '2i echo Skipping du long-from-unreadable test && exit 77' -i ./tests/du/long-from-unreadable.sh\n\n# Some target platforms, especially when building inside a container have\n# issues with the inotify test.\nsed '2i echo Skipping tail inotify dir recreate test && exit 77' -i ./tests/tail-2/inotify-dir-recreate.sh\n\n# sandbox does not allow setgid\nsed '2i echo Skipping chmod setgid test && exit 77' -i ./tests/chmod/setgid.sh\nsubstituteInPlace ./tests/install/install-C.sh \\\n  --replace 'mode3=2755' 'mode3=1755'\n\n# Fails on systems with a rootfs. Looks like a bug in the test, see\n# https://lists.gnu.org/archive/html/bug-coreutils/2019-12/msg00000.html\nsed '2i print \"Skipping df skip-rootfs test\"; exit 77' -i ./tests/df/skip-rootfs.sh\n\n# these tests fail in the unprivileged nix sandbox (without nix-daemon) as we break posix assumptions\nfor f in ./tests/chgrp/{basic.sh,recurse.sh,default-no-deref.sh,no-x.sh,posix-H.sh}; do\n  sed '2i echo Skipping chgrp && exit 77' -i \"$f\"\ndone\nfor f in gnulib-tests/{test-chown.c,test-fchownat.c,test-lchown.c}; do\n  echo \"int main() { return 77; }\" > \"$f\"\ndone\n\n# intermittent failures on builders, unknown reason\nsed '2i echo Skipping du basic test && exit 77' -i ./tests/du/basic.sh\n"),
  ("preInstall",""),
  ("propagatedBuildInputs",""),
  ("propagatedNativeBuildInputs",""),
  ("separateDebugInfo","1"),
  ("src","/nix/store/8f1x5yr083sjbdkv33gxwiybywf560nz-coreutils-9.3.tar.xz"),
  ("stdenv","/nix/store/kv5wkk7xgc8paw9azshzlmxraffqcg0i-stdenv-linux"),
  ("strictDeps",""),
  ("system","x86_64-linux"),
  ("version","9.3")]
)

drvファイルをにらむと、データ型としては以下のパターンしかなさそうだとわかる。

dataclassを用いてパースしたデータを扱うことにする。すると、drvの形式をパースする関数parse_drvは次のように実装できる。ほとんどのparse_*関数は、返り値を(パースした値, 残りの文字列)のタプルで返すように実装している。

（追記・補足）素朴にタプルやリストとしてパースするとすごくシンプルに書けるが、それだとunparse関数を実装するときに扱いづらかったので、dataclassを用いた実装に変えた。

from dataclasses import dataclass
from typing import Callable

@dataclass
class Output:
    id: str
    path: str
    hash_algo: str
    hash: str

@dataclass(frozen=True)
class InputDrv:
    path: str
    ids: frozenset[str]

@dataclass
class Derivation:
    outputs: list[Output]
    input_drvs: set[InputDrv]
    input_srcs: set[str]
    system: str
    builder: str
    args: list[str]
    envs: dict[str, str]

def parse_drv(s: str) -> Derivation:
    _, s = consume("Derive(", s)
    outputs, s = parse_list(parse_output, s)
    _, s = consume(",", s)
    input_drvs, s = parse_list(parse_input_drv, s)
    _, s = consume(",", s)
    input_srcs, s = parse_list(parse_str, s)
    _, s = consume(",", s)
    system, s = parse_str(s)
    _, s = consume(",", s)
    builder, s = parse_str(s)
    _, s = consume(",", s)
    args, s = parse_list(parse_str, s)
    _, s = consume(",", s)
    env_entries, s = parse_list(parse_key_val, s)
    _, s = consume(")", s)
    return Derivation(
        outputs=outputs,
        input_drvs=set(input_drvs),
        input_srcs=set(input_srcs),
        system=system,
        builder=builder,
        args=args,
        envs=dict(env_entries),
    )

def parse_ch(s: str) -> tuple[str, str]:
    return s[0], s[1:]

def consume(expect: str, s: str) -> tuple[str, str]:
    actual, s = s[:len(expect)], s[len(expect):]
    if expect != actual:
        raise ValueError(f"Expect {expect}, but actual {actual}")
    return actual, s

def parse_list[T](
    parse_elem: Callable[[str], tuple[T, str]],
    s: str
) -> tuple[list[T], str]:
    _, s = consume("[", s)
    res = []
    while True:
        match s[0]:
            case "]":
                _, s = parse_ch(s)
                break
            case ",":
                _, s = parse_ch(s)
            case _:
                r, s = parse_elem(s)
                res.append(r)
    return res, s


def parse_output(s: str) -> tuple[Output, str]:
    _, s = consume("(", s)
    id, s = parse_str(s)
    _, s = consume(",", s)
    path, s = parse_str(s)
    _, s = consume(",", s)
    hash_algo, s = parse_str(s)
    _, s = consume(",", s)
    hash, s = parse_str(s)
    _, s = consume(")", s)
    return Output(id, path, hash_algo, hash), s


def parse_input_drv(s: str) -> tuple[InputDrv, str]:
    _, s = consume("(", s)
    path, s = parse_str(s)
    _, s = consume(",", s)
    ids, s = parse_list(parse_str, s)
    _, s = consume(")", s)
    return InputDrv(path, frozenset(ids)), s


def parse_key_val(s: str) -> tuple[tuple[str, str], str]:
    _, s = consume("(", s)
    key, s = parse_str(s)
    _, s = consume(",", s)
    val, s = parse_str(s)
    _, s = consume(")", s)
    return (key, val), s

def parse_str(s: str) -> tuple[str, str]:
    _, s = consume("\"", s)

    res = ""
    while True:
        match s[0]:
            case "\"":
                c, s = parse_ch(s)
                # エスケープされていたらbreakしない
                if res and res[-1] == "\\":
                    res += c
                else:
                    break
            case _:
                c, s = parse_ch(s)
                res += c
    return res, s

(unparse) Derivationクラスをdrv形式で書き出す

前節で作成したDerivation関数を、前節とは逆でdrv形式で書き出す処理unparse_drvを実装する。いくつかの要素は辞書順でソートされていないといけないので要注意。

def unparse_drv(drv: Derivation) -> str:
    outputs = sorted([ output_to_tuple(o) for o in drv.outputs ])
    input_drvs = sorted([ input_drv_to_tuple(o) for o in drv.input_drvs ])
    input_srcs = sorted(drv.input_srcs)
    args = drv.args
    envs = sorted(drv.envs.items())
    return "Derive(" + \
        ",".join([
            unparse_list(unparse_output_tuple, outputs),
            unparse_list(unparse_input_tuple, input_drvs),
            unparse_list(unparse_str, input_srcs),
            unparse_str(drv.system),
            unparse_str(drv.builder),
            unparse_list(unparse_str, args),
            unparse_list(unparse_key_val, envs),
        ]) + ")"

def output_to_tuple(out: Output) -> tuple[str, str, str, str]:
    return out.id, out.path, out.hash_algo, out.hash

def input_drv_to_tuple(ind: InputDrv) -> tuple[str, list[str]]:
    ids = sorted(ind.ids)
    return ind.path, ids

def unparse_output_tuple(out: tuple[str, str, str, str]) -> str:
    return "(" + ",".join([
        unparse_str(out[0]),
        unparse_str(out[1]),
        unparse_str(out[2]),
        unparse_str(out[3]),
    ]) + ")"

def unparse_input_tuple(input: tuple[str, list[str]]) -> str:
    return "(" + ",".join([
        unparse_str(input[0]),
        unparse_list(unparse_str, input[1]),
    ]) + ")"

def unparse_list[T](
    unparse_elem: Callable[[T], str],
    es: list[T]
) -> str:
    return "[" + ",".join([ unparse_elem(e) for e in es ]) + "]"

def unparse_str(s: str) -> str:
    return f"\"{s}\""

def unparse_key_val(
    keyval: tuple[str, str],
) -> str:
    return "(" + ",".join([
        unparse_str(keyval[0]),
        unparse_str(keyval[1]),
    ]) + ")"

derivationの直接的・間接的なbuild dependenciesをすべて出力する

再帰的に依存関係を探る関数dump_build_depsを実装する。

import sys

DRV_CACHE: dict[str, Derivation] = {}

Tree = str | dict[str, "Tree"]

def load_drv(path: str) -> Derivation:
    with open(path) as f:
        return parse_drv(f.read())

def dump_build_deps(path: str) -> Tree:
      if path not in DRV_CACHE:
          DRV_CACHE[path] = load_drv(path)
          input_drvs = DRV_CACHE[path].input_drvs
          input_srcs = DRV_CACHE[path].input_srcs
          res = { input_drv.path: dump_build_deps(input_drv.path) for input_drv in input_drvs }
          res |= { src: "" for src in input_srcs }
          return res
      else:
          return ": cached"

def show_tree(tree: Tree, last: bool, header=""):
    if isinstance(tree, str):
        return

    for i, (k, v) in enumerate(tree.items()):
        last = i == len(tree) - 1
        print(header, end="")
        if last:
            print("└──", end="")
        else:
            print("├──", end="")

        if isinstance(v, str):
            print(f'{k}{v}')
        else:
            print(k)

            if last:
                header_children = header + "   "
            else:
                header_children = header + "│  "
            show_tree(v, last, header_children)

if __name__ == "__main__":
    tree = dump_build_deps(sys.argv[1])
    show_tree(tree, True, "")

実行例。

bombrary@nixos:~/deps$ nix run nixpkgs#python312 -- dump.py /nix/store/g0kqr7b99b70kb10vmqg10vkj9nfk7zm-coreutils-full-9.3.drv
├──/nix/store/5q67fxm276bdp87jpmckvz3n81akw6a5-perl-5.38.2.drv
│  ├──/nix/store/1vzpfyxn64qx5my47kc0hjys37404hls-gcc-12.3.0.drv
│  │  ├──/nix/store/1032as2ph6j8pwan8dijl60jmfnzfi6b-perl-5.38.2.drv
│  │  │  ├──/nix/store/2zsw6v5l9zzhslrrdqpljnb425njg1pf-perl-5.38.2.tar.gz.drv
│  │  │  ├──/nix/store/9xhbdxvc93v7hc4vplng07z3y3lmfwvq-bootstrap-stage1-stdenv-linux.drv
│  │  │  │  ├──/nix/store/271ydjn02v2r49l5nn6yw5lr3nc5ydbi-update-autotools-gnu-config-scripts-hook.drv
│  │  │  │  │  ├──/nix/store/303sqdqr3x78jlgs00pixbdwv7hqizq1-gnu-config-2023-09-19.drv
│  │  │  │  │  │  ├──/nix/store/h11pn2l5rszzgjrl84qw2ifr33rdkjcq-config.sub-28ea239.drv
│  │  │  │  │  │  ├──/nix/store/ks6kir3vky8mb8zqpfhchwasn0rv1ix6-bootstrap-tools.drv
│  │  │  │  │  │  │  ├──/nix/store/b7irlwi2wjlx5aj1dghx4c8k3ax6m56q-busybox.drv
...
│  ├──/nix/store/jm8hin39q3ms3gffpa2w3xk8bxmychm3-make-shell-wrapper-hook.drv: cached
│  ├──/nix/store/mvvhw7jrrr8wnjihpalw4s3y3g7jihgw-stdenv-linux.drv: cached
│  ├──/nix/store/szciaprmwb7kdj7zv1b56midf7jfkjnw-bash-5.2-p15.drv: cached
│  ├──/nix/store/1cwqp9msvi5z8517czfl88dd42yhrdwg-separate-debug-info.sh
│  ├──/nix/store/cklrwbwi889pp2fdsswdjvn12sdy5i5j-openssl-disable-kernel-detection.patch
│  ├──/nix/store/lzmcfv2m4ripknpvbsv8wcg1ik1kif4h-use-etc-ssl-certs.patch
│  ├──/nix/store/sq4h6bqjx12v9whvm65pjss25hg1538q-nix-ssl-cert-file.patch
│  └──/nix/store/v6x3cs394jgqfbi0a42pam708flxaphh-default-builder.sh
├──/nix/store/1cwqp9msvi5z8517czfl88dd42yhrdwg-separate-debug-info.sh
└──/nix/store/v6x3cs394jgqfbi0a42pam708flxaphh-default-builder.sh

ツリー構造ではなくただ一覧で表示したい & cachedの行はいらない場合は、適当にsedやgrepで整形すればよい。

bombrary@nixos:~/deps$ nix run nixpkgs#python312 -- dump.py /nix/store/g0kqr7b99b70kb10vmqg10vkj9nfk7zm-coreutils-full-9.3.drv | sed 's/.*\(\/nix\/store\/.*\)/\1/' | grep -v cached
/nix/store/5q67fxm276bdp87jpmckvz3n81akw6a5-perl-5.38.2.drv
/nix/store/1vzpfyxn64qx5my47kc0hjys37404hls-gcc-12.3.0.drv
/nix/store/1032as2ph6j8pwan8dijl60jmfnzfi6b-perl-5.38.2.drv
/nix/store/2zsw6v5l9zzhslrrdqpljnb425njg1pf-perl-5.38.2.tar.gz.drv
/nix/store/9xhbdxvc93v7hc4vplng07z3y3lmfwvq-bootstrap-stage1-stdenv-linux.drv
/nix/store/271ydjn02v2r49l5nn6yw5lr3nc5ydbi-update-autotools-gnu-config-scripts-hook.drv
/nix/store/303sqdqr3x78jlgs00pixbdwv7hqizq1-gnu-config-2023-09-19.drv
/nix/store/h11pn2l5rszzgjrl84qw2ifr33rdkjcq-config.sub-28ea239.drv
/nix/store/ks6kir3vky8mb8zqpfhchwasn0rv1ix6-bootstrap-tools.drv
/nix/store/b7irlwi2wjlx5aj1dghx4c8k3ax6m56q-busybox.drv
...

ファイル・ディレクトリをNAR形式に変換する

runtime dependenciesを出力するための前準備。なお、同様のことはnix nar dump-pathコマンドでも行える。

edolstra氏のPh.D論文のp.93のFigure 5.2をもとに作成。

import io
import os
import stat

def archiveNAR(out_path: str) -> bytes:
    out = io.BytesIO()
    serialize(out, out_path)
    return out.getvalue()

def serialize(out: io.BytesIO, path: str):
    write_string(out, "nix-archive-1")
    serialize1(out, path)

def serialize1(out: io.BytesIO, path: str):
    write_string(out, "(")
    serialize2(out, path)
    write_string(out, ")")

def serialize_entry(out: io.BytesIO, name: str, path: str):
    write_strings(out, "entry", "(", "name", name, "node")
    serialize1(out, path);
    write_string(out, ")")

def serialize2(out: io.BytesIO, path: str):
    def dump_contents(path):
        with open(path, 'rb') as f:
            bs = f.read()
            write_bytes(out, bs);

    st = os.lstat(path)

    if stat.S_ISREG(st.st_mode):
        write_strings(out, "type", "regular")
        if is_executable(st.st_mode):
            write_strings(out, "executable", "")
        write_string(out, "contents")
        dump_contents(path);
    elif stat.S_ISDIR(st.st_mode):
        write_strings(out, "type", "directory")

        entries = read_directory(path)
        for relpath, _ in sorted(entries.items(), key=lambda e: e[0]):
            abspath = os.path.join(path, relpath)
            serialize_entry(out, relpath, abspath)

    elif stat.S_ISLNK(st.st_mode):
        write_strings(out, "type", "symlink", "target", os.readlink(path))

    else:
        raise ValueError("Invalid filetype")

def read_directory(dirpath: str):
    with os.scandir(dirpath) as it:
        return { e.name: os.lstat(e) for e in it }

def is_executable(mode: int) -> bool:
    return (mode & 0o111) != 0

def write_string(out: io.BytesIO, s: str):
    bs = s.encode(encoding='ascii')
    write_bytes(out, bs)

def write_strings(out: io.BytesIO, *ss: str):
    for s in ss:
        write_string(out, s)

def write_bytes(out: io.BytesIO, bs: bytes):
    out.write(len(bs).to_bytes(8, byteorder='little'))
    out.write(bs)

    m = len(bs) % 8
    if m:
        pad = b"\x00" * (8 - (len(bs) % 8))
        out.write(pad)

mainの部分。

import sys

if __name__ == "__main__":
    sys.stdout.buffer.write(archiveNAR(sys.argv[1]))

nix nar dump-path コマンドの結果と一致していることが分かる。

bombrary@nixos:~/deps$ echo "Hello, World" > hello.txt
bombrary@nixos:~/deps$ paste <(nix nar dump-path hello.txt | od -w8 -tx1z) <(nix run nixpkgs#python312 -- dump.py hello.txt | od -w8 -tx1z)
0000000 0d 00 00 00 00 00 00 00  >........<     0000000 0d 00 00 00 00 00 00 00  >........<
0000010 6e 69 78 2d 61 72 63 68  >nix-arch<     0000010 6e 69 78 2d 61 72 63 68  >nix-arch<
0000020 69 76 65 2d 31 00 00 00  >ive-1...<     0000020 69 76 65 2d 31 00 00 00  >ive-1...<
0000030 01 00 00 00 00 00 00 00  >........<     0000030 01 00 00 00 00 00 00 00  >........<
0000040 28 00 00 00 00 00 00 00  >(.......<     0000040 28 00 00 00 00 00 00 00  >(.......<
0000050 04 00 00 00 00 00 00 00  >........<     0000050 04 00 00 00 00 00 00 00  >........<
0000060 74 79 70 65 00 00 00 00  >type....<     0000060 74 79 70 65 00 00 00 00  >type....<
0000070 07 00 00 00 00 00 00 00  >........<     0000070 07 00 00 00 00 00 00 00  >........<
0000100 72 65 67 75 6c 61 72 00  >regular.<     0000100 72 65 67 75 6c 61 72 00  >regular.<
0000110 08 00 00 00 00 00 00 00  >........<     0000110 08 00 00 00 00 00 00 00  >........<
0000120 63 6f 6e 74 65 6e 74 73  >contents<     0000120 63 6f 6e 74 65 6e 74 73  >contents<
0000130 0d 00 00 00 00 00 00 00  >........<     0000130 0d 00 00 00 00 00 00 00  >........<
0000140 48 65 6c 6c 6f 2c 20 57  >Hello, W<     0000140 48 65 6c 6c 6f 2c 20 57  >Hello, W<
0000150 6f 72 6c 64 0a 00 00 00  >orld....<     0000150 6f 72 6c 64 0a 00 00 00  >orld....<
0000160 01 00 00 00 00 00 00 00  >........<     0000160 01 00 00 00 00 00 00 00  >........<
0000170 29 00 00 00 00 00 00 00  >).......<     0000170 29 00 00 00 00 00 00 00  >).......<
0000200 0000200

output pathからderivationを見つける

runtime dependenciesを出力するための前準備。

output pathからderivationを見つける方法として、公式的にはnix-store --query --deriverコマンドを用いる方法がある。しかし、そもそもoutput pathはderivationの諸々の情報を使いハッシュ化して作成されたものである。ハッシュの不可逆性により、逆にoutput pathから直接導出することは不可能のはずである。

ではNixではどうやってこれを行っているのかというと、別に/nix/store/を全探索とかしている訳ではない。実は/nix/var/nix/db/db.sqliteにSQLiteのDBがあり、そこにいろいろな情報を保管している。このDBはNix内部で用いるものであり、情報がほとんどないのだが、一応GlossalyやLocal Storeの説明から、その存在だけは確認できる。

そこからどうやってSQL文を叩けばoutput pathからderivationが引けるのかというと、少なくともNix 2.21.1の状況では

から分かる。要するに、ValidPathsというテーブルをpathで引いて、deriverのカラムを取り出せばよい。SQL文的には

SELECT deriver FROM ValidPaths WHERE path = '<output path>';

となる。

実際にPythonコードにすると、mainも含め以下のようになる。

import sqlite3
import sys

def get_deriver(conn: sqlite3.Connection, out_path: str):
    cur = conn.cursor()
    cur.execute('select deriver from ValidPaths where path = ?', (out_path, ))
    for row in cur:
        return row[0]
    raise ValueError("Deriver is not found")

if __name__ == "__main__":
    with sqlite3.connect("file:///nix/var/nix/db/db.sqlite?immutable=1", uri=True) as conn:
        print(get_deriver(conn, sys.argv[1]))

実行例。symlink先のパスの判定とかは特にやってないため、which lsをdump.pyの引数に埋め込んでも動作しないことに注意。

bombrary@nixos:~/deps$ realpath `which ls`
/nix/store/03167shkax5dxclnv6r3sd8waa6lq7ny-coreutils-full-9.3/bin/coreutils

bombrary@nixos:~/deps$ nix run nixpkgs#python312 -- dump.py /nix/store/03167shkax5dxclnv6r3sd8waa6lq7ny-coreutils-full-9.3
/nix/store/g0kqr7b99b70kb10vmqg10vkj9nfk7zm-coreutils-full-9.3.drv

パッケージの直接的・間接的なruntime dependenciesをすべて知る

今まで書いたコードを総動員して、あるパッケージのruntime dependenciesを再帰的に探索するコードが書ける。

OUT_CACHE = set()

def dump_runtime_deps(out_path: str) -> Tree:
    res = {}
    nar = archiveNAR(out_path)
    for drv in DRV_CACHE.values():
        # NARに埋め込まれているout_pathを取り出す
        paths = [ out.path for out in drv.outputs if out.path.encode('ascii') in nar ]
        for path in paths:
            if path == out_path:
                continue
            if  path not in OUT_CACHE:
                OUT_CACHE.add(out_path)
                res[path] = dump_runtime_deps(path)
            else:
                res[path] = ": cached"
    return res

if __name__ == "__main__":
    with sqlite3.connect("file:///nix/var/nix/db/db.sqlite?immutable=1", uri=True) as conn:
        out_path = sys.argv[1]
        drv_path = get_deriver(conn, out_path)
        _ = dump_build_deps(drv_path) # DRV_CACHEを事前に生成
        del(DRV_CACHE[drv_path]) # 出力が冗長になるので自身のdrvは削除しておく
        tree = dump_runtime_deps(out_path)
        show_tree(tree, True, "")

実行結果は、nix-store --query --requisitesないしnix-store --query --treeのものと同じである。

bombrary@nixos:~/deps$ nix run nixpkgs#python312 -- dump.py /nix/store/03167shkax5dxclnv6r3sd8waa6lq7ny-coreutils-full-9.3
├──/nix/store/j6mwswpa6zqhdm1lm2lv9iix3arn774g-glibc-2.38-27
│  ├──/nix/store/fhws3x2s9j5932r6ah660nsh41bkrq27-xgcc-12.3.0-libgcc
│  ├──/nix/store/j6mwswpa6zqhdm1lm2lv9iix3arn774g-glibc-2.38-27: cached
│  └──/nix/store/4h5isrbr87jjw69rgdnhi8psi7hhk5im-libidn2-2.3.4
│     ├──/nix/store/4h5isrbr87jjw69rgdnhi8psi7hhk5im-libidn2-2.3.4
│     │  ├──/nix/store/4h5isrbr87jjw69rgdnhi8psi7hhk5im-libidn2-2.3.4: cached
│     │  └──/nix/store/80dld61hbpvy1ay1sdwaqyy4jzhm48xx-libunistring-1.1
│     │     └──/nix/store/80dld61hbpvy1ay1sdwaqyy4jzhm48xx-libunistring-1.1
│     │        └──/nix/store/80dld61hbpvy1ay1sdwaqyy4jzhm48xx-libunistring-1.1: cached
│     └──/nix/store/80dld61hbpvy1ay1sdwaqyy4jzhm48xx-libunistring-1.1: cached
├──/nix/store/idwlqkj1z2cgjcijgnnxgyp0zgzpv7c5-attr-2.5.1
│  ├──/nix/store/j6mwswpa6zqhdm1lm2lv9iix3arn774g-glibc-2.38-27: cached
│  └──/nix/store/idwlqkj1z2cgjcijgnnxgyp0zgzpv7c5-attr-2.5.1
│     ├──/nix/store/j6mwswpa6zqhdm1lm2lv9iix3arn774g-glibc-2.38-27: cached
│     └──/nix/store/idwlqkj1z2cgjcijgnnxgyp0zgzpv7c5-attr-2.5.1: cached
├──/nix/store/wmsmw09x6l3kcl4ng3qs3ircj8h73si3-acl-2.3.1
│  ├──/nix/store/j6mwswpa6zqhdm1lm2lv9iix3arn774g-glibc-2.38-27: cached
│  ├──/nix/store/idwlqkj1z2cgjcijgnnxgyp0zgzpv7c5-attr-2.5.1: cached
│  └──/nix/store/wmsmw09x6l3kcl4ng3qs3ircj8h73si3-acl-2.3.1
│     ├──/nix/store/j6mwswpa6zqhdm1lm2lv9iix3arn774g-glibc-2.38-27: cached
│     ├──/nix/store/idwlqkj1z2cgjcijgnnxgyp0zgzpv7c5-attr-2.5.1: cached
│     └──/nix/store/wmsmw09x6l3kcl4ng3qs3ircj8h73si3-acl-2.3.1: cached
├──/nix/store/9vv53vzx4k988d51xfiq2p46fqrjshv0-gmp-with-cxx-6.3.0
│  ├──/nix/store/giyri337jb6sa1qyff6qp771qfq10yhf-gcc-12.3.0-lib
│  │  ├──/nix/store/giyri337jb6sa1qyff6qp771qfq10yhf-gcc-12.3.0-lib
│  │  │  ├──/nix/store/giyri337jb6sa1qyff6qp771qfq10yhf-gcc-12.3.0-lib: cached
│  │  │  ├──/nix/store/iyw6mm7a75i49h9szc0m08ynay1p7kka-gcc-12.3.0-libgcc
│  │  │  └──/nix/store/j6mwswpa6zqhdm1lm2lv9iix3arn774g-glibc-2.38-27: cached
│  │  ├──/nix/store/iyw6mm7a75i49h9szc0m08ynay1p7kka-gcc-12.3.0-libgcc
│  │  └──/nix/store/j6mwswpa6zqhdm1lm2lv9iix3arn774g-glibc-2.38-27: cached
│  ├──/nix/store/j6mwswpa6zqhdm1lm2lv9iix3arn774g-glibc-2.38-27: cached
│  └──/nix/store/9vv53vzx4k988d51xfiq2p46fqrjshv0-gmp-with-cxx-6.3.0: cached
└──/nix/store/l0rxwrg41k3lsdiybf8q0rf3nk430zr8-openssl-3.0.12
   ├──/nix/store/j6mwswpa6zqhdm1lm2lv9iix3arn774g-glibc-2.38-27: cached
   └──/nix/store/l0rxwrg41k3lsdiybf8q0rf3nk430zr8-openssl-3.0.12
      ├──/nix/store/j6mwswpa6zqhdm1lm2lv9iix3arn774g-glibc-2.38-27: cached
      └──/nix/store/l0rxwrg41k3lsdiybf8q0rf3nk430zr8-openssl-3.0.12: cached