This repository has no description
9.5 kB
251 lines
1"""Render an interactive HTML graph of the packages without maintainers in
2the system closures of a flake's NixOS/nix-darwin machines.
3
4Each machine is evaluated with nixpkgs' `maintainerless` problem switched to
5"warn" (via extendModules, so the flake itself is not changed and no
6derivation changes). The warnings name the unmaintained packages; the graph
7comes from `nix derivation show -r` on the system derivation, so build
8dependencies are included.
9"""
10
11import argparse
12import concurrent.futures
13import json
14import os
15import re
16import subprocess
17import sys
18from collections import defaultdict
19from pathlib import Path
20
21WARNING = re.compile(
22 r"evaluation warning: Package '([^']+)' in (/nix/store/[^:\s]+):\d+ "
23 r"has the following problem: maintainerless"
24)
25MAX_MACHINES = 31 # the page stores machine membership in 32-bit masks
26
27EVAL_EXPR = """
28let
29 flake = builtins.getFlake %(flake)s;
30 base = flake.%(kind)sConfigurations.${%(machine)s};
31 sys = base.extendModules {
32 modules = [
33 { nixpkgs.config.problems.matchers = [ { kind = "maintainerless"; handler = "warn"; } ]; }
34 ];
35 };
36 top = if %(kind_str)s == "nixos" then sys.config.system.build.toplevel else sys.system;
37in
38{
39 drv = top.drvPath;
40 # warnings point into the source nixpkgs was evaluated from, which pkgs.path may only be a copy of
41 nixpkgs = sys.pkgs.hello.meta.position;
42 rev = sys.config.system.nixos.revision or sys.config.system.nixpkgsRevision or "";
43}
44"""
45
46
47def log(msg):
48 print(msg, file=sys.stderr, flush=True)
49
50
51def nix_str(s):
52 return json.dumps(s) # a JSON string literal is a valid Nix string for these inputs
53
54
55def nix_eval_json(expr):
56 return subprocess.run(
57 ["nix", "eval", "--impure", "--json", "--expr", expr],
58 capture_output=True,
59 text=True,
60 )
61
62
63def discover(flake):
64 expr = (
65 f"let f = builtins.getFlake {nix_str(flake)}; in {{"
66 " nixos = builtins.attrNames (f.nixosConfigurations or { });"
67 " darwin = builtins.attrNames (f.darwinConfigurations or { }); }"
68 )
69 proc = nix_eval_json(expr)
70 if proc.returncode != 0:
71 sys.exit(f"could not list machines of {flake}:\n{proc.stderr[-2000:]}")
72 found = json.loads(proc.stdout)
73 return [(m, "nixos") for m in found["nixos"]] + [(m, "darwin") for m in found["darwin"]]
74
75
76def evaluate(flake, machine, kind):
77 expr = EVAL_EXPR % {
78 "flake": nix_str(flake),
79 "kind": kind,
80 "kind_str": nix_str(kind),
81 "machine": nix_str(machine),
82 }
83 proc = nix_eval_json(expr)
84 if proc.returncode != 0:
85 lines = proc.stderr.strip().splitlines()
86 raise RuntimeError(lines[-1] if lines else "evaluation failed")
87 result = json.loads(proc.stdout)
88 result["warned"] = sorted(set(WARNING.findall(proc.stderr)))
89 return result
90
91
92def derivation_graph(drv):
93 """{drv basename: (name, [input drv basenames])} for the whole build closure of drv."""
94 proc = subprocess.run(["nix", "derivation", "show", "-r", drv], capture_output=True, text=True)
95 if proc.returncode != 0:
96 raise RuntimeError(proc.stderr.strip()[-500:])
97 data = json.loads(proc.stdout)
98 data = data.get("derivations", data) # newer nix nests them
99 graph = {}
100 for path, d in data.items():
101 inputs = d.get("inputDrvs") or d.get("inputs", {}).get("drvs") or {}
102 graph[os.path.basename(path)] = (d["name"], [os.path.basename(p) for p in inputs])
103 return graph
104
105
106def dependents_counts(children):
107 """How many nodes transitively depend on each node (bitset DP in topological order)."""
108 n = len(children)
109 indeg = [0] * n
110 for cs in children:
111 for c in cs:
112 indeg[c] += 1
113 order = [i for i in range(n) if indeg[i] == 0]
114 for i in order: # grows while iterating
115 for c in children[i]:
116 indeg[c] -= 1
117 if indeg[c] == 0:
118 order.append(c)
119 anc = [0] * n # bitset of strict ancestors
120 for i in order:
121 a = anc[i] | (1 << i)
122 for c in children[i]:
123 anc[c] |= a
124 return [bin(a).count("1") for a in anc]
125
126
127def file_entry(path, sources):
128 """[path to show, GitHub link or None] for the file a warning points at."""
129 for src, rev in sources.items():
130 if path.startswith(src + "/"):
131 rel = path[len(src) + 1 :]
132 link = re.fullmatch(r"[0-9a-f]{7,40}", rev or "")
133 return [rel, f"https://github.com/NixOS/nixpkgs/blob/{rev}/{rel}" if link else None]
134 m = re.match(r"/nix/store/[^/]+/(.*)", path)
135 return [(m.group(1) if m else path) + " (not nixpkgs)", None]
136
137
138def build_data(title, results, graphs):
139 mach = sorted(results)
140 # "<nixpkgs>/pkgs/by-name/he/hello/package.nix:42" -> "<nixpkgs>"
141 sources = {r["nixpkgs"].rsplit("/pkgs/", 1)[0]: r["rev"] for r in results.values()}
142 warned = {name: path for m in mach for name, path in results[m]["warned"]}
143
144 key2i, names, keys, parents, mask = {}, [], [], [], []
145 direct = defaultdict(int) # node -> machines where it is a direct input of system-path
146 dep = defaultdict(dict) # node -> {machine index: dependents}
147 roots = []
148
149 def nid(key, name):
150 i = key2i.get(key)
151 if i is None:
152 i = key2i[key] = len(names)
153 names.append(name)
154 keys.append(key)
155 parents.append(set())
156 mask.append(0)
157 return i
158
159 for mi, m in enumerate(mach):
160 g = graphs[m]
161 local = list(g)
162 loc = {k: j for j, k in enumerate(local)}
163 gid = [nid(k, g[k][0]) for k in local]
164 children = [[loc[c] for c in g[k][1] if c in loc] for k in local]
165 for j, cs in enumerate(children):
166 mask[gid[j]] |= 1 << mi
167 for c in cs:
168 parents[gid[c]].add(gid[j])
169 if g[local[j]][0] == "system-path":
170 for c in cs:
171 direct[gid[c]] |= 1 << mi
172 for j, count in enumerate(dependents_counts(children)):
173 dep[gid[j]][mi] = count
174 roots.append(key2i[os.path.basename(results[m]["drv"])])
175
176 unmaintained = [i for i, n in enumerate(names) if n in warned]
177 data = {
178 "title": title,
179 "machines": mach,
180 "roots": roots,
181 "names": names,
182 "hash": [k[:8] for k in keys],
183 "mask": mask,
184 "direct": {str(i): v for i, v in direct.items()},
185 "dep": [max(dep[i].values()) for i in range(len(names))],
186 "udep": {str(i): [dep[i].get(k, -1) for k in range(len(mach))] for i in unmaintained},
187 "file": {str(i): file_entry(warned[names[i]], sources) for i in unmaintained},
188 "parents": [sorted(p) for p in parents],
189 }
190 return data, len({names[i] for i in unmaintained})
191
192
193def main():
194 ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
195 ap.add_argument("--flake", default=".", help="flake to inspect (default: .)")
196 ap.add_argument(
197 "-m",
198 "--machine",
199 action="append",
200 default=[],
201 help="only this machine (repeatable; default: all nixos/darwin configurations)",
202 )
203 ap.add_argument("-o", "--output", default="unmaintained-graph.html", help="output HTML file")
204 ap.add_argument("-j", "--jobs", type=int, default=4, help="parallel evaluations (default: 4)")
205 ap.add_argument("--title", help="page title (default: derived from the flake)")
206 args = ap.parse_args()
207
208 flake = args.flake
209 if os.path.exists(flake):
210 flake = str(Path(flake).resolve())
211 machines = discover(flake)
212 if args.machine:
213 known = {m for m, _ in machines}
214 missing = [m for m in args.machine if m not in known]
215 if missing:
216 sys.exit(f"unknown machine(s): {', '.join(missing)}")
217 machines = [(m, k) for m, k in machines if m in args.machine]
218 if not machines:
219 sys.exit("no nixosConfigurations/darwinConfigurations found")
220 if len(machines) > MAX_MACHINES:
221 sys.exit(f"{len(machines)} machines, at most {MAX_MACHINES} supported; pick some with --machine")
222
223 results = {}
224 with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool:
225 futures = {pool.submit(evaluate, flake, m, k): m for m, k in machines}
226 for fut in concurrent.futures.as_completed(futures):
227 m = futures[fut]
228 try:
229 results[m] = fut.result()
230 except Exception as e: # report and keep going with the other machines
231 log(f"[skip] {m}: {e}")
232 continue
233 log(f"[{len(results)}/{len(machines)}] {m}: {len(results[m]['warned'])} unmaintained")
234 if not results:
235 sys.exit("every evaluation failed")
236
237 with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool:
238 graphs = dict(zip(results, pool.map(lambda m: derivation_graph(results[m]["drv"]), results)))
239
240 title = args.title or f"Unmaintained packages in {Path(flake).name if os.path.isabs(flake) else flake}"
241 data, count = build_data(title, results, graphs)
242
243 template = Path(os.environ["UNMAINTAINED_GRAPH_TEMPLATE"]).read_text()
244 d3 = Path(os.environ["UNMAINTAINED_GRAPH_D3"]).read_text()
245 payload = json.dumps(data, separators=(",", ":")).replace("</", "<\\/")
246 Path(args.output).write_text(template.replace("/*D3*/", d3, 1).replace("/*DATA*/", payload, 1))
247 log(f"wrote {args.output}: {count} unmaintained packages across {len(results)} machine(s)")
248
249
250if __name__ == "__main__":
251 main()