This repository has no description
14 kB
357 lines
1"""Render an interactive HTML graph of the packages without maintainers in
2the system closures of a flake's NixOS/nix-darwin machines.
3
4Each machine is evaluated with nixpkgs' `maintainerless` problem switched to
5"warn" (via extendModules, so the flake itself is not changed and no
6derivation changes). The warnings name the unmaintained packages; the graph
7comes from `nix derivation show -r` on the system derivation, so build
8dependencies are included.
9
10Package meta (description, homepage, ...) is collected in the same evaluation
11by walking the derivation values the system is built from. Packages only
12referenced through strings (e.g. "${pkg}/bin/foo") are not reachable that way;
13for those the tool looks up the attribute by name in one extra evaluation.
14"""
15
16import argparse
17import concurrent.futures
18import json
19import os
20import re
21import subprocess
22import sys
23from collections import defaultdict
24from pathlib import Path
25
26# second group is "<file>:<line>", the package's meta.position
27WARNING = re.compile(
28 r"evaluation warning: Package '([^']+)' in (/nix/store/[^:\s]+:\d+) "
29 r"has the following problem: maintainerless"
30)
31MAX_MACHINES = 31 # the page stores machine membership in 32-bit masks
32
33# Nix function: the cheap, JSON-safe subset of a derivation's meta
34META_FN = """
35 metaOf = d:
36 let
37 m = d.meta or { };
38 str = v: if builtins.isString v then v else null;
39 first = v: if builtins.isList v then (if v == [ ] then null else builtins.head v) else v;
40 lic = x: if builtins.isAttrs x then x.spdxId or x.shortName or x.fullName or null else str x;
41 licenses = let l = m.license or [ ]; in if builtins.isList l then l else [ l ];
42 in
43 {
44 description = str (m.description or null);
45 homepage = str (first (m.homepage or null));
46 changelog = str (first (m.changelog or null));
47 license = builtins.filter (x: x != null) (map lic licenses);
48 vulnerabilities = builtins.filter builtins.isString (m.knownVulnerabilities or [ ]);
49 };
50"""
51
52EVAL_EXPR = """
53let
54 flake = builtins.getFlake %(flake)s;
55 base = flake.%(kind)sConfigurations.${%(machine)s};
56 sys = base.extendModules {
57 modules = [
58 { nixpkgs.config.problems.matchers = [ { kind = "maintainerless"; handler = "warn"; } ]; }
59 ];
60 };
61 top = if %(kind_str)s == "nixos" then sys.config.system.build.toplevel else sys.system;
62 %(meta_fn)s
63 isDrv = v: builtins.isAttrs v && (v.type or null) == "derivation";
64 # derivation values a derivation was given as attributes: directly, in lists, or one attrset deep.
65 # All of these were already forced to compute drvPath, so walking them is nearly free.
66 depsOf = d:
67 let
68 pick = v:
69 if isDrv v then [ v ]
70 else if builtins.isList v then builtins.concatMap pick v
71 else if builtins.isAttrs v then builtins.filter isDrv (builtins.attrValues v)
72 else [ ];
73 in
74 builtins.concatMap pick (builtins.attrValues (d.drvAttrs or { }));
75 closure = builtins.genericClosure {
76 startSet = [ { key = top.drvPath; d = top; } ];
77 operator = n: map (d: { key = d.drvPath; inherit d; }) (depsOf n.d);
78 };
79 unmaintained = builtins.filter
80 (n: (n.d.meta.maintainers or [ ]) == [ ] && (n.d.meta.teams or [ ]) == [ ]) closure;
81in
82{
83 drv = top.drvPath;
84 # warnings point into the source nixpkgs was evaluated from, which pkgs.path may only be a copy of
85 nixpkgs = sys.pkgs.hello.meta.position;
86 rev = sys.config.system.nixos.revision or sys.config.system.nixpkgsRevision or "";
87 meta = builtins.listToAttrs (map (n: { name = n.d.name; value = metaOf n.d; }) unmaintained);
88}
89"""
90
91# For packages the walk could not reach: try likely attribute paths, accept one whose name matches.
92LOOKUP_EXPR = """
93let
94 flake = builtins.getFlake %(flake)s;
95 pkgs = flake.%(kind)sConfigurations.${%(machine)s}.pkgs;
96 lib = pkgs.lib;
97 %(meta_fn)s
98 wanted = builtins.fromJSON %(wanted)s;
99 get = path: let r = builtins.tryEval (lib.attrByPath path null pkgs); in if r.success then r.value else null;
100 nameOf = v: let r = builtins.tryEval (v.name or null); in if r.success then r.value else null;
101 find = it:
102 let
103 p = builtins.parseDrvName it.name;
104 py = builtins.match "python3[.0-9]*-(.*)" p.name;
105 candidates =
106 lib.optional (it.byname != null) [ it.byname ]
107 ++ [ [ p.name ] [ "haskellPackages" p.name ] ]
108 ++ lib.optional (py != null) [ "python3Packages" (builtins.head py) ];
109 hits = builtins.filter (c: let v = get c; in lib.isDerivation v && nameOf v == it.name) candidates;
110 in
111 if hits == [ ] then null
112 else metaOf (get (builtins.head hits)) // { attr = lib.concatStringsSep "." (builtins.head hits); };
113in
114lib.filterAttrs (_: v: v != null) (builtins.listToAttrs (map (it: { name = it.name; value = find it; }) wanted))
115"""
116
117
118def log(msg):
119 print(msg, file=sys.stderr, flush=True)
120
121
122def nix_str(s):
123 return json.dumps(s) # a JSON string literal is a valid Nix string for these inputs
124
125
126def nix_eval_json(expr):
127 return subprocess.run(
128 ["nix", "eval", "--impure", "--json", "--expr", expr],
129 capture_output=True,
130 text=True,
131 )
132
133
134def discover(flake):
135 expr = (
136 f"let f = builtins.getFlake {nix_str(flake)}; in {{"
137 " nixos = builtins.attrNames (f.nixosConfigurations or { });"
138 " darwin = builtins.attrNames (f.darwinConfigurations or { }); }"
139 )
140 proc = nix_eval_json(expr)
141 if proc.returncode != 0:
142 sys.exit(f"could not list machines of {flake}:\n{proc.stderr[-2000:]}")
143 found = json.loads(proc.stdout)
144 return [(m, "nixos") for m in found["nixos"]] + [(m, "darwin") for m in found["darwin"]]
145
146
147def evaluate(flake, machine, kind):
148 expr = EVAL_EXPR % {
149 "flake": nix_str(flake),
150 "kind": kind,
151 "kind_str": nix_str(kind),
152 "machine": nix_str(machine),
153 "meta_fn": META_FN,
154 }
155 proc = nix_eval_json(expr)
156 if proc.returncode != 0:
157 lines = proc.stderr.strip().splitlines()
158 raise RuntimeError(lines[-1] if lines else "evaluation failed")
159 result = json.loads(proc.stdout)
160 result["warned"] = sorted(set(WARNING.findall(proc.stderr)))
161 return result
162
163
164def lookup_meta(flake, machine, kind, warned):
165 """Meta for the packages the closure walk missed, via attribute lookup on one machine's pkgs."""
166 wanted = []
167 for name, path in warned.items():
168 m = re.search(r"/pkgs/by-name/[^/]+/([^/]+)/package\.nix:\d+$", path)
169 wanted.append({"name": name, "byname": m.group(1) if m else None})
170 expr = LOOKUP_EXPR % {
171 "flake": nix_str(flake),
172 "kind": kind,
173 "machine": nix_str(machine),
174 "meta_fn": META_FN,
175 "wanted": nix_str(json.dumps(wanted)),
176 }
177 proc = nix_eval_json(expr)
178 if proc.returncode != 0:
179 lines = proc.stderr.strip().splitlines()
180 log(f"[meta] lookup failed, continuing without: {lines[-1] if lines else '?'}")
181 return {}
182 return json.loads(proc.stdout)
183
184
185def derivation_graph(drv):
186 """{drv basename: (name, [input drv basenames])} for the whole build closure of drv."""
187 proc = subprocess.run(["nix", "derivation", "show", "-r", drv], capture_output=True, text=True)
188 if proc.returncode != 0:
189 raise RuntimeError(proc.stderr.strip()[-500:])
190 data = json.loads(proc.stdout)
191 data = data.get("derivations", data) # newer nix nests them
192 graph = {}
193 for path, d in data.items():
194 inputs = d.get("inputDrvs") or d.get("inputs", {}).get("drvs") or {}
195 graph[os.path.basename(path)] = (d["name"], [os.path.basename(p) for p in inputs])
196 return graph
197
198
199def dependents_counts(children):
200 """How many nodes transitively depend on each node (bitset DP in topological order)."""
201 n = len(children)
202 indeg = [0] * n
203 for cs in children:
204 for c in cs:
205 indeg[c] += 1
206 order = [i for i in range(n) if indeg[i] == 0]
207 for i in order: # grows while iterating
208 for c in children[i]:
209 indeg[c] -= 1
210 if indeg[c] == 0:
211 order.append(c)
212 anc = [0] * n # bitset of strict ancestors
213 for i in order:
214 a = anc[i] | (1 << i)
215 for c in children[i]:
216 anc[c] |= a
217 return [bin(a).count("1") for a in anc]
218
219
220def file_entry(position, sources):
221 """[file:line to show, GitHub link to that line or None] for a warning's meta.position."""
222 path, _, line = position.rpartition(":")
223 for src, rev in sources.items():
224 if path.startswith(src + "/"):
225 rel = path[len(src) + 1 :]
226 link = re.fullmatch(r"[0-9a-f]{7,40}", rev or "")
227 url = f"https://github.com/NixOS/nixpkgs/blob/{rev}/{rel}#L{line}" if link else None
228 return [f"{rel}:{line}", url]
229 m = re.match(r"/nix/store/[^/]+/(.*)", path)
230 return [f"{m.group(1) if m else path}:{line} (not nixpkgs)", None]
231
232
233def build_data(title, results, graphs, meta):
234 mach = sorted(results)
235 # "<nixpkgs>/pkgs/by-name/he/hello/package.nix:42" -> "<nixpkgs>"
236 sources = {r["nixpkgs"].rsplit("/pkgs/", 1)[0]: r["rev"] for r in results.values()}
237 warned = {name: path for m in mach for name, path in results[m]["warned"]}
238
239 key2i, names, keys, parents, mask = {}, [], [], [], []
240 direct = defaultdict(int) # node -> machines where it is a direct input of system-path
241 dep = defaultdict(dict) # node -> {machine index: dependents}
242 roots = []
243
244 def nid(key, name):
245 i = key2i.get(key)
246 if i is None:
247 i = key2i[key] = len(names)
248 names.append(name)
249 keys.append(key)
250 parents.append(set())
251 mask.append(0)
252 return i
253
254 for mi, m in enumerate(mach):
255 g = graphs[m]
256 local = list(g)
257 loc = {k: j for j, k in enumerate(local)}
258 gid = [nid(k, g[k][0]) for k in local]
259 children = [[loc[c] for c in g[k][1] if c in loc] for k in local]
260 for j, cs in enumerate(children):
261 mask[gid[j]] |= 1 << mi
262 for c in cs:
263 parents[gid[c]].add(gid[j])
264 if g[local[j]][0] == "system-path":
265 for c in cs:
266 direct[gid[c]] |= 1 << mi
267 for j, count in enumerate(dependents_counts(children)):
268 dep[gid[j]][mi] = count
269 roots.append(key2i[os.path.basename(results[m]["drv"])])
270
271 unmaintained = [i for i, n in enumerate(names) if n in warned]
272 data = {
273 "title": title,
274 "machines": mach,
275 "roots": roots,
276 "names": names,
277 "hash": [k[:8] for k in keys],
278 "mask": mask,
279 "direct": {str(i): v for i, v in direct.items()},
280 "dep": [max(dep[i].values()) for i in range(len(names))],
281 "udep": {str(i): [dep[i].get(k, -1) for k in range(len(mach))] for i in unmaintained},
282 "file": {str(i): file_entry(warned[names[i]], sources) for i in unmaintained},
283 "parents": [sorted(p) for p in parents],
284 "meta": {n: meta[n] for n in {names[i] for i in unmaintained} if n in meta},
285 }
286 return data, len({names[i] for i in unmaintained})
287
288
289def main():
290 ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
291 ap.add_argument("--flake", default=".", help="flake to inspect (default: .)")
292 ap.add_argument(
293 "-m",
294 "--machine",
295 action="append",
296 default=[],
297 help="only this machine (repeatable; default: all nixos/darwin configurations)",
298 )
299 ap.add_argument("-o", "--output", default="unmaintained-graph.html", help="output HTML file")
300 ap.add_argument("-j", "--jobs", type=int, default=4, help="parallel evaluations (default: 4)")
301 ap.add_argument("--title", help="page title (default: derived from the flake)")
302 args = ap.parse_args()
303
304 flake = args.flake
305 if os.path.exists(flake):
306 flake = str(Path(flake).resolve())
307 machines = discover(flake)
308 if args.machine:
309 known = {m for m, _ in machines}
310 missing = [m for m in args.machine if m not in known]
311 if missing:
312 sys.exit(f"unknown machine(s): {', '.join(missing)}")
313 machines = [(m, k) for m, k in machines if m in args.machine]
314 if not machines:
315 sys.exit("no nixosConfigurations/darwinConfigurations found")
316 if len(machines) > MAX_MACHINES:
317 sys.exit(f"{len(machines)} machines, at most {MAX_MACHINES} supported; pick some with --machine")
318
319 results = {}
320 with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool:
321 futures = {pool.submit(evaluate, flake, m, k): m for m, k in machines}
322 for fut in concurrent.futures.as_completed(futures):
323 m = futures[fut]
324 try:
325 results[m] = fut.result()
326 except Exception as e: # report and keep going with the other machines
327 log(f"[skip] {m}: {e}")
328 continue
329 log(f"[{len(results)}/{len(machines)}] {m}: {len(results[m]['warned'])} unmaintained")
330 if not results:
331 sys.exit("every evaluation failed")
332
333 warned = {name: path for r in results.values() for name, path in r["warned"]}
334 meta = {}
335 for r in results.values():
336 meta.update((n, v) for n, v in r.pop("meta").items() if n in warned)
337 missing = {n: p for n, p in warned.items() if n not in meta}
338 if missing:
339 first = next(m for m, _ in machines if m in results)
340 meta.update(lookup_meta(flake, first, dict(machines)[first], missing))
341 log(f"[meta] {len(meta)}/{len(warned)} packages with meta")
342
343 with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool:
344 graphs = dict(zip(results, pool.map(lambda m: derivation_graph(results[m]["drv"]), results)))
345
346 title = args.title or f"Unmaintained packages in {Path(flake).name if os.path.isabs(flake) else flake}"
347 data, count = build_data(title, results, graphs, meta)
348
349 template = Path(os.environ["UNMAINTAINED_GRAPH_TEMPLATE"]).read_text()
350 d3 = Path(os.environ["UNMAINTAINED_GRAPH_D3"]).read_text()
351 payload = json.dumps(data, separators=(",", ":")).replace("</", "<\\/")
352 Path(args.output).write_text(template.replace("/*D3*/", d3, 1).replace("/*DATA*/", payload, 1))
353 log(f"wrote {args.output}: {count} unmaintained packages across {len(results)} machine(s)")
354
355
356if __name__ == "__main__":
357 main()