This repository has no description
16 kB
400 lines
1"""Render an interactive HTML graph of the packages without maintainers in
2the system closures of a flake's NixOS/nix-darwin machines.
3
4Each machine is evaluated with nixpkgs' `maintainerless` problem switched to
5"warn" (via extendModules, so the flake itself is not changed and no
6derivation changes). The warnings name the unmaintained packages; the graph
7comes from `nix derivation show -r` on the system derivation, so build
8dependencies are included.
9
10Package meta (description, homepage, ...) is collected in the same evaluation
11by walking the derivation values the system is built from. Packages only
12referenced through strings (e.g. "${pkg}/bin/foo") are not reachable that way;
13for those the tool looks up the attribute by name in one extra evaluation.
14"""
15
16import argparse
17import concurrent.futures
18import json
19import os
20import re
21import subprocess
22import tempfile
23import sys
24from collections import defaultdict
25from pathlib import Path
26
27# second group is "<file>:<line>", the package's meta.position
28WARNING = re.compile(
29 r"evaluation warning: Package '([^']+)' in (/nix/store/[^:\s]+:\d+) "
30 r"has the following problem: maintainerless"
31)
32MAX_MACHINES = 31 # the page stores machine membership in 32-bit masks
33
34# Nix function: the cheap, JSON-safe subset of a derivation's meta
35META_FN = """
36 metaOf = d:
37 let
38 m = d.meta or { };
39 str = v: if builtins.isString v then v else null;
40 first = v: if builtins.isList v then (if v == [ ] then null else builtins.head v) else v;
41 lic = x: if builtins.isAttrs x then x.spdxId or x.shortName or x.fullName or null else str x;
42 licenses = let l = m.license or [ ]; in if builtins.isList l then l else [ l ];
43 in
44 {
45 description = str (m.description or null);
46 homepage = str (first (m.homepage or null));
47 changelog = str (first (m.changelog or null));
48 license = builtins.filter (x: x != null) (map lic licenses);
49 vulnerabilities = builtins.filter builtins.isString (m.knownVulnerabilities or [ ]);
50 maintainers = builtins.filter builtins.isString
51 (map (x: if builtins.isAttrs x then x.github or x.name or null else null) (m.maintainers or [ ]));
52 teams = builtins.filter builtins.isString
53 (map (t: if builtins.isAttrs t then t.shortName or t.name or null else null) (m.teams or [ ]));
54 position = str (m.position or null);
55 };
56"""
57
58EVAL_EXPR = """
59let
60 flake = builtins.getFlake %(flake)s;
61 base = flake.%(kind)sConfigurations.${%(machine)s};
62 sys = base.extendModules {
63 modules = [
64 { nixpkgs.config.problems.matchers = [ { kind = "maintainerless"; handler = "warn"; } ]; }
65 ];
66 };
67 top = if %(kind_str)s == "nixos" then sys.config.system.build.toplevel else sys.system;
68 %(meta_fn)s
69 isDrv = v: builtins.isAttrs v && (v.type or null) == "derivation";
70 # derivation values a derivation was given as attributes: directly, in lists, or one attrset deep.
71 # All of these were already forced to compute drvPath, so walking them is nearly free.
72 depsOf = d:
73 let
74 pick = v:
75 if isDrv v then [ v ]
76 else if builtins.isList v then builtins.concatMap pick v
77 else if builtins.isAttrs v then builtins.filter isDrv (builtins.attrValues v)
78 else [ ];
79 in
80 builtins.concatMap pick (builtins.attrValues (d.drvAttrs or { }));
81 closure = builtins.genericClosure {
82 startSet = [ { key = top.drvPath; d = top; } ];
83 operator = n: map (d: { key = d.drvPath; inherit d; }) (depsOf n.d);
84 };
85 # generated files (units, etc entries, ...) carry no package meta; leave them out
86 packages = builtins.filter (n: n.d ? meta.description || (n.d.meta.maintainers or [ ]) != [ ]) closure;
87in
88{
89 drv = top.drvPath;
90 # warnings point into the source nixpkgs was evaluated from, which pkgs.path may only be a copy of
91 nixpkgs = sys.pkgs.hello.meta.position;
92 rev = sys.config.system.nixos.revision or sys.config.system.nixpkgsRevision or "";
93 meta = builtins.listToAttrs (map (n: { name = n.d.name; value = metaOf n.d; }) packages);
94}
95"""
96
97# For packages the walk could not reach: try likely attribute paths. Prefer an attribute with exactly
98# this name; otherwise accept the same package in another version (meta is per package, not version).
99LOOKUP_EXPR = """
100let
101 flake = builtins.getFlake %(flake)s;
102 pkgs = flake.%(kind)sConfigurations.${%(machine)s}.pkgs;
103 lib = pkgs.lib;
104 %(meta_fn)s
105 wanted = builtins.fromJSON %(wanted)s;
106 get = path: let r = builtins.tryEval (lib.attrByPath path null pkgs); in if r.success then r.value else null;
107 nameOf = v: let r = builtins.tryEval (v.name or null); in if r.success then r.value else null;
108 find = it:
109 let
110 p = builtins.parseDrvName it.name;
111 py = builtins.match "python3[.]?([0-9]*)-(.*)" p.name;
112 perl = builtins.match "perl[.0-9]*-(.*)" p.name;
113 candidates =
114 lib.optional (it.byname != null) [ it.byname ]
115 ++ [ [ p.name ] [ "${p.name}-unwrapped" ] [ "haskellPackages" p.name ] ]
116 ++ lib.optionals (py != null) [
117 [ "python3${builtins.head py}Packages" (lib.last py) ]
118 [ "python3Packages" (lib.last py) ]
119 ]
120 ++ lib.optional (perl != null) [ "perlPackages" (builtins.replaceStrings [ "-" ] [ "" ] (builtins.head perl)) ];
121 found = map (c: { path = c; name = let v = get c; in if lib.isDerivation v then nameOf v else null; }) candidates;
122 exact = builtins.filter (c: c.name == it.name) found;
123 samePackage = builtins.filter (c: c.name != null && (builtins.parseDrvName c.name).name == p.name) found;
124 hit = if exact != [ ] then builtins.head exact else if samePackage != [ ] then builtins.head samePackage else null;
125 in
126 if hit == null then null
127 else metaOf (get hit.path) // {
128 attr = lib.concatStringsSep "." hit.path;
129 otherVersion = if hit.name == it.name then null else hit.name;
130 };
131in
132lib.filterAttrs (_: v: v != null) (builtins.listToAttrs (map (it: { name = it.name; value = find it; }) wanted))
133"""
134
135
136def log(msg):
137 print(msg, file=sys.stderr, flush=True)
138
139
140def nix_str(s):
141 return json.dumps(s) # a JSON string literal is a valid Nix string for these inputs
142
143
144def nix_eval_json(expr):
145 return subprocess.run(
146 ["nix", "eval", "--impure", "--json", "--expr", expr],
147 capture_output=True,
148 text=True,
149 )
150
151
152def discover(flake):
153 expr = (
154 f"let f = builtins.getFlake {nix_str(flake)}; in {{"
155 " nixos = builtins.attrNames (f.nixosConfigurations or { });"
156 " darwin = builtins.attrNames (f.darwinConfigurations or { }); }"
157 )
158 proc = nix_eval_json(expr)
159 if proc.returncode != 0:
160 sys.exit(f"could not list machines of {flake}:\n{proc.stderr[-2000:]}")
161 found = json.loads(proc.stdout)
162 return [(m, "nixos") for m in found["nixos"]] + [(m, "darwin") for m in found["darwin"]]
163
164
165def evaluate(flake, machine, kind):
166 expr = EVAL_EXPR % {
167 "flake": nix_str(flake),
168 "kind": kind,
169 "kind_str": nix_str(kind),
170 "machine": nix_str(machine),
171 "meta_fn": META_FN,
172 }
173 proc = nix_eval_json(expr)
174 if proc.returncode != 0:
175 lines = proc.stderr.strip().splitlines()
176 raise RuntimeError(lines[-1] if lines else "evaluation failed")
177 result = json.loads(proc.stdout)
178 result["warned"] = sorted(set(WARNING.findall(proc.stderr)))
179 return result
180
181
182def lookup_meta(flake, machine, kind, names):
183 """Meta for packages the closure walk missed, via attribute lookup on one machine's pkgs.
184
185 `names` maps package name -> warning position ("" if none), used to find by-name attributes.
186 """
187 wanted = []
188 for name, position in names.items():
189 m = re.search(r"/pkgs/by-name/[^/]+/([^/]+)/package\.nix:\d+$", position)
190 wanted.append({"name": name, "byname": m.group(1) if m else None})
191 # tens of thousands of names do not fit on a command line
192 with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
193 json.dump(wanted, f)
194 f.flush()
195 expr = LOOKUP_EXPR % {
196 "flake": nix_str(flake),
197 "kind": kind,
198 "machine": nix_str(machine),
199 "meta_fn": META_FN,
200 "wanted": f"(builtins.readFile {nix_str(f.name)})",
201 }
202 proc = nix_eval_json(expr)
203 if proc.returncode != 0:
204 lines = proc.stderr.strip().splitlines()
205 log(f"[meta] lookup failed, continuing without: {lines[-1] if lines else '?'}")
206 return {}
207 return json.loads(proc.stdout)
208
209
210def derivation_graph(drv):
211 """{drv basename: (name, [input drv basenames])} for the whole build closure of drv."""
212 proc = subprocess.run(["nix", "derivation", "show", "-r", drv], capture_output=True, text=True)
213 if proc.returncode != 0:
214 raise RuntimeError(proc.stderr.strip()[-500:])
215 data = json.loads(proc.stdout)
216 data = data.get("derivations", data) # newer nix nests them
217 graph = {}
218 for path, d in data.items():
219 inputs = d.get("inputDrvs") or d.get("inputs", {}).get("drvs") or {}
220 graph[os.path.basename(path)] = (d["name"], [os.path.basename(p) for p in inputs])
221 return graph
222
223
224def dependents_counts(children):
225 """How many nodes transitively depend on each node (bitset DP in topological order)."""
226 n = len(children)
227 indeg = [0] * n
228 for cs in children:
229 for c in cs:
230 indeg[c] += 1
231 order = [i for i in range(n) if indeg[i] == 0]
232 for i in order: # grows while iterating
233 for c in children[i]:
234 indeg[c] -= 1
235 if indeg[c] == 0:
236 order.append(c)
237 anc = [0] * n # bitset of strict ancestors
238 for i in order:
239 a = anc[i] | (1 << i)
240 for c in children[i]:
241 anc[c] |= a
242 return [bin(a).count("1") for a in anc]
243
244
245def file_entry(position, sources):
246 """[file:line to show, GitHub link to that line or None] for a warning's meta.position."""
247 path, _, line = position.rpartition(":")
248 for src, rev in sources.items():
249 if path.startswith(src + "/"):
250 rel = path[len(src) + 1 :]
251 link = re.fullmatch(r"[0-9a-f]{7,40}", rev or "")
252 url = f"https://github.com/NixOS/nixpkgs/blob/{rev}/{rel}#L{line}" if link else None
253 return [f"{rel}:{line}", url]
254 m = re.match(r"/nix/store/[^/]+/(.*)", path)
255 return [f"{m.group(1) if m else path}:{line} (not nixpkgs)", None]
256
257
258def build_data(title, results, graphs, meta):
259 mach = sorted(results)
260 # "<nixpkgs>/pkgs/by-name/he/hello/package.nix:42" -> "<nixpkgs>"
261 sources = {r["nixpkgs"].rsplit("/pkgs/", 1)[0]: r["rev"] for r in results.values()}
262 warned = {name: path for m in mach for name, path in results[m]["warned"]}
263
264 key2i, names, keys, parents, mask = {}, [], [], [], []
265 direct = defaultdict(int) # node -> machines where it is a direct input of system-path
266 dep = defaultdict(dict) # node -> {machine index: dependents}
267 roots = []
268
269 def nid(key, name):
270 i = key2i.get(key)
271 if i is None:
272 i = key2i[key] = len(names)
273 names.append(name)
274 keys.append(key)
275 parents.append(set())
276 mask.append(0)
277 return i
278
279 for mi, m in enumerate(mach):
280 g = graphs[m]
281 local = list(g)
282 loc = {k: j for j, k in enumerate(local)}
283 gid = [nid(k, g[k][0]) for k in local]
284 children = [[loc[c] for c in g[k][1] if c in loc] for k in local]
285 for j, cs in enumerate(children):
286 mask[gid[j]] |= 1 << mi
287 for c in cs:
288 parents[gid[c]].add(gid[j])
289 if g[local[j]][0] == "system-path":
290 for c in cs:
291 direct[gid[c]] |= 1 << mi
292 for j, count in enumerate(dependents_counts(children)):
293 dep[gid[j]][mi] = count
294 roots.append(key2i[os.path.basename(results[m]["drv"])])
295
296 unmaintained = [i for i, n in enumerate(names) if n in warned]
297 data = {
298 "title": title,
299 "machines": mach,
300 "roots": roots,
301 "names": names,
302 "hash": [k[:8] for k in keys],
303 "mask": mask,
304 "direct": {str(i): v for i, v in direct.items()},
305 "dep": [max(dep[i].values()) for i in range(len(names))],
306 "udep": {str(i): [dep[i].get(k, -1) for k in range(len(mach))] for i in unmaintained},
307 "file": {str(i): file_entry(warned[names[i]], sources) for i in unmaintained},
308 "parents": [sorted(p) for p in parents],
309 "meta": {},
310 }
311 # meta for every package in the graph, with its definition linked like the unmaintained ones
312 for n in set(names):
313 m = meta.get(n)
314 if m is None:
315 continue
316 m = dict(m)
317 position = m.pop("position", None)
318 if position:
319 m["file"] = file_entry(position, sources)
320 data["meta"][n] = m
321 unmaintained_names = {names[i] for i in unmaintained}
322 with_meta = sum(1 for n in unmaintained_names if n in data["meta"])
323 log(
324 f"[meta] {with_meta}/{len(unmaintained_names)} unmaintained and "
325 f"{len(data['meta']) - with_meta} other packages with meta"
326 )
327 return data, len(unmaintained_names)
328
329
330def main():
331 ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
332 ap.add_argument("--flake", default=".", help="flake to inspect (default: .)")
333 ap.add_argument(
334 "-m",
335 "--machine",
336 action="append",
337 default=[],
338 help="only this machine (repeatable; default: all nixos/darwin configurations)",
339 )
340 ap.add_argument("-o", "--output", default="unmaintained-graph.html", help="output HTML file")
341 ap.add_argument("-j", "--jobs", type=int, default=4, help="parallel evaluations (default: 4)")
342 ap.add_argument("--title", help="page title (default: derived from the flake)")
343 args = ap.parse_args()
344
345 flake = args.flake
346 if os.path.exists(flake):
347 flake = str(Path(flake).resolve())
348 machines = discover(flake)
349 if args.machine:
350 known = {m for m, _ in machines}
351 missing = [m for m in args.machine if m not in known]
352 if missing:
353 sys.exit(f"unknown machine(s): {', '.join(missing)}")
354 machines = [(m, k) for m, k in machines if m in args.machine]
355 if not machines:
356 sys.exit("no nixosConfigurations/darwinConfigurations found")
357 if len(machines) > MAX_MACHINES:
358 sys.exit(f"{len(machines)} machines, at most {MAX_MACHINES} supported; pick some with --machine")
359
360 results = {}
361 with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool:
362 futures = {pool.submit(evaluate, flake, m, k): m for m, k in machines}
363 for fut in concurrent.futures.as_completed(futures):
364 m = futures[fut]
365 try:
366 results[m] = fut.result()
367 except Exception as e: # report and keep going with the other machines
368 log(f"[skip] {m}: {e}")
369 continue
370 log(f"[{len(results)}/{len(machines)}] {m}: {len(results[m]['warned'])} unmaintained")
371 if not results:
372 sys.exit("every evaluation failed")
373
374 warned = {name: path for r in results.values() for name, path in r["warned"]}
375 meta = {}
376 for r in results.values():
377 for n, v in r.pop("meta").items():
378 meta.setdefault(n, v)
379
380 with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool:
381 graphs = dict(zip(results, pool.map(lambda m: derivation_graph(results[m]["drv"]), results)))
382
383 # everything in the graph the walk did not reach; names that are no attribute just miss cheaply
384 missing = {n: warned.get(n, "") for g in graphs.values() for n, _ in g.values() if n not in meta}
385 if missing:
386 first = next(m for m, _ in machines if m in results)
387 meta.update(lookup_meta(flake, first, dict(machines)[first], missing))
388
389 title = args.title or f"Unmaintained packages in {Path(flake).name if os.path.isabs(flake) else flake}"
390 data, count = build_data(title, results, graphs, meta)
391
392 template = Path(os.environ["UNMAINTAINED_GRAPH_TEMPLATE"]).read_text()
393 d3 = Path(os.environ["UNMAINTAINED_GRAPH_D3"]).read_text()
394 payload = json.dumps(data, separators=(",", ":")).replace("</", "<\\/")
395 Path(args.output).write_text(template.replace("/*D3*/", d3, 1).replace("/*DATA*/", payload, 1))
396 log(f"wrote {args.output}: {count} unmaintained packages across {len(results)} machine(s)")
397
398
399if __name__ == "__main__":
400 main()