+25 -4

tools/unmaintained-graph/graph.html

View file Reviewed

··· 134 134 border-left: 1px solid var(--line); 135 135 padding: 10px 12px; 136 136 } 137 137 + #details .desc { 138 138 + margin: 4px 0 8px; 139 139 + font-size: 13px; 140 140 + } 137 141 #details h2 { 138 142 font-size: 14px; 139 143 margin: 0 0 6px; ··· 198 202 >machine 199 203 <select id="machine"></select 200 204 ></label> 201 201 - <input id="search" placeholder="filter list…" size="24" /> 205 205 + <input id="search" placeholder="filter name or description…" size="28" /> 202 206 <label 203 207 ><input type="checkbox" id="onlyInst" /> directly installed only</label 204 208 > ··· 290 294 }; 291 295 document.title = $("#title").textContent = D.title; 292 296 297 297 + // package meta (description, homepage, ...) by package name; strings come from nixpkgs, so escape them 298 298 + const metaOf = n => (D.meta || {})[n]; 299 299 + const esc = s => String(s).replace(/[&<>"']/g, c => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" })[c]); 300 300 + const urlHtml = s => /^https?:\/\//.test(s) ? `<a target="_blank" rel="noreferrer" href="${esc(s)}">${esc(s)}</a>` : esc(s); 301 301 + function metaHtml(n) { 302 302 + const m = metaOf(n); if (!m) return ""; 303 303 + return [ 304 304 + m.description && `<div class="desc">${esc(m.description)}</div>`, 305 305 + m.homepage && `<div><span class="k">homepage</span> ${urlHtml(m.homepage)}</div>`, 306 306 + m.license?.length && `<div><span class="k">license</span> ${esc(m.license.join(", "))}</div>`, 307 307 + m.changelog && `<div><span class="k">changelog</span> ${urlHtml(m.changelog)}</div>`, 308 308 + m.attr && `<div><span class="k">attribute</span> pkgs.${esc(m.attr)}</div>`, 309 309 + m.vulnerabilities?.length && `<div class="u">known vulnerabilities:<ul>${m.vulnerabilities.map(v => `<li>${esc(v)}</li>`).join("")}</ul></div>`, 310 310 + ].filter(Boolean).join(""); 311 311 + } 312 312 + 293 313 // ---------- list ---------- 294 314 let selected = -1; 295 315 $("#list").addEventListener("click", e => { const r = e.target.closest(".row"); if (r) focusNode(+r.dataset.i); }); ··· 304 324 const q = $("#search").value.toLowerCase(), oi = $("#onlyInst").checked; 305 325 const rows = []; 306 326 for (const [n, vs] of variants) { 307 307 - if (q && !n.toLowerCase().includes(q)) continue; 327 327 + if (q && !n.toLowerCase().includes(q) && !(metaOf(n)?.description || "").toLowerCase().includes(q)) continue; 308 328 const v = vs.filter(inMachine); if (!v.length) continue; 309 329 if (oi && !v.some(isInst)) continue; 310 330 v.sort((a, b) => depOf(b) - depOf(a)); ··· 315 335 const shown = rows.slice(0, 800); 316 336 const sel = selected >= 0 ? D.names[selected] : null; 317 337 $("#list").innerHTML = `<div class="meta">sorted by #dependents (best variant)${rows.length > shown.length ? ` · showing ${shown.length} of ${rows.length}, filter to narrow` : ""}</div>` + 318 318 - shown.map(r => `<div class="row${r.n === sel ? " sel" : ""}" data-i="${r.best}"><span class="n">${r.inst ? '<span class="inst">●</span> ' : ""}${r.n}${r.count > 1 ? ` <span class="k">×${r.count}</span>` : ""}</span><span class="c">${r.dep}</span></div>`).join(""); 338 338 + shown.map(r => `<div class="row${r.n === sel ? " sel" : ""}" data-i="${r.best}" title="${esc(metaOf(r.n)?.description || "")}"><span class="n">${r.inst ? '<span class="inst">●</span> ' : ""}${metaOf(r.n)?.vulnerabilities?.length ? '<span class="u" title="known vulnerabilities">⚠</span> ' : ""}${r.n}${r.count > 1 ? ` <span class="k">×${r.count}</span>` : ""}</span><span class="c">${r.dep}</span></div>`).join(""); 319 339 } 320 340 const svg = d3.select("#graph svg"), g = svg.append("g"); 321 341 svg.append("defs").append("marker").attr("id", "arrow").attr("viewBox", "0 -4 8 8").attr("refX", 8).attr("refY", 0) ··· 418 438 nm.attr("class", d => `node${d.id === focusId ? " focus" : ""}${isInst(d.id) ? " inst" : ""}`); 419 439 nm.select("circle").attr("r", d => radius(d.id)).attr("fill", d => color(d.id)); 420 440 nm.select("text").text(d => label(d.id)); 421 421 - nm.select("title").text(d => `${label(d.id)}\n${depOf(d.id)} dependents · ${parentsOf(d.id).length} direct users`); 441 441 + nm.select("title").text(d => `${label(d.id)}${metaOf(D.names[d.id])?.description ? "\n" + metaOf(D.names[d.id]).description : ""}\n${depOf(d.id)} dependents · ${parentsOf(d.id).length} direct users`); 422 442 sim.nodes(nodes); 423 443 sim.force("link").links(all); 424 444 sim.alpha(0.8).restart(); ··· 460 480 $("#details").innerHTML = ` 461 481 <h2 class="${isU[i] ? "u" : ""}">${label(i)}</h2> 462 482 <div>${isU[i] ? '<span class="u">no maintainer</span>' : '<span class="k">not flagged</span>'}</div> 483 483 + ${metaHtml(D.names[i])} 463 484 ${D.file[i] ? `<div><span class="k">file</span> ${fileLink(i)}</div>` : ""} 464 485 <div><span class="k">drv</span> ${D.hash[i]}…-${D.names[i]}.drv</div> 465 486 <div><span class="k">dependents</span> ${depOf(i)} ${mi < 0 ? "(max over machines)" : "on " + D.machines[mi]}</div>

+113 -7

tools/unmaintained-graph/unmaintained_graph.py

View file Reviewed

··· 6 6 derivation changes). The warnings name the unmaintained packages; the graph 7 7 comes from `nix derivation show -r` on the system derivation, so build 8 8 dependencies are included. 9 9 + 10 10 + Package meta (description, homepage, ...) is collected in the same evaluation 11 11 + by walking the derivation values the system is built from. Packages only 12 12 + referenced through strings (e.g. "${pkg}/bin/foo") are not reachable that way; 13 13 + for those the tool looks up the attribute by name in one extra evaluation. 9 14 """ 10 15 11 16 import argparse ··· 18 23 from collections import defaultdict 19 24 from pathlib import Path 20 25 26 26 + # second group is "<file>:<line>", the package's meta.position 21 27 WARNING = re.compile( 22 22 - r"evaluation warning: Package '([^']+)' in (/nix/store/[^:\s]+):\d+ " 28 28 + r"evaluation warning: Package '([^']+)' in (/nix/store/[^:\s]+:\d+) " 23 29 r"has the following problem: maintainerless" 24 30 ) 25 31 MAX_MACHINES = 31 # the page stores machine membership in 32-bit masks 32 32 + 33 33 + # Nix function: the cheap, JSON-safe subset of a derivation's meta 34 34 + META_FN = """ 35 35 + metaOf = d: 36 36 + let 37 37 + m = d.meta or { }; 38 38 + str = v: if builtins.isString v then v else null; 39 39 + first = v: if builtins.isList v then (if v == [ ] then null else builtins.head v) else v; 40 40 + lic = x: if builtins.isAttrs x then x.spdxId or x.shortName or x.fullName or null else str x; 41 41 + licenses = let l = m.license or [ ]; in if builtins.isList l then l else [ l ]; 42 42 + in 43 43 + { 44 44 + description = str (m.description or null); 45 45 + homepage = str (first (m.homepage or null)); 46 46 + changelog = str (first (m.changelog or null)); 47 47 + license = builtins.filter (x: x != null) (map lic licenses); 48 48 + vulnerabilities = builtins.filter builtins.isString (m.knownVulnerabilities or [ ]); 49 49 + }; 50 50 + """ 26 51 27 52 EVAL_EXPR = """ 28 53 let ··· 34 59 ]; 35 60 }; 36 61 top = if %(kind_str)s == "nixos" then sys.config.system.build.toplevel else sys.system; 62 62 + %(meta_fn)s 63 63 + isDrv = v: builtins.isAttrs v && (v.type or null) == "derivation"; 64 64 + # derivation values a derivation was given as attributes: directly, in lists, or one attrset deep. 65 65 + # All of these were already forced to compute drvPath, so walking them is nearly free. 66 66 + depsOf = d: 67 67 + let 68 68 + pick = v: 69 69 + if isDrv v then [ v ] 70 70 + else if builtins.isList v then builtins.concatMap pick v 71 71 + else if builtins.isAttrs v then builtins.filter isDrv (builtins.attrValues v) 72 72 + else [ ]; 73 73 + in 74 74 + builtins.concatMap pick (builtins.attrValues (d.drvAttrs or { })); 75 75 + closure = builtins.genericClosure { 76 76 + startSet = [ { key = top.drvPath; d = top; } ]; 77 77 + operator = n: map (d: { key = d.drvPath; inherit d; }) (depsOf n.d); 78 78 + }; 79 79 + unmaintained = builtins.filter 80 80 + (n: (n.d.meta.maintainers or [ ]) == [ ] && (n.d.meta.teams or [ ]) == [ ]) closure; 37 81 in 38 82 { 39 83 drv = top.drvPath; 40 84 # warnings point into the source nixpkgs was evaluated from, which pkgs.path may only be a copy of 41 85 nixpkgs = sys.pkgs.hello.meta.position; 42 86 rev = sys.config.system.nixos.revision or sys.config.system.nixpkgsRevision or ""; 87 87 + meta = builtins.listToAttrs (map (n: { name = n.d.name; value = metaOf n.d; }) unmaintained); 43 88 } 89 89 + """ 90 90 + 91 91 + # For packages the walk could not reach: try likely attribute paths, accept one whose name matches. 92 92 + LOOKUP_EXPR = """ 93 93 + let 94 94 + flake = builtins.getFlake %(flake)s; 95 95 + pkgs = flake.%(kind)sConfigurations.${%(machine)s}.pkgs; 96 96 + lib = pkgs.lib; 97 97 + %(meta_fn)s 98 98 + wanted = builtins.fromJSON %(wanted)s; 99 99 + get = path: let r = builtins.tryEval (lib.attrByPath path null pkgs); in if r.success then r.value else null; 100 100 + nameOf = v: let r = builtins.tryEval (v.name or null); in if r.success then r.value else null; 101 101 + find = it: 102 102 + let 103 103 + p = builtins.parseDrvName it.name; 104 104 + py = builtins.match "python3[.0-9]*-(.*)" p.name; 105 105 + candidates = 106 106 + lib.optional (it.byname != null) [ it.byname ] 107 107 + ++ [ [ p.name ] [ "haskellPackages" p.name ] ] 108 108 + ++ lib.optional (py != null) [ "python3Packages" (builtins.head py) ]; 109 109 + hits = builtins.filter (c: let v = get c; in lib.isDerivation v && nameOf v == it.name) candidates; 110 110 + in 111 111 + if hits == [ ] then null 112 112 + else metaOf (get (builtins.head hits)) // { attr = lib.concatStringsSep "." (builtins.head hits); }; 113 113 + in 114 114 + lib.filterAttrs (_: v: v != null) (builtins.listToAttrs (map (it: { name = it.name; value = find it; }) wanted)) 44 115 """ 45 116 46 117 ··· 79 150 "kind": kind, 80 151 "kind_str": nix_str(kind), 81 152 "machine": nix_str(machine), 153 153 + "meta_fn": META_FN, 82 154 } 83 155 proc = nix_eval_json(expr) 84 156 if proc.returncode != 0: ··· 87 159 result = json.loads(proc.stdout) 88 160 result["warned"] = sorted(set(WARNING.findall(proc.stderr))) 89 161 return result 162 162 + 163 163 + 164 164 + def lookup_meta(flake, machine, kind, warned): 165 165 + """Meta for the packages the closure walk missed, via attribute lookup on one machine's pkgs.""" 166 166 + wanted = [] 167 167 + for name, path in warned.items(): 168 168 + m = re.search(r"/pkgs/by-name/[^/]+/([^/]+)/package\.nix:\d+$", path) 169 169 + wanted.append({"name": name, "byname": m.group(1) if m else None}) 170 170 + expr = LOOKUP_EXPR % { 171 171 + "flake": nix_str(flake), 172 172 + "kind": kind, 173 173 + "machine": nix_str(machine), 174 174 + "meta_fn": META_FN, 175 175 + "wanted": nix_str(json.dumps(wanted)), 176 176 + } 177 177 + proc = nix_eval_json(expr) 178 178 + if proc.returncode != 0: 179 179 + lines = proc.stderr.strip().splitlines() 180 180 + log(f"[meta] lookup failed, continuing without: {lines[-1] if lines else '?'}") 181 181 + return {} 182 182 + return json.loads(proc.stdout) 90 183 91 184 92 185 def derivation_graph(drv): ··· 124 217 return [bin(a).count("1") for a in anc] 125 218 126 219 127 127 - def file_entry(path, sources): 128 128 - """[path to show, GitHub link or None] for the file a warning points at.""" 220 220 + def file_entry(position, sources): 221 221 + """[file:line to show, GitHub link to that line or None] for a warning's meta.position.""" 222 222 + path, _, line = position.rpartition(":") 129 223 for src, rev in sources.items(): 130 224 if path.startswith(src + "/"): 131 225 rel = path[len(src) + 1 :] 132 226 link = re.fullmatch(r"[0-9a-f]{7,40}", rev or "") 133 133 - return [rel, f"https://github.com/NixOS/nixpkgs/blob/{rev}/{rel}" if link else None] 227 227 + url = f"https://github.com/NixOS/nixpkgs/blob/{rev}/{rel}#L{line}" if link else None 228 228 + return [f"{rel}:{line}", url] 134 229 m = re.match(r"/nix/store/[^/]+/(.*)", path) 135 135 - return [(m.group(1) if m else path) + " (not nixpkgs)", None] 230 230 + return [f"{m.group(1) if m else path}:{line} (not nixpkgs)", None] 136 231 137 232 138 138 - def build_data(title, results, graphs): 233 233 + def build_data(title, results, graphs, meta): 139 234 mach = sorted(results) 140 235 # "<nixpkgs>/pkgs/by-name/he/hello/package.nix:42" -> "<nixpkgs>" 141 236 sources = {r["nixpkgs"].rsplit("/pkgs/", 1)[0]: r["rev"] for r in results.values()} ··· 186 281 "udep": {str(i): [dep[i].get(k, -1) for k in range(len(mach))] for i in unmaintained}, 187 282 "file": {str(i): file_entry(warned[names[i]], sources) for i in unmaintained}, 188 283 "parents": [sorted(p) for p in parents], 284 284 + "meta": {n: meta[n] for n in {names[i] for i in unmaintained} if n in meta}, 189 285 } 190 286 return data, len({names[i] for i in unmaintained}) 191 287 ··· 234 330 if not results: 235 331 sys.exit("every evaluation failed") 236 332 333 333 + warned = {name: path for r in results.values() for name, path in r["warned"]} 334 334 + meta = {} 335 335 + for r in results.values(): 336 336 + meta.update((n, v) for n, v in r.pop("meta").items() if n in warned) 337 337 + missing = {n: p for n, p in warned.items() if n not in meta} 338 338 + if missing: 339 339 + first = next(m for m, _ in machines if m in results) 340 340 + meta.update(lookup_meta(flake, first, dict(machines)[first], missing)) 341 341 + log(f"[meta] {len(meta)}/{len(warned)} packages with meta") 342 342 + 237 343 with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool: 238 344 graphs = dict(zip(results, pool.map(lambda m: derivation_graph(results[m]["drv"]), results))) 239 345 240 346 title = args.title or f"Unmaintained packages in {Path(flake).name if os.path.isabs(flake) else flake}" 241 241 - data, count = build_data(title, results, graphs) 347 347 + data, count = build_data(title, results, graphs, meta) 242 348 243 349 template = Path(os.environ["UNMAINTAINED_GRAPH_TEMPLATE"]).read_text() 244 350 d3 = Path(os.environ["UNMAINTAINED_GRAPH_D3"]).read_text()