lass.pds.lassul.us / superconfig
This repository has no description
0
Fork 1

Configure Feed

Select the types of activity you want to include in your feed.

archiver-bot: compute IPFS CID locally with --only-hash --raw-leaves, fix key upload error

author
lassulus date (Apr 3, 2026, 11:53 PM CEST) commit 45003813 parent 05182e8f
+47 -31
+1 -2
2configs/services/archiver-bot.nix
··· 29 29 "jellyseerr.service" 30 30 ]; 31 31 wants = [ "network-online.target" ]; 32 + path = [ pkgs.kubo ]; 32 33 33 34 environment = { 34 35 MATRIX_HOMESERVER = "https://matrix.lassul.us"; ··· 41 42 JELLYFIN_URL = "http://localhost:8096"; 42 43 43 44 MEDIA_PATH_PREFIX = "/var/download/"; 44 - IPFS_CID_MAP = "/var/lib/ipfs/cid-map.txt"; 45 - IPFS_PATH_PREFIX = "/var/lib/ipfs/download/"; 46 45 WEBHOOK_PORT = "8099"; 47 46 STORE_PATH = "/var/lib/archiver-bot"; 48 47 };
+1 -4
2configs/services/flix/container-host.nix
··· 16 16 hostPath = "/var/download"; 17 17 isReadOnly = false; 18 18 }; 19 - containers.yellow.bindMounts."/var/lib/ipfs/cid-map.txt" = { 20 - hostPath = "/var/lib/ipfs/cid-map.txt"; 21 - isReadOnly = true; 22 - }; 19 + 23 20 networking.firewall.allowedTCPPorts = [ 24 21 8096 25 22 8920
+45 -25
5pkgs/archiver-bot/archiver_bot/__init__.py
··· 40 40 JELLYFIN_API_KEY = os.environ.get("JELLYFIN_API_KEY", "") 41 41 IPFS_GATEWAY_URL = os.environ.get("IPFS_GATEWAY_URL", "https://ipfs.lassul.us") 42 42 MEDIA_PATH_PREFIX = os.environ.get("MEDIA_PATH_PREFIX", "/var/download/") 43 - IPFS_CID_MAP = os.environ.get("IPFS_CID_MAP", "/var/lib/ipfs/cid-map.txt") 44 - IPFS_PATH_PREFIX = os.environ.get("IPFS_PATH_PREFIX", "/var/lib/ipfs/download/") 43 + IPFS_BIN = os.environ.get("IPFS_BIN", "ipfs") 45 44 46 45 # Persistent tracking of requested media -> rooms 47 46 TRACKING_FILE = Path(STORE_PATH) / "tracked_requests.json" ··· 89 88 return f"{FLAX_URL}/{quote(relative)}" 90 89 91 90 92 - def lookup_cid(path: str) -> str | None: 93 - """Look up the IPFS CID for a file from the cid-map.""" 94 - ipfs_path = path 95 - if ipfs_path.startswith(MEDIA_PATH_PREFIX): 96 - ipfs_path = IPFS_PATH_PREFIX + ipfs_path[len(MEDIA_PATH_PREFIX):] 91 + async def compute_cid(path: str) -> str | None: 92 + """Compute the IPFS CID for a file using ipfs add --only-hash --raw-leaves.""" 97 93 try: 98 - with open(IPFS_CID_MAP) as f: 99 - for line in f: 100 - line = line.strip() 101 - if not line: 102 - continue 103 - cid, _, file_path = line.partition(" ") 104 - if file_path == ipfs_path: 105 - return cid 94 + ipfs_path = "/tmp/ipfs-hash-repo" 95 + if not os.path.exists(os.path.join(ipfs_path, "config")): 96 + init_proc = await asyncio.create_subprocess_exec( 97 + IPFS_BIN, "init", 98 + stdout=asyncio.subprocess.PIPE, 99 + stderr=asyncio.subprocess.PIPE, 100 + env={**os.environ, "IPFS_PATH": ipfs_path}, 101 + ) 102 + await init_proc.communicate() 103 + env = {**os.environ, "IPFS_PATH": ipfs_path} 104 + proc = await asyncio.create_subprocess_exec( 105 + IPFS_BIN, "add", "--only-hash", "--raw-leaves", "--quieter", path, 106 + stdout=asyncio.subprocess.PIPE, 107 + stderr=asyncio.subprocess.PIPE, 108 + env=env, 109 + ) 110 + try: 111 + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=120) 112 + except asyncio.TimeoutError: 113 + proc.kill() 114 + await proc.communicate() 115 + log.warning(f"ipfs add --only-hash timed out for {path}") 116 + return None 117 + if proc.returncode == 0: 118 + cid = stdout.decode().strip() 119 + if cid: 120 + return cid 121 + else: 122 + log.warning(f"ipfs add --only-hash failed for {path}: {stderr.decode().strip()}") 106 123 except Exception as e: 107 - log.warning(f"Failed to look up CID for {path}: {e}") 124 + log.warning(f"Failed to compute CID for {path}: {e}") 108 125 return None 109 126 110 127 ··· 166 183 return None 167 184 168 185 169 - def make_media_links(path: str, is_dir: bool = False, jellyfin_id: str | None = None) -> tuple[str, str]: 186 + async def make_media_links(path: str, is_dir: bool = False, jellyfin_id: str | None = None) -> tuple[str, str]: 170 187 """Generate plain and HTML text with all available links for a media path.""" 171 188 from urllib.parse import quote 172 189 ··· 188 205 189 206 # IPFS link (compute CID for the file) 190 207 if not is_dir: 191 - cid = lookup_cid(path) 208 + cid = await compute_cid(path) 192 209 if cid: 193 210 ipfs_link = f"{IPFS_GATEWAY_URL}/ipfs/{cid}" 194 211 lines_plain.append(f"🌐 IPFS: {ipfs_link}") ··· 308 325 if path: 309 326 is_dir = media_type == "tv" 310 327 jf_id = await lookup_jellyfin_id(session, media_type, media_id) 311 - links_plain, links_html = make_media_links(path, is_dir=is_dir, jellyfin_id=jf_id) 328 + links_plain, links_html = await make_media_links(path, is_dir=is_dir, jellyfin_id=jf_id) 312 329 return (f"✅ {display} is already available!\n{links_plain}", f"✅ <b>{display}</b> is already available!<br>{links_html}") 313 330 return (f"✅ {display} is already available!", f"✅ <b>{display}</b> is already available!") 314 331 elif status in (2, 3, 4): ··· 324 341 if existing_path: 325 342 is_dir = media_type == "tv" 326 343 jf_id = await lookup_jellyfin_id(session, media_type, media_id) 327 - links_plain, links_html = make_media_links(existing_path, is_dir=is_dir, jellyfin_id=jf_id) 344 + links_plain, links_html = await make_media_links(existing_path, is_dir=is_dir, jellyfin_id=jf_id) 328 345 # Still send the request to Jellyseerr so it's tracked there too 329 346 await jellyseerr_request(session, "/request", method="POST", json=payload) 330 347 return (f"✅ {display} is already available!\n{links_plain}", f"✅ <b>{display}</b> is already available!<br>{links_html}") ··· 470 487 await trigger_jellyfin_scan(http) 471 488 await asyncio.sleep(10) 472 489 jf_id = await lookup_jellyfin_id(http, "movie", tmdb_id) 473 - links_plain, links_html = make_media_links(file_path, jellyfin_id=jf_id) 490 + links_plain, links_html = await make_media_links(file_path, jellyfin_id=jf_id) 474 491 plain = f"✅ {display} has been downloaded and is now available!\n{links_plain}" 475 492 html = f"✅ <b>{display}</b> has been downloaded and is now available!<br>{links_html}" 476 493 for room_id in rooms: ··· 514 531 await trigger_jellyfin_scan(http) 515 532 await asyncio.sleep(10) 516 533 jf_id = await lookup_jellyfin_id(http, "tv", tmdb_id) 517 - links_plain, links_html = make_media_links(folder_path, is_dir=True, jellyfin_id=jf_id) 534 + links_plain, links_html = await make_media_links(folder_path, is_dir=True, jellyfin_id=jf_id) 518 535 plain = f"✅ {display}{ep_info} has been downloaded and is now available!\n{links_plain}" 519 536 html = f"✅ <b>{display}</b>{ep_info} has been downloaded and is now available!<br>{links_html}" 520 537 for room_id in rooms: ··· 561 578 562 579 # E2EE setup 563 580 if client.olm: 564 - log.info("Uploading device keys...") 565 - key_upload = await client.keys_upload() 566 - log.info(f"Key upload result: {key_upload}") 581 + try: 582 + log.info("Uploading device keys...") 583 + key_upload = await client.keys_upload() 584 + log.info(f"Key upload result: {key_upload}") 585 + except Exception as e: 586 + log.info(f"Key upload skipped: {e}") 567 587 568 588 # Trust all known device keys so we can decrypt messages 569 589 for user_id in client.device_store.users: