forked from
lass.pds.lassul.us/superconfig
This repository has no description
2configs.services.flix.ipfs: add anonymous resumable upload endpoint
Adds an ipfs-upload service (Python http.server on 127.0.0.1:8090) and
routes POSTs on ipfs.lassul.us to it (GET/HEAD still go to the kubo
gateway, via nginx upstream + map on $request_method).
Each chunk is written to a deterministic staging file under
/var/download/incoming/uploader/<sha256(rel)>.upload (same fs as the
destination, outside the pin-watcher's watch dirs). When the staging
size reaches the Content-Range total, it is atomically renamed to
/var/lib/ipfs/download/<rel> and pinned with ipfs add --nocopy --pin.
Protocol:
POST /<path> one-shot upload
POST /<path> with Content-Range bytes a-b/T resumable chunk
POST /<path> with Content-Length: 0 returns current offset
Assisted-by: Claude Code (claude-opus-4-7)
author
lassulus
date
(May 24, 2026, 2:34 PM CEST)
commit
ac9fc7f6
ac9fc7f6cab3e186c93f32a7a30948899ce7b2fd
parent
e6bc3cc9
e6bc3cc9e9ad0ea917d9655681ae6efff61d0665
+281
-1
+34
2configs/services/flix/ipfs.nix
+34
2configs/services/flix/ipfs.nix
···
227
227
# files that no longer exist on disk. Protects against inotify event
228
228
# drops (e.g. queue overflow during bulk renames/deletes) and any other
229
229
# path that leaves cid-map/MFS out of sync with the filesystem.
230
+
uploadServer = pkgs.writers.writePython3Bin "ipfs-upload-server" {
231
+
flakeIgnore = [ "E501" ];
232
+
} ./ipfs_upload.py;
233
+
230
234
reconcileScript = pkgs.writers.writeBash "flix-index-reconcile" ''
231
235
set -efu
232
236
···
271
275
# symlink /var/download into IPFS root so --nocopy works
272
276
systemd.tmpfiles.rules = [
273
277
"L+ /var/lib/ipfs/download - - - - /var/download"
278
+
# Staging area for ipfs-upload: same filesystem as the destination, but
279
+
# outside the pin-watcher's watch dirs (movies/shows/games) so inotify
280
+
# never fires on in-flight uploads.
281
+
"d /var/download/incoming 0775 ${config.services.kubo.user} download -"
282
+
"d /var/download/incoming/uploader 0775 ${config.services.kubo.user} download -"
274
283
];
275
284
276
285
services.kubo = {
···
333
342
User = config.services.kubo.user;
334
343
Group = config.services.kubo.group;
335
344
SupplementaryGroups = [ "radio_container" ];
345
+
};
346
+
};
347
+
348
+
systemd.services.ipfs-upload = {
349
+
description = "HTTP upload endpoint that writes files under /var/lib/ipfs/download and pins them";
350
+
after = [ "ipfs.service" ];
351
+
requires = [ "ipfs.service" ];
352
+
wantedBy = [ "multi-user.target" ];
353
+
serviceConfig = {
354
+
Environment = [
355
+
"IPFS_PATH=/var/lib/ipfs"
356
+
"IPFS_BIN=${pkgs.kubo}/bin/ipfs"
357
+
"DOWNLOAD_ROOT=/var/lib/ipfs/download"
358
+
"BIND=127.0.0.1"
359
+
"PORT=8090"
360
+
];
361
+
ExecStart = "${uploadServer}/bin/ipfs-upload-server";
362
+
Restart = "always";
363
+
RestartSec = "10s";
364
+
User = config.services.kubo.user;
365
+
Group = config.services.kubo.group;
366
+
# /var/download is owned by ipfs:ipfs on neoprism, so no extra
367
+
# groups needed. Existing subdirs owned by other users (e.g. games
368
+
# → lass:users) are not writable by this service; uploads to those
369
+
# paths will get a 500 unless ownership is adjusted out-of-band.
336
370
};
337
371
};
338
372
+229
2configs/services/flix/ipfs_upload.py
+229
2configs/services/flix/ipfs_upload.py
···
1
+
"""HTTP upload endpoint that writes files under /var/lib/ipfs/download and pins them.
2
+
3
+
Supports resumable uploads via `Content-Range: bytes <start>-<end>/<total>`.
4
+
The staging file is `<STAGING_DIR>/<sha256(rel)>.upload` so the staging path
5
+
is deterministic from the destination path; clients resume by re-POSTing
6
+
the same URL.
7
+
8
+
Protocol:
9
+
POST /<path> body=N bytes, no Content-Range
10
+
One-shot upload: equivalent to a single chunk covering the whole file.
11
+
12
+
POST /<path> body=N bytes, Content-Range: bytes <start>-<end>/<total>
13
+
Chunked upload. `start` must equal the current staging size, or 0
14
+
to truncate and start fresh. When the staging size reaches `total`,
15
+
the staging file is moved to /var/lib/ipfs/download/<path> and
16
+
`ipfs add --nocopy --pin` is run. Returns 200 with the CID on
17
+
completion, 308 with `Upload-Offset: <size>` while incomplete.
18
+
19
+
POST /<path> body=0, no Content-Range
20
+
Status check. Returns 200 with `Upload-Offset: <size>` if a
21
+
staging file exists, 404 otherwise.
22
+
23
+
URL path segments are restricted to [A-Za-z0-9._-]. Re-uploading the same
24
+
path overwrites the file; the previous CID is left in IPFS but unreferenced
25
+
by this service.
26
+
"""
27
+
28
+
from __future__ import annotations
29
+
30
+
import hashlib
31
+
import http.server
32
+
import os
33
+
import re
34
+
import socketserver
35
+
import subprocess
36
+
import sys
37
+
38
+
DOWNLOAD_ROOT = os.environ.get("DOWNLOAD_ROOT", "/var/lib/ipfs/download")
39
+
STAGING_DIR = os.environ.get("STAGING_DIR", "/var/lib/ipfs/download/incoming/uploader")
40
+
IPFS_BIN = os.environ.get("IPFS_BIN", "ipfs")
41
+
CHUNK = 1 << 20 # 1 MiB
42
+
_CONTENT_RANGE = re.compile(r"^bytes (\d+)-(\d+)/(\d+)$")
43
+
44
+
_SAFE_SEGMENT = re.compile(r"^[A-Za-z0-9._\-]+$")
45
+
46
+
47
+
def safe_relpath(url_path: str) -> str | None:
48
+
raw = url_path.split("?", 1)[0].strip("/")
49
+
if not raw:
50
+
return None
51
+
segments = raw.split("/")
52
+
for seg in segments:
53
+
if seg in {"", ".", ".."} or not _SAFE_SEGMENT.match(seg):
54
+
return None
55
+
return "/".join(segments)
56
+
57
+
58
+
def staging_path(rel: str) -> str:
59
+
sid = hashlib.sha256(rel.encode("utf-8")).hexdigest()
60
+
return os.path.join(STAGING_DIR, f"{sid}.upload")
61
+
62
+
63
+
class Handler(http.server.BaseHTTPRequestHandler):
64
+
server_version = "ipfs-upload/1"
65
+
66
+
def do_POST(self) -> None: # noqa: N802 - stdlib interface
67
+
rel = safe_relpath(self.path)
68
+
if rel is None:
69
+
self.send_error(400, "invalid path")
70
+
return
71
+
72
+
try:
73
+
length = int(self.headers.get("Content-Length", "0"))
74
+
except ValueError:
75
+
self.send_error(400, "bad Content-Length")
76
+
return
77
+
if length < 0:
78
+
self.send_error(400, "bad Content-Length")
79
+
return
80
+
81
+
cr_header = self.headers.get("Content-Range")
82
+
staging = staging_path(rel)
83
+
84
+
# Status check: empty body + no Content-Range.
85
+
if length == 0 and not cr_header:
86
+
try:
87
+
size = os.path.getsize(staging)
88
+
except OSError:
89
+
self.send_error(404, "no upload in progress")
90
+
return
91
+
body = (str(size) + "\n").encode("ascii")
92
+
self.send_response(200)
93
+
self.send_header("Content-Type", "text/plain; charset=utf-8")
94
+
self.send_header("Upload-Offset", str(size))
95
+
self.send_header("Content-Length", str(len(body)))
96
+
self.end_headers()
97
+
self.wfile.write(body)
98
+
return
99
+
100
+
# Parse Content-Range or derive a single-chunk request.
101
+
if cr_header:
102
+
m = _CONTENT_RANGE.match(cr_header)
103
+
if not m:
104
+
self.send_error(400, "bad Content-Range")
105
+
return
106
+
start, end, total = int(m.group(1)), int(m.group(2)), int(m.group(3))
107
+
if start < 0 or end < start or end >= total:
108
+
self.send_error(400, "bad Content-Range")
109
+
return
110
+
chunk_len = end - start + 1
111
+
if length != chunk_len:
112
+
self.send_error(400, "Content-Length doesn't match Content-Range")
113
+
return
114
+
else:
115
+
if length == 0:
116
+
self.send_error(411, "Content-Length required")
117
+
return
118
+
start, total, chunk_len = 0, length, length
119
+
120
+
# Current staging size and offset enforcement.
121
+
try:
122
+
current_size = os.path.getsize(staging)
123
+
except OSError:
124
+
current_size = 0
125
+
126
+
if start == 0:
127
+
mode = "wb" # truncate / fresh start
128
+
elif start == current_size:
129
+
mode = "ab"
130
+
else:
131
+
self.send_error(
132
+
409, f"offset mismatch: expected {current_size}, got {start}"
133
+
)
134
+
return
135
+
136
+
try:
137
+
with open(staging, mode) as f:
138
+
remaining = chunk_len
139
+
while remaining > 0:
140
+
data = self.rfile.read(min(CHUNK, remaining))
141
+
if not data:
142
+
break
143
+
f.write(data)
144
+
remaining -= len(data)
145
+
if remaining != 0:
146
+
raise OSError(f"short read: {remaining} bytes missing")
147
+
except OSError as e:
148
+
# Roll the staging file back to the chunk's start so the client
149
+
# can retry this chunk from a consistent offset.
150
+
try:
151
+
with open(staging, "r+b") as fix:
152
+
fix.truncate(start)
153
+
except OSError:
154
+
pass
155
+
self.send_error(500, f"write failed: {e}")
156
+
return
157
+
158
+
try:
159
+
new_size = os.path.getsize(staging)
160
+
except OSError as e:
161
+
self.send_error(500, f"stat failed: {e}")
162
+
return
163
+
164
+
if new_size < total:
165
+
self.send_response(308)
166
+
self.send_header("Upload-Offset", str(new_size))
167
+
self.send_header("Range", f"bytes=0-{new_size - 1}")
168
+
self.send_header("Content-Length", "0")
169
+
self.end_headers()
170
+
return
171
+
172
+
if new_size > total:
173
+
# Client over-sent (shouldn't happen given our checks, but guard).
174
+
try:
175
+
with open(staging, "r+b") as fix:
176
+
fix.truncate(total)
177
+
except OSError as e:
178
+
self.send_error(500, f"truncate failed: {e}")
179
+
return
180
+
181
+
# Complete: move staging → dst and pin.
182
+
dst = os.path.join(DOWNLOAD_ROOT, rel)
183
+
dst_dir = os.path.dirname(dst) or DOWNLOAD_ROOT
184
+
try:
185
+
os.makedirs(dst_dir, exist_ok=True)
186
+
os.replace(staging, dst)
187
+
except OSError as e:
188
+
self.send_error(500, f"finalize failed: {e}")
189
+
return
190
+
191
+
try:
192
+
result = subprocess.run(
193
+
[IPFS_BIN, "add", "--nocopy", "--pin", "--quieter", dst],
194
+
check=True,
195
+
capture_output=True,
196
+
text=True,
197
+
)
198
+
except subprocess.CalledProcessError as e:
199
+
msg = (e.stderr or e.stdout or "").strip() or "unknown error"
200
+
self.send_error(500, f"ipfs add failed: {msg}")
201
+
return
202
+
203
+
cid = result.stdout.strip()
204
+
body = (cid + "\n").encode("ascii")
205
+
self.send_response(200)
206
+
self.send_header("Content-Type", "text/plain; charset=utf-8")
207
+
self.send_header("Content-Length", str(len(body)))
208
+
self.end_headers()
209
+
self.wfile.write(body)
210
+
211
+
def log_message(self, fmt: str, *args: object) -> None:
212
+
sys.stderr.write(f"{self.address_string()} {fmt % args}\n")
213
+
214
+
215
+
class Server(socketserver.ThreadingMixIn, http.server.HTTPServer):
216
+
daemon_threads = True
217
+
allow_reuse_address = True
218
+
219
+
220
+
def main() -> None:
221
+
os.makedirs(STAGING_DIR, exist_ok=True)
222
+
bind = os.environ.get("BIND", "127.0.0.1")
223
+
port = int(os.environ.get("PORT", "8090"))
224
+
with Server((bind, port), Handler) as s:
225
+
s.serve_forever()
226
+
227
+
228
+
if __name__ == "__main__":
229
+
main()
+18
-1
2configs/services/flix/proxy.nix
+18
-1
2configs/services/flix/proxy.nix
···
1
1
{ pkgs, ... }:
2
2
{
3
+
# POST → ipfs-upload backend (writes file to /var/lib/ipfs/download/<path>
4
+
# and returns the CID). Anything else → the kubo gateway.
5
+
services.nginx.upstreams.ipfs-gateway.servers."127.0.0.1:8089" = { };
6
+
services.nginx.upstreams.ipfs-upload.servers."127.0.0.1:8090" = { };
7
+
services.nginx.appendHttpConfig = ''
8
+
map $request_method $ipfs_lassul_upstream {
9
+
default ipfs-gateway;
10
+
POST ipfs-upload;
11
+
}
12
+
'';
13
+
3
14
services.nginx.virtualHosts."ipfs.lassul.us" = {
4
15
forceSSL = true;
5
16
enableACME = true;
6
17
locations."/" = {
7
-
proxyPass = "http://localhost:8089";
18
+
proxyPass = "http://$ipfs_lassul_upstream";
8
19
recommendedProxySettings = true;
20
+
extraConfig = ''
21
+
client_max_body_size 0;
22
+
proxy_request_buffering off;
23
+
proxy_read_timeout 1h;
24
+
proxy_send_timeout 1h;
25
+
'';
9
26
};
10
27
};
11
28
services.nginx.virtualHosts."flix.lassul.us" = {