forked from
lass.pds.lassul.us/superconfig
This repository has no description
machines/starkstrom: IPFS pinning endpoint (starkstrom.lassul.us)
kubo node (swarm :4001, gateway NoFetch) + anonymous tarball -> ipfs add
(blockstore, no --nocopy) -> pinned -> returns CID; refuses uploads with
<300GiB free; explicit ipfs-gc + ipfs-roots (pins are the stored GC roots).
dns: ipfs -> CNAME neoprism, starkstrom -> A 194.110.87.67.
author
lassulus
date
(Aug 31, 2026, 12:27 PM CEST)
commit
a92df57f
a92df57f5972748a7ede216ae88ba8f2083d6f74
parent
8c1de15a
8c1de15acc638955f3aa788b620cd9f1a90e5afb
+310
-1
+3
-1
2configs/dns/lassul.us.zone
+3
-1
2configs/dns/lassul.us.zone
···
1
-
@ 3600 IN SOA lassul.us. ns1.lassul.us. 2026080900 7200 3600 86400 3600
1
+
@ 3600 IN SOA lassul.us. ns1.lassul.us. 2026083000 7200 3600 86400 3600
2
2
3
3
;;@ 3600 IN NS ns1
4
4
@ 3600 IN NS ns1
···
24
24
4.neoprism IN A 95.217.192.59
25
25
neoprism IN AAAA 2a01:4f9:4a:4f1a::2
26
26
6.neoprism IN AAAA 2a01:4f9:4a:4f1a::2
27
+
ipfs IN CNAME neoprism
28
+
starkstrom IN A 194.110.87.67
27
29
28
30
;; GLUE
29
31
ns1 IN A 95.217.192.59
+2
machines/starkstrom/config.nix
+2
machines/starkstrom/config.nix
+87
machines/starkstrom/ipfs-endpoint.nix
+87
machines/starkstrom/ipfs-endpoint.nix
···
1
+
{
2
+
config,
3
+
pkgs,
4
+
...
5
+
}:
6
+
let
7
+
api = "/ip4/127.0.0.1/tcp/5001";
8
+
ipfs = "${pkgs.kubo}/bin/ipfs";
9
+
uploadServer = pkgs.writers.writePython3Bin "ipfs-upload-server" {
10
+
flakeIgnore = [ "E501" ];
11
+
} ./ipfs_upload.py;
12
+
in
13
+
{
14
+
imports = [ ../../2configs/nginx.nix ];
15
+
16
+
# scratch area for in-flight uploads (extracted, added, then deleted)
17
+
systemd.tmpfiles.rules = [
18
+
"d /var/lib/ipfs/incoming 0700 ${config.services.kubo.user} ${config.services.kubo.group} -"
19
+
];
20
+
21
+
# anonymous tarball -> `ipfs add` (into the blockstore) -> pinned -> returns CID
22
+
systemd.services.ipfs-upload = {
23
+
description = "Anonymous tarball -> IPFS pinning endpoint";
24
+
after = [ "ipfs.service" ];
25
+
requires = [ "ipfs.service" ];
26
+
wantedBy = [ "multi-user.target" ];
27
+
serviceConfig = {
28
+
Environment = [
29
+
"IPFS_BIN=${pkgs.kubo}/bin/ipfs"
30
+
"IPFS_API=${api}"
31
+
"STAGING=/var/lib/ipfs/incoming"
32
+
"ROOTS_FILE=/var/lib/ipfs/gc-roots"
33
+
"GATEWAY_BASE=https://starkstrom.lassul.us"
34
+
"BIND=127.0.0.1"
35
+
"PORT=8090"
36
+
"MAX_BYTES=8589934592" # 8 GiB compressed upload
37
+
"MAX_EXTRACT_BYTES=21474836480" # 20 GiB extracted
38
+
"DISK_PATH=/var/lib/ipfs"
39
+
"MIN_FREE_BYTES=322122547200" # refuse uploads with <300 GiB free
40
+
];
41
+
ExecStart = "${uploadServer}/bin/ipfs-upload-server";
42
+
Restart = "always";
43
+
RestartSec = "10s";
44
+
User = config.services.kubo.user;
45
+
Group = config.services.kubo.group;
46
+
};
47
+
};
48
+
49
+
# public vhost (starkstrom's own name): writable upload at /, read-only gateway at /ipfs/
50
+
services.nginx.virtualHosts."starkstrom.lassul.us" = {
51
+
enableACME = true;
52
+
forceSSL = true;
53
+
locations."/ipfs/".extraConfig = ''
54
+
proxy_pass http://127.0.0.1:8089;
55
+
proxy_set_header Host $host;
56
+
'';
57
+
locations."/".extraConfig = ''
58
+
proxy_pass http://127.0.0.1:8090;
59
+
client_max_body_size 8g;
60
+
proxy_request_buffering off;
61
+
proxy_read_timeout 3600s;
62
+
proxy_send_timeout 3600s;
63
+
'';
64
+
};
65
+
66
+
# Explicit GC + gc-root management. Pins are the GC roots (persisted by kubo);
67
+
# `ipfs-gc` reclaims everything not reachable from a pin. Prune first with
68
+
# `ipfs-roots rm <cid>`, then `ipfs-gc`.
69
+
environment.systemPackages = [
70
+
(pkgs.writeShellScriptBin "ipfs-gc" ''
71
+
exec ${ipfs} --api ${api} repo gc "$@"
72
+
'')
73
+
(pkgs.writeShellScriptBin "ipfs-roots" ''
74
+
cmd="''${1-list}"
75
+
shift || true
76
+
case "$cmd" in
77
+
list) exec ${ipfs} --api ${api} pin ls --type=recursive ;;
78
+
ledger) exec ${pkgs.coreutils}/bin/cat /var/lib/ipfs/gc-roots ;;
79
+
rm) exec ${ipfs} --api ${api} pin rm "$@" ;;
80
+
*)
81
+
echo "usage: ipfs-roots [list|ledger|rm <cid>...]" >&2
82
+
exit 1
83
+
;;
84
+
esac
85
+
'')
86
+
];
87
+
}
+26
machines/starkstrom/ipfs.nix
+26
machines/starkstrom/ipfs.nix
···
1
+
{ ... }:
2
+
{
3
+
# Full kubo (IPFS) node. Data lives under /var/lib/ipfs on the 4 TB root.
4
+
# Uploads are copied into the blockstore (NO filestore/--nocopy on this host).
5
+
services.kubo = {
6
+
enable = true;
7
+
dataDir = "/var/lib/ipfs";
8
+
# No automatic GC: content is pinned on upload and only reclaimed by the
9
+
# explicit `ipfs-gc` command (see ipfs-endpoint.nix).
10
+
settings = {
11
+
Addresses = {
12
+
API = [ "/ip4/127.0.0.1/tcp/5001" ];
13
+
Gateway = [ "/ip4/127.0.0.1/tcp/8089" ];
14
+
};
15
+
# Serve only local/pinned blocks -> the public gateway is not an open
16
+
# proxy for arbitrary network CIDs.
17
+
Gateway.NoFetch = true;
18
+
# participate in the public swarm/DHT
19
+
Datastore.StorageMax = "500GB";
20
+
};
21
+
};
22
+
23
+
# incoming swarm connections (TCP + QUIC/UDP on 4001)
24
+
networking.firewall.allowedTCPPorts = [ 4001 ];
25
+
networking.firewall.allowedUDPPorts = [ 4001 ];
26
+
}
+192
machines/starkstrom/ipfs_upload.py
+192
machines/starkstrom/ipfs_upload.py
···
1
+
"""Anonymous tarball -> IPFS pinning endpoint for ipfs.lassul.us.
2
+
3
+
POST a tar archive (optionally gzip/xz/bzip2 compressed) as the raw request
4
+
body. It is extracted to a scratch dir, added to IPFS recursively (copied into
5
+
the blockstore -- NOT --nocopy), and recursively pinned. Each upload's pin is a
6
+
GC root, so the content survives `ipfs repo gc` until an operator unpins it.
7
+
The root CID is returned as JSON; the scratch files are then deleted.
8
+
9
+
There is no mutable name: reads are immutable at /ipfs/<cid>/. Re-upload to
10
+
"update" -- you get a new CID back.
11
+
"""
12
+
import http.server
13
+
import json
14
+
import os
15
+
import shutil
16
+
import socketserver
17
+
import subprocess
18
+
import tarfile
19
+
import tempfile
20
+
import time
21
+
22
+
IPFS_BIN = os.environ.get("IPFS_BIN", "ipfs")
23
+
IPFS_API = os.environ.get("IPFS_API", "/ip4/127.0.0.1/tcp/5001")
24
+
STAGING = os.environ.get("STAGING", "/var/lib/ipfs/incoming")
25
+
ROOTS_FILE = os.environ.get("ROOTS_FILE", "/var/lib/ipfs/gc-roots")
26
+
GATEWAY_BASE = os.environ.get("GATEWAY_BASE", "https://ipfs.lassul.us")
27
+
BIND = os.environ.get("BIND", "127.0.0.1")
28
+
PORT = int(os.environ.get("PORT", "8090"))
29
+
MAX_BYTES = int(os.environ.get("MAX_BYTES", str(8 * 1024**3)))
30
+
MAX_EXTRACT_BYTES = int(os.environ.get("MAX_EXTRACT_BYTES", str(20 * 1024**3)))
31
+
MAX_FILES = int(os.environ.get("MAX_FILES", "200000"))
32
+
# reject uploads once the IPFS filesystem has less than this much free space
33
+
MIN_FREE_BYTES = int(os.environ.get("MIN_FREE_BYTES", str(300 * 1024**3)))
34
+
DISK_PATH = os.environ.get("DISK_PATH", "/var/lib/ipfs")
35
+
36
+
_HOST = GATEWAY_BASE.split("://", 1)[-1]
37
+
HELP = (
38
+
"%s - anonymous IPFS pinning\n\n"
39
+
"Upload a directory; the response is its CID. Read it at /ipfs/<cid>/.\n\n"
40
+
" tar -C mydir -c . | curl --data-binary @- %s/\n"
41
+
" # compressed tars work too (tar -cz / -cJ / -cj); do NOT set Content-Encoding.\n\n"
42
+
"Re-upload to 'update' (new CID). Content stays pinned until an operator\n"
43
+
"prunes it (ipfs-roots rm <cid> && ipfs-gc).\n"
44
+
).encode() % (_HOST.encode(), GATEWAY_BASE.encode())
45
+
46
+
47
+
def _add_to_ipfs(path):
48
+
out = subprocess.run(
49
+
[IPFS_BIN, "--api", IPFS_API, "add", "-Q", "-r",
50
+
"--cid-version=1", "--pin=true", path],
51
+
capture_output=True, text=True, check=True,
52
+
)
53
+
return out.stdout.strip().splitlines()[-1]
54
+
55
+
56
+
class Handler(http.server.BaseHTTPRequestHandler):
57
+
server_version = "ipfs-upload/1"
58
+
59
+
def _json(self, code, obj):
60
+
body = json.dumps(obj).encode()
61
+
self.send_response(code)
62
+
self.send_header("Content-Type", "application/json")
63
+
self.send_header("Content-Length", str(len(body)))
64
+
self.end_headers()
65
+
self.wfile.write(body)
66
+
67
+
def do_GET(self):
68
+
self.send_response(200)
69
+
self.send_header("Content-Type", "text/plain; charset=utf-8")
70
+
self.send_header("Content-Length", str(len(HELP)))
71
+
self.end_headers()
72
+
self.wfile.write(HELP)
73
+
74
+
def do_POST(self):
75
+
try:
76
+
free = shutil.disk_usage(DISK_PATH).free
77
+
except OSError:
78
+
free = None
79
+
if free is not None and free < MIN_FREE_BYTES:
80
+
return self._json(507, {
81
+
"error": "not enough space left",
82
+
"free_bytes": free,
83
+
"min_free_bytes": MIN_FREE_BYTES,
84
+
})
85
+
os.makedirs(STAGING, exist_ok=True)
86
+
tmp = tempfile.NamedTemporaryFile(dir=STAGING, suffix=".tar", delete=False)
87
+
workdir = None
88
+
try:
89
+
if not self._recv_body(tmp):
90
+
return
91
+
tmp.close()
92
+
workdir = tempfile.mkdtemp(dir=STAGING)
93
+
if not self._extract(tmp.name, workdir):
94
+
return
95
+
try:
96
+
cid = _add_to_ipfs(workdir)
97
+
except subprocess.CalledProcessError as e:
98
+
return self._json(500, {
99
+
"error": "ipfs add failed",
100
+
"detail": (e.stderr or "").strip()[-500:],
101
+
})
102
+
try:
103
+
with open(ROOTS_FILE, "a") as f:
104
+
f.write("%s\t%s\t%s\n" % (
105
+
cid,
106
+
time.strftime("%Y-%m-%dT%H:%M:%S%z"),
107
+
self.client_address[0],
108
+
))
109
+
except OSError:
110
+
pass
111
+
self._json(200, {"cid": cid, "url": "%s/ipfs/%s/" % (GATEWAY_BASE, cid)})
112
+
finally:
113
+
if workdir:
114
+
shutil.rmtree(workdir, ignore_errors=True)
115
+
try:
116
+
os.unlink(tmp.name)
117
+
except OSError:
118
+
pass
119
+
120
+
def _recv_body(self, tmp):
121
+
length = self.headers.get("Content-Length")
122
+
total = 0
123
+
if length is not None:
124
+
try:
125
+
remaining = int(length)
126
+
except ValueError:
127
+
self._json(400, {"error": "bad Content-Length"})
128
+
return False
129
+
if remaining > MAX_BYTES:
130
+
self._json(413, {"error": "archive too large"})
131
+
return False
132
+
while remaining > 0:
133
+
chunk = self.rfile.read(min(1 << 16, remaining))
134
+
if not chunk:
135
+
break
136
+
tmp.write(chunk)
137
+
total += len(chunk)
138
+
remaining -= len(chunk)
139
+
else:
140
+
while True:
141
+
chunk = self.rfile.read(1 << 16)
142
+
if not chunk:
143
+
break
144
+
total += len(chunk)
145
+
if total > MAX_BYTES:
146
+
self._json(413, {"error": "archive too large"})
147
+
return False
148
+
tmp.write(chunk)
149
+
if total == 0:
150
+
self._json(400, {"error": "empty body; POST a tar archive"})
151
+
return False
152
+
return True
153
+
154
+
def _extract(self, tar_path, workdir):
155
+
try:
156
+
with tarfile.open(tar_path, "r:*") as tar:
157
+
nfiles = 0
158
+
nbytes = 0
159
+
for m in tar.getmembers():
160
+
if m.isdir():
161
+
continue
162
+
nfiles += 1
163
+
nbytes += m.size
164
+
if nfiles > MAX_FILES:
165
+
self._json(413, {"error": "too many files"})
166
+
return False
167
+
if nbytes > MAX_EXTRACT_BYTES:
168
+
self._json(413, {"error": "extracted size too large"})
169
+
return False
170
+
# Python 3.12 'data' filter: strips setuid/dev nodes and
171
+
# rejects absolute paths, .. traversal and links escaping dest.
172
+
tar.extractall(workdir, filter="data")
173
+
except (tarfile.TarError, OSError, ValueError) as e:
174
+
self._json(400, {"error": "bad archive", "detail": str(e)[:500]})
175
+
return False
176
+
return True
177
+
178
+
179
+
class Server(socketserver.ThreadingMixIn, http.server.HTTPServer):
180
+
daemon_threads = True
181
+
allow_reuse_address = True
182
+
183
+
184
+
def main():
185
+
os.makedirs(STAGING, exist_ok=True)
186
+
httpd = Server((BIND, PORT), Handler)
187
+
print("ipfs-upload listening on %s:%d" % (BIND, PORT), flush=True)
188
+
httpd.serve_forever()
189
+
190
+
191
+
if __name__ == "__main__":
192
+
main()