This repository has no description
7.1 kB
192 lines
1"""Anonymous tarball -> IPFS pinning endpoint for ipfs.lassul.us.
2
3POST a tar archive (optionally gzip/xz/bzip2 compressed) as the raw request
4body. It is extracted to a scratch dir, added to IPFS recursively (copied into
5the blockstore -- NOT --nocopy), and recursively pinned. Each upload's pin is a
6GC root, so the content survives `ipfs repo gc` until an operator unpins it.
7The root CID is returned as JSON; the scratch files are then deleted.
8
9There is no mutable name: reads are immutable at /ipfs/<cid>/. Re-upload to
10"update" -- you get a new CID back.
11"""
12import http.server
13import json
14import os
15import shutil
16import socketserver
17import subprocess
18import tarfile
19import tempfile
20import time
21
22IPFS_BIN = os.environ.get("IPFS_BIN", "ipfs")
23IPFS_API = os.environ.get("IPFS_API", "/ip4/127.0.0.1/tcp/5001")
24STAGING = os.environ.get("STAGING", "/var/lib/ipfs/incoming")
25ROOTS_FILE = os.environ.get("ROOTS_FILE", "/var/lib/ipfs/gc-roots")
26GATEWAY_BASE = os.environ.get("GATEWAY_BASE", "https://ipfs.lassul.us")
27BIND = os.environ.get("BIND", "127.0.0.1")
28PORT = int(os.environ.get("PORT", "8090"))
29MAX_BYTES = int(os.environ.get("MAX_BYTES", str(8 * 1024**3)))
30MAX_EXTRACT_BYTES = int(os.environ.get("MAX_EXTRACT_BYTES", str(20 * 1024**3)))
31MAX_FILES = int(os.environ.get("MAX_FILES", "200000"))
32# reject uploads once the IPFS filesystem has less than this much free space
33MIN_FREE_BYTES = int(os.environ.get("MIN_FREE_BYTES", str(300 * 1024**3)))
34DISK_PATH = os.environ.get("DISK_PATH", "/var/lib/ipfs")
35
36_HOST = GATEWAY_BASE.split("://", 1)[-1]
37HELP = (
38 "%s - anonymous IPFS pinning\n\n"
39 "Upload a directory; the response is its CID. Read it at /ipfs/<cid>/.\n\n"
40 " tar -C mydir -c . | curl --data-binary @- %s/\n"
41 " # compressed tars work too (tar -cz / -cJ / -cj); do NOT set Content-Encoding.\n\n"
42 "Re-upload to 'update' (new CID). Content stays pinned until an operator\n"
43 "prunes it (ipfs-roots rm <cid> && ipfs-gc).\n"
44).encode() % (_HOST.encode(), GATEWAY_BASE.encode())
45
46
47def _add_to_ipfs(path):
48 out = subprocess.run(
49 [IPFS_BIN, "--api", IPFS_API, "add", "-Q", "-r",
50 "--cid-version=1", "--pin=true", path],
51 capture_output=True, text=True, check=True,
52 )
53 return out.stdout.strip().splitlines()[-1]
54
55
56class Handler(http.server.BaseHTTPRequestHandler):
57 server_version = "ipfs-upload/1"
58
59 def _json(self, code, obj):
60 body = json.dumps(obj).encode()
61 self.send_response(code)
62 self.send_header("Content-Type", "application/json")
63 self.send_header("Content-Length", str(len(body)))
64 self.end_headers()
65 self.wfile.write(body)
66
67 def do_GET(self):
68 self.send_response(200)
69 self.send_header("Content-Type", "text/plain; charset=utf-8")
70 self.send_header("Content-Length", str(len(HELP)))
71 self.end_headers()
72 self.wfile.write(HELP)
73
74 def do_POST(self):
75 try:
76 free = shutil.disk_usage(DISK_PATH).free
77 except OSError:
78 free = None
79 if free is not None and free < MIN_FREE_BYTES:
80 return self._json(507, {
81 "error": "not enough space left",
82 "free_bytes": free,
83 "min_free_bytes": MIN_FREE_BYTES,
84 })
85 os.makedirs(STAGING, exist_ok=True)
86 tmp = tempfile.NamedTemporaryFile(dir=STAGING, suffix=".tar", delete=False)
87 workdir = None
88 try:
89 if not self._recv_body(tmp):
90 return
91 tmp.close()
92 workdir = tempfile.mkdtemp(dir=STAGING)
93 if not self._extract(tmp.name, workdir):
94 return
95 try:
96 cid = _add_to_ipfs(workdir)
97 except subprocess.CalledProcessError as e:
98 return self._json(500, {
99 "error": "ipfs add failed",
100 "detail": (e.stderr or "").strip()[-500:],
101 })
102 try:
103 with open(ROOTS_FILE, "a") as f:
104 f.write("%s\t%s\t%s\n" % (
105 cid,
106 time.strftime("%Y-%m-%dT%H:%M:%S%z"),
107 self.client_address[0],
108 ))
109 except OSError:
110 pass
111 self._json(200, {"cid": cid, "url": "%s/ipfs/%s/" % (GATEWAY_BASE, cid)})
112 finally:
113 if workdir:
114 shutil.rmtree(workdir, ignore_errors=True)
115 try:
116 os.unlink(tmp.name)
117 except OSError:
118 pass
119
120 def _recv_body(self, tmp):
121 length = self.headers.get("Content-Length")
122 total = 0
123 if length is not None:
124 try:
125 remaining = int(length)
126 except ValueError:
127 self._json(400, {"error": "bad Content-Length"})
128 return False
129 if remaining > MAX_BYTES:
130 self._json(413, {"error": "archive too large"})
131 return False
132 while remaining > 0:
133 chunk = self.rfile.read(min(1 << 16, remaining))
134 if not chunk:
135 break
136 tmp.write(chunk)
137 total += len(chunk)
138 remaining -= len(chunk)
139 else:
140 while True:
141 chunk = self.rfile.read(1 << 16)
142 if not chunk:
143 break
144 total += len(chunk)
145 if total > MAX_BYTES:
146 self._json(413, {"error": "archive too large"})
147 return False
148 tmp.write(chunk)
149 if total == 0:
150 self._json(400, {"error": "empty body; POST a tar archive"})
151 return False
152 return True
153
154 def _extract(self, tar_path, workdir):
155 try:
156 with tarfile.open(tar_path, "r:*") as tar:
157 nfiles = 0
158 nbytes = 0
159 for m in tar.getmembers():
160 if m.isdir():
161 continue
162 nfiles += 1
163 nbytes += m.size
164 if nfiles > MAX_FILES:
165 self._json(413, {"error": "too many files"})
166 return False
167 if nbytes > MAX_EXTRACT_BYTES:
168 self._json(413, {"error": "extracted size too large"})
169 return False
170 # Python 3.12 'data' filter: strips setuid/dev nodes and
171 # rejects absolute paths, .. traversal and links escaping dest.
172 tar.extractall(workdir, filter="data")
173 except (tarfile.TarError, OSError, ValueError) as e:
174 self._json(400, {"error": "bad archive", "detail": str(e)[:500]})
175 return False
176 return True
177
178
179class Server(socketserver.ThreadingMixIn, http.server.HTTPServer):
180 daemon_threads = True
181 allow_reuse_address = True
182
183
184def main():
185 os.makedirs(STAGING, exist_ok=True)
186 httpd = Server((BIND, PORT), Handler)
187 print("ipfs-upload listening on %s:%d" % (BIND, PORT), flush=True)
188 httpd.serve_forever()
189
190
191if __name__ == "__main__":
192 main()