This repository has no description
2configs/ipfs: add IPFS with auto-pin watcher for flix, client config for xerxes
author
lassulus
date
(Apr 3, 2026, 9:11 AM CEST)
commit
0ce40a35
0ce40a35d58837fac921b476bacf85a85cf24a98
parent
cc555725
cc555725c31536fe8e49f193315e377033f7c9cd
+236
-174
-164
2configs/ipfs-pin-watcher.nix
-164
2configs/ipfs-pin-watcher.nix
···
1
-
{ config, pkgs, lib, ... }:
2
-
let
3
-
cfg = config.services.ipfs-pin-watcher;
4
-
5
-
pinWatcherScript = pkgs.writers.writeBash "ipfs-pin-watcher" ''
6
-
set -efu
7
-
8
-
IPFS="${pkgs.kubo}/bin/ipfs"
9
-
INOTIFYWAIT="${pkgs.inotify-tools}/bin/inotifywait"
10
-
CID_MAP="${cfg.cidMapFile}"
11
-
12
-
# ensure cid map file exists
13
-
touch "$CID_MAP"
14
-
15
-
log() {
16
-
echo "[$(date -Iseconds)] $*"
17
-
}
18
-
19
-
add_file() {
20
-
local path="$1"
21
-
# skip if not a regular file
22
-
[ -f "$path" ] || return 0
23
-
# skip partial/temp files
24
-
case "$(basename "$path")" in
25
-
.* | *.part | *.tmp) return 0 ;;
26
-
esac
27
-
28
-
log "Adding $path"
29
-
cid=$($IPFS add --nocopy --pin --quieter "$path" 2>&1) || {
30
-
log "Failed to add $path: $cid"
31
-
return 0
32
-
}
33
-
log "Pinned $path -> $cid"
34
-
# store mapping (remove old entry for this path first)
35
-
${pkgs.gnused}/bin/sed -i "\|^[^ ]* $path$|d" "$CID_MAP"
36
-
echo "$cid $path" >> "$CID_MAP"
37
-
}
38
-
39
-
remove_file() {
40
-
local path="$1"
41
-
# look up CID from our map
42
-
cid=$(${pkgs.gawk}/bin/awk -v p="$path" '$0 ~ p {print $1; exit}' "$CID_MAP")
43
-
if [ -n "$cid" ]; then
44
-
log "Unpinning $cid ($path)"
45
-
$IPFS pin rm "$cid" 2>/dev/null || true
46
-
# remove from map
47
-
${pkgs.gnused}/bin/sed -i "\|^[^ ]* $path$|d" "$CID_MAP"
48
-
else
49
-
log "No CID found for removed file: $path"
50
-
fi
51
-
}
52
-
53
-
# initial sync: add all existing files
54
-
log "Starting initial sync of watched directories..."
55
-
for dir in ${lib.escapeShellArgs cfg.watchDirs}; do
56
-
if [ -d "$dir" ]; then
57
-
${pkgs.findutils}/bin/find "$dir" -type f | while read -r f; do
58
-
# skip if already in map
59
-
if ! ${pkgs.gnugrep}/bin/grep -q " $f$" "$CID_MAP" 2>/dev/null; then
60
-
add_file "$f"
61
-
fi
62
-
done
63
-
fi
64
-
done
65
-
log "Initial sync complete"
66
-
67
-
# clean up stale entries (files in map but no longer on disk)
68
-
log "Cleaning stale entries..."
69
-
while IFS=' ' read -r cid path; do
70
-
if [ ! -f "$path" ]; then
71
-
log "Stale entry: $path (CID: $cid)"
72
-
$IPFS pin rm "$cid" 2>/dev/null || true
73
-
${pkgs.gnused}/bin/sed -i "\|^[^ ]* $path$|d" "$CID_MAP"
74
-
fi
75
-
done < "$CID_MAP"
76
-
log "Cleanup complete"
77
-
78
-
# watch for changes
79
-
log "Watching directories: ${lib.concatStringsSep ", " cfg.watchDirs}"
80
-
$INOTIFYWAIT -m -r \
81
-
-e close_write \
82
-
-e moved_to \
83
-
-e moved_from \
84
-
-e delete \
85
-
${lib.escapeShellArgs cfg.watchDirs} |
86
-
while read -r dir event file; do
87
-
path="''${dir}''${file}"
88
-
case "$event" in
89
-
CLOSE_WRITE*|MOVED_TO*)
90
-
add_file "$path"
91
-
;;
92
-
DELETE*|MOVED_FROM*)
93
-
remove_file "$path"
94
-
;;
95
-
esac
96
-
done
97
-
'';
98
-
in
99
-
{
100
-
options.services.ipfs-pin-watcher = {
101
-
enable = lib.mkEnableOption "IPFS auto-pin watcher";
102
-
watchDirs = lib.mkOption {
103
-
type = lib.types.listOf lib.types.str;
104
-
default = [ ];
105
-
description = "Directories to watch for new files to auto-pin in IPFS";
106
-
example = [ "/var/download/movies" "/var/download/shows" "/var/download/games" ];
107
-
};
108
-
cidMapFile = lib.mkOption {
109
-
type = lib.types.str;
110
-
default = "/var/lib/ipfs/cid-map.txt";
111
-
description = "File to store CID-to-path mappings for unpinning";
112
-
};
113
-
};
114
-
115
-
config = lib.mkIf cfg.enable {
116
-
services.kubo = {
117
-
enable = true;
118
-
settings = {
119
-
Experimental.FilestoreEnabled = true;
120
-
# listen on all interfaces so it's reachable via retiolum
121
-
Addresses = {
122
-
API = [
123
-
"/ip4/0.0.0.0/tcp/5001"
124
-
"/ip6/::/tcp/5001"
125
-
];
126
-
Gateway = [
127
-
"/ip4/0.0.0.0/tcp/8080"
128
-
"/ip6/::/tcp/8080"
129
-
];
130
-
};
131
-
# set a reasonable storage max for garbage collection
132
-
Datastore.StorageMax = "100GB";
133
-
};
134
-
};
135
-
136
-
# bump inotify limits for large directories
137
-
boot.kernel.sysctl."fs.inotify.max_user_watches" = 1048576;
138
-
139
-
systemd.services.ipfs-pin-watcher = {
140
-
description = "Auto-pin files to IPFS using inotify";
141
-
after = [ "kubo.service" ];
142
-
requires = [ "kubo.service" ];
143
-
wantedBy = [ "multi-user.target" ];
144
-
serviceConfig = {
145
-
ExecStart = pinWatcherScript;
146
-
Restart = "always";
147
-
RestartSec = "10s";
148
-
User = config.services.kubo.user;
149
-
Group = config.services.kubo.group;
150
-
# ipfs needs access to download dirs
151
-
SupplementaryGroups = [ "download" ];
152
-
};
153
-
};
154
-
155
-
networking.firewall.allowedTCPPorts = [
156
-
4001 # IPFS swarm
157
-
5001 # IPFS API
158
-
8080 # IPFS gateway
159
-
];
160
-
networking.firewall.allowedUDPPorts = [
161
-
4001 # IPFS swarm (QUIC)
162
-
];
163
-
};
164
-
}
+63
-10
2configs/ipfs.nix
+63
-10
2configs/ipfs.nix
···
1
-
{ pkgs, ... }:
2
-
{
3
-
services.ipfs = {
4
-
enable = true;
5
-
};
6
-
users.users.mainUser.extraGroups = [ "ipfs" ];
7
-
systemd.services.ipfs.serviceConfig.ExecStartPost = [
8
-
(pkgs.writers.writeDash "fix-permission" ''
9
-
chmod g+r /var/lib/ipfs/config
10
-
'')
1
+
{ ... }:
2
+
let
3
+
neoprismPeerID = "12D3KooWDccB4dVFt6uyfD5Yc3brwj4JLSztPeJpY3yg965iB5uT";
4
+
neoprismAddrs = [
5
+
"/ip6/42:0:ce16::99/tcp/4001/p2p/${neoprismPeerID}"
6
+
"/ip4/95.217.192.59/tcp/4001/p2p/${neoprismPeerID}"
11
7
];
8
+
in
9
+
{
10
+
services.kubo = {
11
+
enable = true;
12
+
settings = {
13
+
Addresses = {
14
+
API = [
15
+
"/ip4/127.0.0.1/tcp/5001"
16
+
"/ip6/::1/tcp/5001"
17
+
];
18
+
Gateway = [
19
+
"/ip4/127.0.0.1/tcp/8089"
20
+
"/ip6/::1/tcp/8089"
21
+
];
22
+
# don't listen for incoming swarm connections
23
+
Swarm = [ ];
24
+
};
25
+
26
+
# only connect to neoprism, no public DHT
27
+
Bootstrap = neoprismAddrs;
28
+
Peering.Peers = [
29
+
{
30
+
ID = neoprismPeerID;
31
+
Addrs = [
32
+
"/ip6/42:0:ce16::99/tcp/4001"
33
+
"/ip4/95.217.192.59/tcp/4001"
34
+
];
35
+
}
36
+
];
37
+
38
+
# minimal connections
39
+
Swarm.ConnMgr = {
40
+
LowWater = 1;
41
+
HighWater = 5;
42
+
GracePeriod = "10s";
43
+
};
44
+
Swarm.Transports.Network.TCP = true;
45
+
Swarm.Transports.Network.QUIC = false;
46
+
Swarm.RelayClient.Enabled = false;
47
+
Swarm.RelayService.Enabled = false;
48
+
Swarm.ResourceMgr = {
49
+
Enabled = true;
50
+
MaxMemory = "128MB";
51
+
};
52
+
53
+
# don't route for others, don't announce to DHT
54
+
Routing.Type = "none";
55
+
AutoTLS.AutoWSS = false;
56
+
57
+
# don't provide/announce content to the network
58
+
Provide.Enabled = false;
59
+
60
+
Datastore.StorageMax = "10GB";
61
+
};
62
+
};
63
+
64
+
# no firewall ports needed — we don't accept incoming connections
12
65
}
+3
machines/neoprism/config.nix
+3
machines/neoprism/config.nix
+1
machines/xerxes/config.nix
+1
machines/xerxes/config.nix
+169
2configs/services/flix/ipfs.nix
+169
2configs/services/flix/ipfs.nix
···
1
+
{
2
+
config,
3
+
pkgs,
4
+
lib,
5
+
...
6
+
}:
7
+
let
8
+
watchDirs = [
9
+
"/var/lib/ipfs/download/movies"
10
+
"/var/lib/ipfs/download/shows"
11
+
"/var/lib/ipfs/download/games"
12
+
];
13
+
14
+
cidMapFile = "/var/lib/ipfs/cid-map.txt";
15
+
16
+
pinWatcherScript = pkgs.writers.writeBash "ipfs-pin-watcher" ''
17
+
set -efu
18
+
19
+
IPFS="${pkgs.kubo}/bin/ipfs"
20
+
INOTIFYWAIT="${pkgs.inotify-tools}/bin/inotifywait"
21
+
CID_MAP="${cidMapFile}"
22
+
23
+
touch "$CID_MAP"
24
+
25
+
log() {
26
+
echo "[$(date -Iseconds)] $*"
27
+
}
28
+
29
+
add_file() {
30
+
local path="$1"
31
+
[ -f "$path" ] || return 0
32
+
case "$(basename "$path")" in
33
+
.* | *.part | *.tmp) return 0 ;;
34
+
esac
35
+
36
+
log "Adding $path"
37
+
cid=$($IPFS add --nocopy --pin --quieter "$path" 2>&1) || {
38
+
log "Failed to add $path: $cid"
39
+
return 0
40
+
}
41
+
log "Pinned $path -> $cid"
42
+
${pkgs.gnused}/bin/sed -i "\|^[^ ]* $path$|d" "$CID_MAP"
43
+
echo "$cid $path" >> "$CID_MAP"
44
+
}
45
+
46
+
remove_file() {
47
+
local path="$1"
48
+
cid=$(${pkgs.gawk}/bin/awk -v p="$path" '$0 ~ p {print $1; exit}' "$CID_MAP")
49
+
if [ -n "$cid" ]; then
50
+
log "Unpinning $cid ($path)"
51
+
$IPFS pin rm "$cid" 2>/dev/null || true
52
+
${pkgs.gnused}/bin/sed -i "\|^[^ ]* $path$|d" "$CID_MAP"
53
+
else
54
+
log "No CID found for removed file: $path"
55
+
fi
56
+
}
57
+
58
+
# initial sync
59
+
log "Starting initial sync..."
60
+
for dir in ${lib.escapeShellArgs watchDirs}; do
61
+
if [ -d "$dir" ]; then
62
+
${pkgs.findutils}/bin/find "$dir" -type f | while read -r f; do
63
+
if ! ${pkgs.gnugrep}/bin/grep -q " $f$" "$CID_MAP" 2>/dev/null; then
64
+
add_file "$f"
65
+
fi
66
+
done
67
+
fi
68
+
done
69
+
log "Initial sync complete"
70
+
71
+
# clean stale entries
72
+
log "Cleaning stale entries..."
73
+
while IFS=' ' read -r cid path; do
74
+
if [ ! -f "$path" ]; then
75
+
log "Stale entry: $path (CID: $cid)"
76
+
$IPFS pin rm "$cid" 2>/dev/null || true
77
+
${pkgs.gnused}/bin/sed -i "\|^[^ ]* $path$|d" "$CID_MAP"
78
+
fi
79
+
done < "$CID_MAP"
80
+
log "Cleanup complete"
81
+
82
+
# watch for changes
83
+
log "Watching directories: ${lib.concatStringsSep ", " watchDirs}"
84
+
$INOTIFYWAIT -m -r \
85
+
-e close_write \
86
+
-e moved_to \
87
+
-e moved_from \
88
+
-e delete \
89
+
${lib.escapeShellArgs watchDirs} |
90
+
while read -r dir event file; do
91
+
path="''${dir}''${file}"
92
+
case "$event" in
93
+
CLOSE_WRITE*|MOVED_TO*)
94
+
add_file "$path"
95
+
;;
96
+
DELETE*|MOVED_FROM*)
97
+
remove_file "$path"
98
+
;;
99
+
esac
100
+
done
101
+
'';
102
+
in
103
+
{
104
+
# symlink /var/download into IPFS root so --nocopy works
105
+
systemd.tmpfiles.rules = [
106
+
"L+ /var/lib/ipfs/download - - - - /var/download"
107
+
];
108
+
109
+
services.kubo = {
110
+
enable = true;
111
+
settings = {
112
+
Experimental.FilestoreEnabled = true;
113
+
Addresses = {
114
+
API = [
115
+
"/ip4/127.0.0.1/tcp/5001"
116
+
"/ip6/::1/tcp/5001"
117
+
];
118
+
Gateway = [
119
+
"/ip4/0.0.0.0/tcp/8089"
120
+
"/ip6/::/tcp/8089"
121
+
];
122
+
};
123
+
Datastore.StorageMax = "100GB";
124
+
Swarm.ConnMgr = {
125
+
LowWater = 20;
126
+
HighWater = 50;
127
+
GracePeriod = "10s";
128
+
};
129
+
# limit bandwidth: 5MB/s out, 5MB/s in
130
+
Swarm.Transports.Network.TCP = true;
131
+
Swarm.Transports.Network.QUIC = false;
132
+
Swarm.ResourceMgr = {
133
+
Enabled = true;
134
+
MaxMemory = "256MB";
135
+
};
136
+
# disable relay to reduce overhead
137
+
Swarm.RelayClient.Enabled = false;
138
+
Swarm.RelayService.Enabled = false;
139
+
# reduce DHT overhead - client mode only announces, doesn't route for others
140
+
Routing.Type = "autoclient";
141
+
};
142
+
};
143
+
144
+
boot.kernel.sysctl."fs.inotify.max_user_watches" = 1048576;
145
+
146
+
systemd.services.ipfs-pin-watcher = {
147
+
description = "Auto-pin files to IPFS using inotify";
148
+
after = [ "ipfs.service" ];
149
+
requires = [ "ipfs.service" ];
150
+
wantedBy = [ "multi-user.target" ];
151
+
serviceConfig = {
152
+
Environment = [ "IPFS_PATH=/var/lib/ipfs" ];
153
+
ExecStart = pinWatcherScript;
154
+
Restart = "always";
155
+
RestartSec = "10s";
156
+
User = config.services.kubo.user;
157
+
Group = config.services.kubo.group;
158
+
SupplementaryGroups = [ "radio_container" ];
159
+
};
160
+
};
161
+
162
+
networking.firewall.allowedTCPPorts = [
163
+
4001 # IPFS swarm
164
+
8089 # IPFS gateway
165
+
];
166
+
networking.firewall.allowedUDPPorts = [
167
+
4001 # IPFS swarm (QUIC)
168
+
];
169
+
}