This repository has no description
5.3 kB
141 lines
1{
2 config,
3 pkgs,
4 self,
5 ...
6}:
7let
8 # TODO: create this bucket (public) + an application key for it in B2
9 bucket = "lassulus-nix-cache";
10 region = "eu-central-003";
11 endpoint = "s3.${region}.backblazeb2.com";
12
13 vars = config.clan.core.vars.generators;
14in
15{
16 imports = [ self.inputs.niks3.nixosModules.niks3 ];
17
18 # B2 credentials (prompted), the niks3 API token (generated; one copy per
19 # service user that reads it).
20 clan.core.vars.generators.niks3 = {
21 prompts.b2_key_id = {
22 description = "Backblaze B2 keyID with access to the ${bucket} bucket";
23 type = "line";
24 };
25 prompts.b2_application_key = {
26 description = "Backblaze B2 applicationKey for that keyID";
27 type = "hidden";
28 };
29 files."s3-access-key".owner = "niks3";
30 files."s3-secret-key".owner = "niks3";
31 files."api-token".owner = "niks3";
32 files."api-token-grpc".owner = "nix-grpc-daemon";
33 runtimeInputs = [
34 pkgs.coreutils
35 pkgs.openssl
36 ];
37 script = ''
38 tr -d '\n' < "$prompts/b2_key_id" > "$out/s3-access-key"
39 tr -d '\n' < "$prompts/b2_application_key" > "$out/s3-secret-key"
40 openssl rand -hex 32 | tr -d '\n' > "$out/api-token"
41 cp "$out/api-token" "$out/api-token-grpc"
42 '';
43 };
44
45 # Public half is read by configs/default.nix from the vars dir.
46 clan.core.vars.generators.nix-cache-signing = {
47 files."key.sec".owner = "niks3";
48 files."key.pub".secret = false;
49 runtimeInputs = [ pkgs.nix ];
50 script = ''
51 nix --extra-experimental-features nix-command key generate-secret \
52 --key-name cache.lassul.us-1 > "$out/key.sec"
53 nix --extra-experimental-features nix-command key convert-secret-to-public \
54 < "$out/key.sec" > "$out/key.pub"
55 '';
56 };
57
58 services.niks3 = {
59 enable = true;
60 s3 = {
61 inherit endpoint bucket region;
62 accessKeyFile = vars.niks3.files."s3-access-key".path;
63 secretKeyFile = vars.niks3.files."s3-secret-key".path;
64 };
65 apiTokenFile = vars.niks3.files."api-token".path;
66 signKeyFiles = [ vars.nix-cache-signing.files."key.sec".path ];
67 cacheUrl = "https://cache.lassul.us";
68 # Behind cache.nixos.org (40): nix asks upstream first, so B2 only sees
69 # lookups for our own paths. At niks3's default 30 every closure path hit
70 # B2 first, and a farm run's ~1M misses got throttled with 503s.
71 priority = 50;
72 # The CI build farm (tools/ci-farm) pushes with the job's GitHub OIDC
73 # token. Fork PRs don't get one.
74 oidc.providers.github = {
75 issuer = "https://token.actions.githubusercontent.com";
76 audience = "https://niks3.lassul.us";
77 boundClaims.repository = [ "lassulus/superconfig" ];
78 };
79 nginx = {
80 enable = true;
81 domain = "niks3.lassul.us";
82 };
83 };
84
85 # Reads straight from the public bucket. niks3 uploads the metadata
86 # zstd-compressed with Content-Encoding: zstd; set it here so nix decodes
87 # it whether or not the bucket hands the header back. Same rules as
88 # clan-infra's Fastly config (modules/terranix/cache-new.nix): narinfo, .ls,
89 # realisations/ and log/. NARs (nar/*.nar.zst) are plain objects, nix
90 # decodes them via the narinfo's Compression field.
91 # `resolve`: looked up at runtime via resolved's stub, so an unresolvable
92 # B2 at startup doesn't keep nginx (and every other vhost) from starting.
93 # `keepalive` + HTTP/1.1 below: reuse TLS connections to B2. Without them
94 # nginx spoke HTTP/1.0 with a new handshake per request; at a farm run's
95 # ~400 req/s B2 started closing connections mid-response.
96 # `max_fails=0`: B2 closes idle keepalive connections now and then. With the
97 # default max_fails=1 one such close marked that B2 address down for 10s;
98 # once all of them were, every request got "no live upstreams" and a 502
99 # (failed a farm run). A GET/HEAD on a closed reused connection is retried
100 # (default `proxy_next_upstream error timeout`) without using up a try, on
101 # the next address, or on the same one if it resolves to only one.
102 services.nginx.appendHttpConfig = ''
103 upstream cache_lassul_us_b2 {
104 zone cache_lassul_us_b2 64k;
105 resolver 127.0.0.53 ipv6=off;
106 server ${endpoint}:443 resolve max_fails=0;
107 keepalive 64;
108 keepalive_requests 10000;
109 keepalive_timeout 60s;
110 }
111 map $uri $cache_lassul_us_encoding {
112 default "";
113 ~\.(narinfo|ls)$ zstd;
114 ~^/realisations/ zstd;
115 ~^/log/ zstd;
116 }
117 '';
118 services.nginx.virtualHosts."cache.lassul.us" = {
119 enableACME = true;
120 forceSSL = true;
121 locations."/".extraConfig = ''
122 proxy_pass https://cache_lassul_us_b2/${bucket}/;
123 proxy_set_header Host ${endpoint};
124 proxy_ssl_server_name on;
125 proxy_ssl_name ${endpoint};
126 proxy_http_version 1.1;
127 proxy_set_header Connection "";
128 # cache keys never have a query string; don't forward one to S3
129 set $args "";
130 # S3 answers 403 for keys missing from a bucket it won't list
131 proxy_intercept_errors on;
132 error_page 403 =404 @notfound;
133
134 gzip off;
135 proxy_hide_header Content-Encoding;
136 # only sent on 2xx/3xx, and not at all when the map yields ""
137 add_header Content-Encoding $cache_lassul_us_encoding;
138 '';
139 locations."@notfound".return = "404";
140 };
141}