This repository has no description
4.8 kB
135 lines
1{
2 config,
3 pkgs,
4 self,
5 ...
6}:
7let
8 # TODO: create this bucket (public) + an application key for it in B2
9 bucket = "lassulus-nix-cache";
10 region = "eu-central-003";
11 endpoint = "s3.${region}.backblazeb2.com";
12
13 vars = config.clan.core.vars.generators;
14in
15{
16 imports = [ self.inputs.niks3.nixosModules.niks3 ];
17
18 # B2 credentials (prompted), the niks3 API token (generated; one copy per
19 # service user that reads it).
20 clan.core.vars.generators.niks3 = {
21 prompts.b2_key_id = {
22 description = "Backblaze B2 keyID with access to the ${bucket} bucket";
23 type = "line";
24 };
25 prompts.b2_application_key = {
26 description = "Backblaze B2 applicationKey for that keyID";
27 type = "hidden";
28 };
29 files."s3-access-key".owner = "niks3";
30 files."s3-secret-key".owner = "niks3";
31 files."api-token".owner = "niks3";
32 files."api-token-grpc".owner = "nix-grpc-daemon";
33 runtimeInputs = [
34 pkgs.coreutils
35 pkgs.openssl
36 ];
37 script = ''
38 tr -d '\n' < "$prompts/b2_key_id" > "$out/s3-access-key"
39 tr -d '\n' < "$prompts/b2_application_key" > "$out/s3-secret-key"
40 openssl rand -hex 32 | tr -d '\n' > "$out/api-token"
41 cp "$out/api-token" "$out/api-token-grpc"
42 '';
43 };
44
45 # Public half is read by configs/default.nix from the vars dir.
46 clan.core.vars.generators.nix-cache-signing = {
47 files."key.sec".owner = "niks3";
48 files."key.pub".secret = false;
49 runtimeInputs = [ pkgs.nix ];
50 script = ''
51 nix --extra-experimental-features nix-command key generate-secret \
52 --key-name cache.lassul.us-1 > "$out/key.sec"
53 nix --extra-experimental-features nix-command key convert-secret-to-public \
54 < "$out/key.sec" > "$out/key.pub"
55 '';
56 };
57
58 services.niks3 = {
59 enable = true;
60 s3 = {
61 inherit endpoint bucket region;
62 accessKeyFile = vars.niks3.files."s3-access-key".path;
63 secretKeyFile = vars.niks3.files."s3-secret-key".path;
64 };
65 apiTokenFile = vars.niks3.files."api-token".path;
66 signKeyFiles = [ vars.nix-cache-signing.files."key.sec".path ];
67 cacheUrl = "https://cache.lassul.us";
68 # Behind cache.nixos.org (40): nix asks upstream first, so B2 only sees
69 # lookups for our own paths. At niks3's default 30 every closure path hit
70 # B2 first, and a farm run's ~1M misses got throttled with 503s.
71 priority = 50;
72 # The CI build farm (tools/ci-farm) pushes with the job's GitHub OIDC
73 # token. Fork PRs don't get one.
74 oidc.providers.github = {
75 issuer = "https://token.actions.githubusercontent.com";
76 audience = "https://niks3.lassul.us";
77 boundClaims.repository = [ "lassulus/superconfig" ];
78 };
79 nginx = {
80 enable = true;
81 domain = "niks3.lassul.us";
82 };
83 };
84
85 # Reads straight from the public bucket. niks3 uploads the metadata
86 # zstd-compressed with Content-Encoding: zstd; set it here so nix decodes
87 # it whether or not the bucket hands the header back. Same rules as
88 # clan-infra's Fastly config (modules/terranix/cache-new.nix): narinfo, .ls,
89 # realisations/ and log/. NARs (nar/*.nar.zst) are plain objects, nix
90 # decodes them via the narinfo's Compression field.
91 # `resolve`: looked up at runtime via resolved's stub, so an unresolvable
92 # B2 at startup doesn't keep nginx (and every other vhost) from starting.
93 # `keepalive` + HTTP/1.1 below: reuse TLS connections to B2. Without them
94 # nginx spoke HTTP/1.0 with a new handshake per request; at a farm run's
95 # ~400 req/s B2 started closing connections mid-response.
96 services.nginx.appendHttpConfig = ''
97 upstream cache_lassul_us_b2 {
98 zone cache_lassul_us_b2 64k;
99 resolver 127.0.0.53 ipv6=off;
100 server ${endpoint}:443 resolve;
101 keepalive 64;
102 keepalive_requests 10000;
103 keepalive_timeout 60s;
104 }
105 map $uri $cache_lassul_us_encoding {
106 default "";
107 ~\.(narinfo|ls)$ zstd;
108 ~^/realisations/ zstd;
109 ~^/log/ zstd;
110 }
111 '';
112 services.nginx.virtualHosts."cache.lassul.us" = {
113 enableACME = true;
114 forceSSL = true;
115 locations."/".extraConfig = ''
116 proxy_pass https://cache_lassul_us_b2/${bucket}/;
117 proxy_set_header Host ${endpoint};
118 proxy_ssl_server_name on;
119 proxy_ssl_name ${endpoint};
120 proxy_http_version 1.1;
121 proxy_set_header Connection "";
122 # cache keys never have a query string; don't forward one to S3
123 set $args "";
124 # S3 answers 403 for keys missing from a bucket it won't list
125 proxy_intercept_errors on;
126 error_page 403 =404 @notfound;
127
128 gzip off;
129 proxy_hide_header Content-Encoding;
130 # only sent on 2xx/3xx, and not at all when the map yields ""
131 add_header Content-Encoding $cache_lassul_us_encoding;
132 '';
133 locations."@notfound".return = "404";
134 };
135}