This repository has no description
5.1 kB
139 lines
1{
2 config,
3 pkgs,
4 self,
5 ...
6}:
7let
8 # TODO: create this bucket (public) + an application key for it in B2
9 bucket = "lassulus-nix-cache";
10 region = "eu-central-003";
11 endpoint = "s3.${region}.backblazeb2.com";
12
13 vars = config.clan.core.vars.generators;
14in
15{
16 imports = [ self.inputs.niks3.nixosModules.niks3 ];
17
18 # B2 credentials (prompted), the niks3 API token (generated).
19 clan.core.vars.generators.niks3 = {
20 prompts.b2_key_id = {
21 description = "Backblaze B2 keyID with access to the ${bucket} bucket";
22 type = "line";
23 };
24 prompts.b2_application_key = {
25 description = "Backblaze B2 applicationKey for that keyID";
26 type = "hidden";
27 };
28 files."s3-access-key".owner = "niks3";
29 files."s3-secret-key".owner = "niks3";
30 files."api-token".owner = "niks3";
31 runtimeInputs = [
32 pkgs.coreutils
33 pkgs.openssl
34 ];
35 script = ''
36 tr -d '\n' < "$prompts/b2_key_id" > "$out/s3-access-key"
37 tr -d '\n' < "$prompts/b2_application_key" > "$out/s3-secret-key"
38 openssl rand -hex 32 | tr -d '\n' > "$out/api-token"
39 '';
40 };
41
42 # Public half is read by configs/default.nix from the vars dir.
43 clan.core.vars.generators.nix-cache-signing = {
44 files."key.sec".owner = "niks3";
45 files."key.pub".secret = false;
46 runtimeInputs = [ pkgs.nix ];
47 script = ''
48 nix --extra-experimental-features nix-command key generate-secret \
49 --key-name cache.lassul.us-1 > "$out/key.sec"
50 nix --extra-experimental-features nix-command key convert-secret-to-public \
51 < "$out/key.sec" > "$out/key.pub"
52 '';
53 };
54
55 services.niks3 = {
56 enable = true;
57 s3 = {
58 inherit endpoint bucket region;
59 accessKeyFile = vars.niks3.files."s3-access-key".path;
60 secretKeyFile = vars.niks3.files."s3-secret-key".path;
61 };
62 apiTokenFile = vars.niks3.files."api-token".path;
63 signKeyFiles = [ vars.nix-cache-signing.files."key.sec".path ];
64 cacheUrl = "https://cache.lassul.us";
65 # Behind cache.nixos.org (40): nix asks upstream first, so B2 only sees
66 # lookups for our own paths. At niks3's default 30 every closure path of
67 # a CI build would hit B2 first, and B2 throttles that many misses with
68 # 503s.
69 priority = 50;
70 # CI (.github/workflows/ci.yml) pushes with the job's GitHub OIDC token.
71 # Fork PRs don't get one.
72 oidc.providers.github = {
73 issuer = "https://token.actions.githubusercontent.com";
74 audience = "https://niks3.lassul.us";
75 boundClaims.repository = [ "lassulus/superconfig" ];
76 };
77 nginx = {
78 enable = true;
79 domain = "niks3.lassul.us";
80 };
81 };
82
83 # Reads straight from the public bucket. niks3 uploads the metadata
84 # zstd-compressed with Content-Encoding: zstd; set it here so nix decodes
85 # it whether or not the bucket hands the header back. Same rules as
86 # clan-infra's Fastly config (modules/terranix/cache-new.nix): narinfo, .ls,
87 # realisations/ and log/. NARs (nar/*.nar.zst) are plain objects, nix
88 # decodes them via the narinfo's Compression field.
89 # `resolve`: looked up at runtime via resolved's stub, so an unresolvable
90 # B2 at startup doesn't keep nginx (and every other vhost) from starting.
91 # `keepalive` + HTTP/1.1 below: reuse TLS connections to B2. Without them
92 # nginx speaks HTTP/1.0 with a new handshake per request, and at hundreds
93 # of requests per second B2 closes connections mid-response.
94 # `max_fails=0`: B2 closes idle keepalive connections now and then. With the
95 # default max_fails=1 one such close marks that B2 address down for 10s;
96 # once all of them are, every request gets "no live upstreams" and a 502.
97 # A GET/HEAD on a closed reused connection is retried
98 # (default `proxy_next_upstream error timeout`) without using up a try, on
99 # the next address, or on the same one if it resolves to only one.
100 services.nginx.appendHttpConfig = ''
101 upstream cache_lassul_us_b2 {
102 zone cache_lassul_us_b2 64k;
103 resolver 127.0.0.53 ipv6=off;
104 server ${endpoint}:443 resolve max_fails=0;
105 keepalive 64;
106 keepalive_requests 10000;
107 keepalive_timeout 60s;
108 }
109 map $uri $cache_lassul_us_encoding {
110 default "";
111 ~\.(narinfo|ls)$ zstd;
112 ~^/realisations/ zstd;
113 ~^/log/ zstd;
114 }
115 '';
116 services.nginx.virtualHosts."cache.lassul.us" = {
117 enableACME = true;
118 forceSSL = true;
119 locations."/".extraConfig = ''
120 proxy_pass https://cache_lassul_us_b2/${bucket}/;
121 proxy_set_header Host ${endpoint};
122 proxy_ssl_server_name on;
123 proxy_ssl_name ${endpoint};
124 proxy_http_version 1.1;
125 proxy_set_header Connection "";
126 # cache keys never have a query string; don't forward one to S3
127 set $args "";
128 # S3 answers 403 for keys missing from a bucket it won't list
129 proxy_intercept_errors on;
130 error_page 403 =404 @notfound;
131
132 gzip off;
133 proxy_hide_header Content-Encoding;
134 # only sent on 2xx/3xx, and not at all when the map yields ""
135 add_header Content-Encoding $cache_lassul_us_encoding;
136 '';
137 locations."@notfound".return = "404";
138 };
139}