This repository has no description
6.7 kB
231 lines
1{
2 self,
3 config,
4 lib,
5 pkgs,
6 ...
7}:
8let
9 inherit (self.inputs.kartei) hosts;
10
11 # consul server peers; stockholm policy (krebs/3modules/kartei/overlay.nix),
12 # not kartei data
13 consulHosts = [
14 "aergia"
15 "blue"
16 "coaxmetal"
17 "daedalus"
18 "green"
19 "icarus"
20 "ignavia"
21 "littleT"
22 "massulus"
23 "neoprism"
24 "orange"
25 "radio"
26 "shodan"
27 "skynet"
28 "styx"
29 "ubik"
30 "yellow"
31 ];
32in
33{
34 services.consul = {
35 enable = true;
36 # dropPrivileges = false;
37 webUi = true;
38 # interface.bind = "retiolum";
39 extraConfig = {
40 bind_addr = hosts.${config.networking.hostName}.nets.retiolum.ip4.addr;
41 bootstrap_expect = 3;
42 server = true;
43 # retry_join = config.services.consul.extraConfig.start_join;
44 retry_join = map (name: lib.head hosts.${name}.nets.retiolum.aliases) consulHosts;
45 rejoin_after_leave = true;
46
47 # try to fix random lock loss on leader reelection
48 retry_interval = "3s";
49
50 # Autopilot configuration for better automatic failure handling
51 autopilot = {
52 # Automatically clean up dead servers
53 cleanup_dead_servers = true;
54 # How long a server must be stable before promoting
55 server_stabilization_time = "10s";
56 # Allow cluster to operate with 2 of 3 nodes (majority quorum)
57 min_quorum = 2;
58 };
59
60 # Performance tuning - slightly relaxed for network tolerance
61 performance = {
62 # 2 = default, gives more time for slow/unstable networks
63 # 1 was too aggressive and caused leader flapping
64 raft_multiplier = 2;
65 };
66
67 # Reconnect timeout for failed nodes (default 72h, reduce for faster cleanup)
68 reconnect_timeout = "24h";
69
70 # Leave on terminate for cleaner shutdowns
71 leave_on_terminate = true;
72 };
73 };
74
75 # Add systemd override for easier recovery
76 systemd.services.consul = {
77 # Add pre-start script to clean up potential issues
78 preStart = ''
79 # Ensure proper permissions
80 chown -R consul:consul /var/lib/consul
81 '';
82
83 # Add recovery commands to service
84 serviceConfig = {
85 # Allow consul to restart on failure
86 Restart = "on-failure";
87 RestartSec = "5s";
88 StartLimitBurst = 3;
89 StartLimitInterval = "60s";
90 };
91 };
92
93 # Add recovery helper script
94 environment.systemPackages = [
95 (pkgs.writeScriptBin "consul-recover" ''
96 #!${pkgs.stdenv.shell}
97 set -e
98
99 echo "Consul Recovery Tool"
100 echo "==================="
101 echo ""
102
103 case "''${1:-}" in
104 force-leave)
105 if [ -z "''$2" ]; then
106 echo "Usage: consul-recover force-leave <node-name>"
107 exit 1
108 fi
109 echo "Force removing node ''$2 from cluster..."
110 consul force-leave "''$2"
111 ;;
112
113 reset-cluster)
114 echo "WARNING: This will completely reset the local consul cluster data!"
115 echo "Only use this if the cluster is completely broken."
116 read -p "Are you sure? (yes/no): " confirm
117 if [ "''$confirm" = "yes" ]; then
118 echo "Stopping consul..."
119 systemctl stop consul
120 echo "Clearing consul data..."
121 rm -rf /var/lib/consul/*
122 echo "Starting consul..."
123 systemctl start consul
124 echo "Reset complete. This node will rejoin or form a new cluster."
125 else
126 echo "Cancelled."
127 fi
128 ;;
129
130 bootstrap-single)
131 echo "WARNING: This will force this node to bootstrap as a single-node cluster!"
132 echo "Only use this in emergency recovery scenarios."
133 read -p "Are you sure? (yes/no): " confirm
134 if [ "''$confirm" = "yes" ]; then
135 echo "Stopping consul..."
136 systemctl stop consul
137 echo "Clearing consul data..."
138 rm -rf /var/lib/consul/*
139 echo "Creating temporary bootstrap configuration..."
140 cat /etc/consul.json | ${pkgs.jq}/bin/jq '.bootstrap_expect = 1' > /tmp/consul-bootstrap.json
141 echo "Starting consul in bootstrap mode..."
142 consul agent -server -data-dir=/var/lib/consul -config-file=/tmp/consul-bootstrap.json -config-file=/etc/consul-addrs.json -config-dir=/etc/consul.d &
143 CONSUL_PID=$!
144 sleep 10
145 echo "Checking cluster status..."
146 if consul operator raft list-peers; then
147 echo "Bootstrap successful! Restarting with normal configuration..."
148 kill $CONSUL_PID
149 systemctl start consul
150 else
151 echo "Bootstrap failed!"
152 kill $CONSUL_PID 2>/dev/null || true
153 exit 1
154 fi
155 else
156 echo "Cancelled."
157 fi
158 ;;
159
160 status)
161 echo "Consul Cluster Status:"
162 echo "====================="
163 echo ""
164 echo "Members:"
165 consul members || echo "Failed to get members (no leader?)"
166 echo ""
167 echo "Raft Peers:"
168 consul operator raft list-peers || echo "Failed to get raft peers (no leader?)"
169 echo ""
170 echo "Service Status:"
171 systemctl status consul --no-pager | head -20
172 ;;
173
174 *)
175 echo "Usage: consul-recover <command>"
176 echo ""
177 echo "Commands:"
178 echo " status - Show cluster status"
179 echo " force-leave <node> - Force remove a node from the cluster"
180 echo " reset-cluster - Completely reset local consul data"
181 echo " bootstrap-single - Emergency: Bootstrap as single-node cluster"
182 ;;
183 esac
184 '')
185 ];
186
187 environment.etc."consul.d/testservice.json".text = builtins.toJSON {
188 service = {
189 name = "testing";
190 };
191 };
192
193 krebs.iptables.tables.filter.INPUT.rules = [
194 {
195 predicate = "-i retiolum -p tcp --dport 8300";
196 target = "ACCEPT";
197 }
198 {
199 predicate = "-i retiolum -p tcp --dport 8301";
200 target = "ACCEPT";
201 }
202 {
203 predicate = "-i retiolum -p udp --dport 8301";
204 target = "ACCEPT";
205 }
206 {
207 predicate = "-i retiolum -p tcp --dport 8302";
208 target = "ACCEPT";
209 }
210 {
211 predicate = "-i retiolum -p udp --dport 8302";
212 target = "ACCEPT";
213 }
214 {
215 predicate = "-i retiolum -p tcp --dport 8400";
216 target = "ACCEPT";
217 }
218 {
219 predicate = "-i retiolum -p tcp --dport 8500";
220 target = "ACCEPT";
221 }
222 {
223 predicate = "-i retiolum -p tcp --dport 8600";
224 target = "ACCEPT";
225 }
226 {
227 predicate = "-i retiolum -p udp --dport 8500";
228 target = "ACCEPT";
229 }
230 ];
231}