This repository has no description
6.4 kB
209 lines
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7{
8 services.consul = {
9 enable = true;
10 # dropPrivileges = false;
11 webUi = true;
12 # interface.bind = "retiolum";
13 extraConfig = {
14 bind_addr = config.krebs.build.host.nets.retiolum.ip4.addr;
15 bootstrap_expect = 3;
16 server = true;
17 # retry_join = config.services.consul.extraConfig.start_join;
18 retry_join = lib.mapAttrsToList (_n: h: lib.head h.nets.retiolum.aliases) (
19 lib.filterAttrs (_n: h: h.consul) config.krebs.hosts
20 );
21 rejoin_after_leave = true;
22
23 # try to fix random lock loss on leader reelection
24 retry_interval = "3s";
25
26 # Autopilot configuration for better automatic failure handling
27 autopilot = {
28 # Automatically clean up dead servers
29 cleanup_dead_servers = true;
30 # How long a server must be stable before promoting
31 server_stabilization_time = "10s";
32 # Only requires 3 servers minimum instead of all configured servers
33 min_quorum = 3;
34 };
35
36 # Performance tuning for faster leader elections
37 performance = {
38 # Reduce raft timing for faster recovery
39 raft_multiplier = 1;
40 };
41
42 # Leave on terminate for cleaner shutdowns
43 leave_on_terminate = true;
44
45 # Disable leave intent to prevent stuck "leaving" state
46 disable_leave_intent = true;
47 };
48 };
49
50 # Add systemd override for easier recovery
51 systemd.services.consul = {
52 # Add pre-start script to clean up potential issues
53 preStart = ''
54 # Remove any stale peers.json from failed recovery attempts
55 rm -f /var/lib/consul/raft/peers.json
56
57 # Ensure proper permissions
58 chown -R consul:consul /var/lib/consul
59 '';
60
61 # Add recovery commands to service
62 serviceConfig = {
63 # Allow consul to restart on failure
64 Restart = "on-failure";
65 RestartSec = "5s";
66 StartLimitBurst = 3;
67 StartLimitInterval = "60s";
68 };
69 };
70
71 # Add recovery helper script
72 environment.systemPackages = [
73 (pkgs.writeScriptBin "consul-recover" ''
74 #!${pkgs.stdenv.shell}
75 set -e
76
77 echo "Consul Recovery Tool"
78 echo "==================="
79 echo ""
80
81 case "''${1:-}" in
82 force-leave)
83 if [ -z "''$2" ]; then
84 echo "Usage: consul-recover force-leave <node-name>"
85 exit 1
86 fi
87 echo "Force removing node ''$2 from cluster..."
88 consul force-leave "''$2"
89 ;;
90
91 reset-cluster)
92 echo "WARNING: This will completely reset the local consul cluster data!"
93 echo "Only use this if the cluster is completely broken."
94 read -p "Are you sure? (yes/no): " confirm
95 if [ "''$confirm" = "yes" ]; then
96 echo "Stopping consul..."
97 systemctl stop consul
98 echo "Clearing consul data..."
99 rm -rf /var/lib/consul/*
100 echo "Starting consul..."
101 systemctl start consul
102 echo "Reset complete. This node will rejoin or form a new cluster."
103 else
104 echo "Cancelled."
105 fi
106 ;;
107
108 bootstrap-single)
109 echo "WARNING: This will force this node to bootstrap as a single-node cluster!"
110 echo "Only use this in emergency recovery scenarios."
111 read -p "Are you sure? (yes/no): " confirm
112 if [ "''$confirm" = "yes" ]; then
113 echo "Stopping consul..."
114 systemctl stop consul
115 echo "Clearing consul data..."
116 rm -rf /var/lib/consul/*
117 echo "Creating temporary bootstrap configuration..."
118 cat /etc/consul.json | ${pkgs.jq}/bin/jq '.bootstrap_expect = 1' > /tmp/consul-bootstrap.json
119 echo "Starting consul in bootstrap mode..."
120 consul agent -server -data-dir=/var/lib/consul -config-file=/tmp/consul-bootstrap.json -config-file=/etc/consul-addrs.json -config-dir=/etc/consul.d &
121 CONSUL_PID=$!
122 sleep 10
123 echo "Checking cluster status..."
124 if consul operator raft list-peers; then
125 echo "Bootstrap successful! Restarting with normal configuration..."
126 kill $CONSUL_PID
127 systemctl start consul
128 else
129 echo "Bootstrap failed!"
130 kill $CONSUL_PID 2>/dev/null || true
131 exit 1
132 fi
133 else
134 echo "Cancelled."
135 fi
136 ;;
137
138 status)
139 echo "Consul Cluster Status:"
140 echo "====================="
141 echo ""
142 echo "Members:"
143 consul members || echo "Failed to get members (no leader?)"
144 echo ""
145 echo "Raft Peers:"
146 consul operator raft list-peers || echo "Failed to get raft peers (no leader?)"
147 echo ""
148 echo "Service Status:"
149 systemctl status consul --no-pager | head -20
150 ;;
151
152 *)
153 echo "Usage: consul-recover <command>"
154 echo ""
155 echo "Commands:"
156 echo " status - Show cluster status"
157 echo " force-leave <node> - Force remove a node from the cluster"
158 echo " reset-cluster - Completely reset local consul data"
159 echo " bootstrap-single - Emergency: Bootstrap as single-node cluster"
160 ;;
161 esac
162 '')
163 ];
164
165 environment.etc."consul.d/testservice.json".text = builtins.toJSON {
166 service = {
167 name = "testing";
168 };
169 };
170
171 krebs.iptables.tables.filter.INPUT.rules = [
172 {
173 predicate = "-i retiolum -p tcp --dport 8300";
174 target = "ACCEPT";
175 }
176 {
177 predicate = "-i retiolum -p tcp --dport 8301";
178 target = "ACCEPT";
179 }
180 {
181 predicate = "-i retiolum -p udp --dport 8301";
182 target = "ACCEPT";
183 }
184 {
185 predicate = "-i retiolum -p tcp --dport 8302";
186 target = "ACCEPT";
187 }
188 {
189 predicate = "-i retiolum -p udp --dport 8302";
190 target = "ACCEPT";
191 }
192 {
193 predicate = "-i retiolum -p tcp --dport 8400";
194 target = "ACCEPT";
195 }
196 {
197 predicate = "-i retiolum -p tcp --dport 8500";
198 target = "ACCEPT";
199 }
200 {
201 predicate = "-i retiolum -p tcp --dport 8600";
202 target = "ACCEPT";
203 }
204 {
205 predicate = "-i retiolum -p udp --dport 8500";
206 target = "ACCEPT";
207 }
208 ];
209}