This repository has no description
6.4 kB
207 lines
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7{
8 services.consul = {
9 enable = true;
10 # dropPrivileges = false;
11 webUi = true;
12 # interface.bind = "retiolum";
13 extraConfig = {
14 bind_addr = config.krebs.build.host.nets.retiolum.ip4.addr;
15 bootstrap_expect = 3;
16 server = true;
17 # retry_join = config.services.consul.extraConfig.start_join;
18 retry_join = lib.mapAttrsToList (_n: h: lib.head h.nets.retiolum.aliases) (
19 lib.filterAttrs (_n: h: h.consul) config.krebs.hosts
20 );
21 rejoin_after_leave = true;
22
23 # try to fix random lock loss on leader reelection
24 retry_interval = "3s";
25
26 # Autopilot configuration for better automatic failure handling
27 autopilot = {
28 # Automatically clean up dead servers
29 cleanup_dead_servers = true;
30 # How long a server must be stable before promoting
31 server_stabilization_time = "10s";
32 # Allow cluster to operate with 2 of 3 nodes (majority quorum)
33 min_quorum = 2;
34 };
35
36 # Performance tuning - slightly relaxed for network tolerance
37 performance = {
38 # 2 = default, gives more time for slow/unstable networks
39 # 1 was too aggressive and caused leader flapping
40 raft_multiplier = 2;
41 };
42
43 # Reconnect timeout for failed nodes (default 72h, reduce for faster cleanup)
44 reconnect_timeout = "24h";
45
46 # Leave on terminate for cleaner shutdowns
47 leave_on_terminate = true;
48 };
49 };
50
51 # Add systemd override for easier recovery
52 systemd.services.consul = {
53 # Add pre-start script to clean up potential issues
54 preStart = ''
55 # Ensure proper permissions
56 chown -R consul:consul /var/lib/consul
57 '';
58
59 # Add recovery commands to service
60 serviceConfig = {
61 # Allow consul to restart on failure
62 Restart = "on-failure";
63 RestartSec = "5s";
64 StartLimitBurst = 3;
65 StartLimitInterval = "60s";
66 };
67 };
68
69 # Add recovery helper script
70 environment.systemPackages = [
71 (pkgs.writeScriptBin "consul-recover" ''
72 #!${pkgs.stdenv.shell}
73 set -e
74
75 echo "Consul Recovery Tool"
76 echo "==================="
77 echo ""
78
79 case "''${1:-}" in
80 force-leave)
81 if [ -z "''$2" ]; then
82 echo "Usage: consul-recover force-leave <node-name>"
83 exit 1
84 fi
85 echo "Force removing node ''$2 from cluster..."
86 consul force-leave "''$2"
87 ;;
88
89 reset-cluster)
90 echo "WARNING: This will completely reset the local consul cluster data!"
91 echo "Only use this if the cluster is completely broken."
92 read -p "Are you sure? (yes/no): " confirm
93 if [ "''$confirm" = "yes" ]; then
94 echo "Stopping consul..."
95 systemctl stop consul
96 echo "Clearing consul data..."
97 rm -rf /var/lib/consul/*
98 echo "Starting consul..."
99 systemctl start consul
100 echo "Reset complete. This node will rejoin or form a new cluster."
101 else
102 echo "Cancelled."
103 fi
104 ;;
105
106 bootstrap-single)
107 echo "WARNING: This will force this node to bootstrap as a single-node cluster!"
108 echo "Only use this in emergency recovery scenarios."
109 read -p "Are you sure? (yes/no): " confirm
110 if [ "''$confirm" = "yes" ]; then
111 echo "Stopping consul..."
112 systemctl stop consul
113 echo "Clearing consul data..."
114 rm -rf /var/lib/consul/*
115 echo "Creating temporary bootstrap configuration..."
116 cat /etc/consul.json | ${pkgs.jq}/bin/jq '.bootstrap_expect = 1' > /tmp/consul-bootstrap.json
117 echo "Starting consul in bootstrap mode..."
118 consul agent -server -data-dir=/var/lib/consul -config-file=/tmp/consul-bootstrap.json -config-file=/etc/consul-addrs.json -config-dir=/etc/consul.d &
119 CONSUL_PID=$!
120 sleep 10
121 echo "Checking cluster status..."
122 if consul operator raft list-peers; then
123 echo "Bootstrap successful! Restarting with normal configuration..."
124 kill $CONSUL_PID
125 systemctl start consul
126 else
127 echo "Bootstrap failed!"
128 kill $CONSUL_PID 2>/dev/null || true
129 exit 1
130 fi
131 else
132 echo "Cancelled."
133 fi
134 ;;
135
136 status)
137 echo "Consul Cluster Status:"
138 echo "====================="
139 echo ""
140 echo "Members:"
141 consul members || echo "Failed to get members (no leader?)"
142 echo ""
143 echo "Raft Peers:"
144 consul operator raft list-peers || echo "Failed to get raft peers (no leader?)"
145 echo ""
146 echo "Service Status:"
147 systemctl status consul --no-pager | head -20
148 ;;
149
150 *)
151 echo "Usage: consul-recover <command>"
152 echo ""
153 echo "Commands:"
154 echo " status - Show cluster status"
155 echo " force-leave <node> - Force remove a node from the cluster"
156 echo " reset-cluster - Completely reset local consul data"
157 echo " bootstrap-single - Emergency: Bootstrap as single-node cluster"
158 ;;
159 esac
160 '')
161 ];
162
163 environment.etc."consul.d/testservice.json".text = builtins.toJSON {
164 service = {
165 name = "testing";
166 };
167 };
168
169 krebs.iptables.tables.filter.INPUT.rules = [
170 {
171 predicate = "-i retiolum -p tcp --dport 8300";
172 target = "ACCEPT";
173 }
174 {
175 predicate = "-i retiolum -p tcp --dport 8301";
176 target = "ACCEPT";
177 }
178 {
179 predicate = "-i retiolum -p udp --dport 8301";
180 target = "ACCEPT";
181 }
182 {
183 predicate = "-i retiolum -p tcp --dport 8302";
184 target = "ACCEPT";
185 }
186 {
187 predicate = "-i retiolum -p udp --dport 8302";
188 target = "ACCEPT";
189 }
190 {
191 predicate = "-i retiolum -p tcp --dport 8400";
192 target = "ACCEPT";
193 }
194 {
195 predicate = "-i retiolum -p tcp --dport 8500";
196 target = "ACCEPT";
197 }
198 {
199 predicate = "-i retiolum -p tcp --dport 8600";
200 target = "ACCEPT";
201 }
202 {
203 predicate = "-i retiolum -p udp --dport 8500";
204 target = "ACCEPT";
205 }
206 ];
207}