This repository has no description
6.3 kB
206 lines
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7{
8 services.consul = {
9 enable = true;
10 # dropPrivileges = false;
11 webUi = true;
12 # interface.bind = "retiolum";
13 extraConfig = {
14 bind_addr = config.krebs.build.host.nets.retiolum.ip4.addr;
15 bootstrap_expect = 1;
16 server = true;
17 # retry_join = config.services.consul.extraConfig.start_join;
18 retry_join = lib.mapAttrsToList (_n: h: lib.head h.nets.retiolum.aliases) (
19 lib.filterAttrs (_n: h: h.consul) config.krebs.hosts
20 );
21 rejoin_after_leave = true;
22
23 # try to fix random lock loss on leader reelection
24 retry_interval = "3s";
25
26 # Autopilot configuration for better automatic failure handling
27 autopilot = {
28 # Automatically clean up dead servers
29 cleanup_dead_servers = true;
30 # How long a server must be stable before promoting
31 server_stabilization_time = "10s";
32 # Only requires 3 servers minimum instead of all configured servers
33 min_quorum = 3;
34 };
35
36 # Performance tuning for faster leader elections
37 performance = {
38 # Reduce raft timing for faster recovery
39 raft_multiplier = 1;
40 };
41
42 # Leave on terminate for cleaner shutdowns
43 leave_on_terminate = true;
44 };
45 };
46
47 # Add systemd override for easier recovery
48 systemd.services.consul = {
49 # Add pre-start script to clean up potential issues
50 preStart = ''
51 # Remove any stale peers.json from failed recovery attempts
52 rm -f /var/lib/consul/raft/peers.json
53
54 # Ensure proper permissions
55 chown -R consul:consul /var/lib/consul
56 '';
57
58 # Add recovery commands to service
59 serviceConfig = {
60 # Allow consul to restart on failure
61 Restart = "on-failure";
62 RestartSec = "5s";
63 StartLimitBurst = 3;
64 StartLimitInterval = "60s";
65 };
66 };
67
68 # Add recovery helper script
69 environment.systemPackages = [
70 (pkgs.writeScriptBin "consul-recover" ''
71 #!${pkgs.stdenv.shell}
72 set -e
73
74 echo "Consul Recovery Tool"
75 echo "==================="
76 echo ""
77
78 case "''${1:-}" in
79 force-leave)
80 if [ -z "''$2" ]; then
81 echo "Usage: consul-recover force-leave <node-name>"
82 exit 1
83 fi
84 echo "Force removing node ''$2 from cluster..."
85 consul force-leave "''$2"
86 ;;
87
88 reset-cluster)
89 echo "WARNING: This will completely reset the local consul cluster data!"
90 echo "Only use this if the cluster is completely broken."
91 read -p "Are you sure? (yes/no): " confirm
92 if [ "''$confirm" = "yes" ]; then
93 echo "Stopping consul..."
94 systemctl stop consul
95 echo "Clearing consul data..."
96 rm -rf /var/lib/consul/*
97 echo "Starting consul..."
98 systemctl start consul
99 echo "Reset complete. This node will rejoin or form a new cluster."
100 else
101 echo "Cancelled."
102 fi
103 ;;
104
105 bootstrap-single)
106 echo "WARNING: This will force this node to bootstrap as a single-node cluster!"
107 echo "Only use this in emergency recovery scenarios."
108 read -p "Are you sure? (yes/no): " confirm
109 if [ "''$confirm" = "yes" ]; then
110 echo "Stopping consul..."
111 systemctl stop consul
112 echo "Clearing consul data..."
113 rm -rf /var/lib/consul/*
114 echo "Creating temporary bootstrap configuration..."
115 cat /etc/consul.json | ${pkgs.jq}/bin/jq '.bootstrap_expect = 1' > /tmp/consul-bootstrap.json
116 echo "Starting consul in bootstrap mode..."
117 consul agent -server -data-dir=/var/lib/consul -config-file=/tmp/consul-bootstrap.json -config-file=/etc/consul-addrs.json -config-dir=/etc/consul.d &
118 CONSUL_PID=$!
119 sleep 10
120 echo "Checking cluster status..."
121 if consul operator raft list-peers; then
122 echo "Bootstrap successful! Restarting with normal configuration..."
123 kill $CONSUL_PID
124 systemctl start consul
125 else
126 echo "Bootstrap failed!"
127 kill $CONSUL_PID 2>/dev/null || true
128 exit 1
129 fi
130 else
131 echo "Cancelled."
132 fi
133 ;;
134
135 status)
136 echo "Consul Cluster Status:"
137 echo "====================="
138 echo ""
139 echo "Members:"
140 consul members || echo "Failed to get members (no leader?)"
141 echo ""
142 echo "Raft Peers:"
143 consul operator raft list-peers || echo "Failed to get raft peers (no leader?)"
144 echo ""
145 echo "Service Status:"
146 systemctl status consul --no-pager | head -20
147 ;;
148
149 *)
150 echo "Usage: consul-recover <command>"
151 echo ""
152 echo "Commands:"
153 echo " status - Show cluster status"
154 echo " force-leave <node> - Force remove a node from the cluster"
155 echo " reset-cluster - Completely reset local consul data"
156 echo " bootstrap-single - Emergency: Bootstrap as single-node cluster"
157 ;;
158 esac
159 '')
160 ];
161
162 environment.etc."consul.d/testservice.json".text = builtins.toJSON {
163 service = {
164 name = "testing";
165 };
166 };
167
168 krebs.iptables.tables.filter.INPUT.rules = [
169 {
170 predicate = "-i retiolum -p tcp --dport 8300";
171 target = "ACCEPT";
172 }
173 {
174 predicate = "-i retiolum -p tcp --dport 8301";
175 target = "ACCEPT";
176 }
177 {
178 predicate = "-i retiolum -p udp --dport 8301";
179 target = "ACCEPT";
180 }
181 {
182 predicate = "-i retiolum -p tcp --dport 8302";
183 target = "ACCEPT";
184 }
185 {
186 predicate = "-i retiolum -p udp --dport 8302";
187 target = "ACCEPT";
188 }
189 {
190 predicate = "-i retiolum -p tcp --dport 8400";
191 target = "ACCEPT";
192 }
193 {
194 predicate = "-i retiolum -p tcp --dport 8500";
195 target = "ACCEPT";
196 }
197 {
198 predicate = "-i retiolum -p tcp --dport 8600";
199 target = "ACCEPT";
200 }
201 {
202 predicate = "-i retiolum -p udp --dport 8500";
203 target = "ACCEPT";
204 }
205 ];
206}