This repository has no description
configs.consul: add recover helper
author
lassulus
date
(Jul 9, 2025, 8:56 PM CEST)
commit
1b8bbb72
1b8bbb721dc7032154fc3ddcfe8d052099330237
parent
8255b0b0
8255b0b03e5350e8ace41432c0fa168277e95097
+143
-1
+143
-1
2configs/consul.nix
+143
-1
2configs/consul.nix
···
1
-
{ config, lib, ... }:
1
+
{
2
+
config,
3
+
lib,
4
+
pkgs,
5
+
...
6
+
}:
2
7
{
3
8
services.consul = {
4
9
enable = true;
···
17
22
18
23
# try to fix random lock loss on leader reelection
19
24
retry_interval = "3s";
25
+
26
+
# Autopilot configuration for better automatic failure handling
27
+
autopilot = {
28
+
# Automatically clean up dead servers
29
+
cleanup_dead_servers = true;
30
+
# How long a server must be stable before promoting
31
+
server_stabilization_time = "10s";
32
+
# Only requires 3 servers minimum instead of all configured servers
33
+
min_quorum = 3;
34
+
};
35
+
36
+
# Performance tuning for faster leader elections
37
+
performance = {
38
+
# Reduce raft timing for faster recovery
39
+
raft_multiplier = 1;
40
+
};
41
+
42
+
# Leave on terminate for cleaner shutdowns
43
+
leave_on_terminate = true;
44
+
45
+
# Disable leave intent to prevent stuck "leaving" state
46
+
disable_leave_intent = true;
20
47
};
21
48
};
49
+
50
+
# Add systemd override for easier recovery
51
+
systemd.services.consul = {
52
+
# Add pre-start script to clean up potential issues
53
+
preStart = ''
54
+
# Remove any stale peers.json from failed recovery attempts
55
+
rm -f /var/lib/consul/raft/peers.json
56
+
57
+
# Ensure proper permissions
58
+
chown -R consul:consul /var/lib/consul
59
+
'';
60
+
61
+
# Add recovery commands to service
62
+
serviceConfig = {
63
+
# Allow consul to restart on failure
64
+
Restart = "on-failure";
65
+
RestartSec = "5s";
66
+
StartLimitBurst = 3;
67
+
StartLimitInterval = "60s";
68
+
};
69
+
};
70
+
71
+
# Add recovery helper script
72
+
environment.systemPackages = [
73
+
(pkgs.writeScriptBin "consul-recover" ''
74
+
#!${pkgs.stdenv.shell}
75
+
set -e
76
+
77
+
echo "Consul Recovery Tool"
78
+
echo "==================="
79
+
echo ""
80
+
81
+
case "''${1:-}" in
82
+
force-leave)
83
+
if [ -z "''$2" ]; then
84
+
echo "Usage: consul-recover force-leave <node-name>"
85
+
exit 1
86
+
fi
87
+
echo "Force removing node ''$2 from cluster..."
88
+
consul force-leave "''$2"
89
+
;;
90
+
91
+
reset-cluster)
92
+
echo "WARNING: This will completely reset the local consul cluster data!"
93
+
echo "Only use this if the cluster is completely broken."
94
+
read -p "Are you sure? (yes/no): " confirm
95
+
if [ "''$confirm" = "yes" ]; then
96
+
echo "Stopping consul..."
97
+
systemctl stop consul
98
+
echo "Clearing consul data..."
99
+
rm -rf /var/lib/consul/*
100
+
echo "Starting consul..."
101
+
systemctl start consul
102
+
echo "Reset complete. This node will rejoin or form a new cluster."
103
+
else
104
+
echo "Cancelled."
105
+
fi
106
+
;;
107
+
108
+
bootstrap-single)
109
+
echo "WARNING: This will force this node to bootstrap as a single-node cluster!"
110
+
echo "Only use this in emergency recovery scenarios."
111
+
read -p "Are you sure? (yes/no): " confirm
112
+
if [ "''$confirm" = "yes" ]; then
113
+
echo "Stopping consul..."
114
+
systemctl stop consul
115
+
echo "Clearing consul data..."
116
+
rm -rf /var/lib/consul/*
117
+
echo "Creating temporary bootstrap configuration..."
118
+
cat /etc/consul.json | ${pkgs.jq}/bin/jq '.bootstrap_expect = 1' > /tmp/consul-bootstrap.json
119
+
echo "Starting consul in bootstrap mode..."
120
+
consul agent -server -data-dir=/var/lib/consul -config-file=/tmp/consul-bootstrap.json -config-file=/etc/consul-addrs.json -config-dir=/etc/consul.d &
121
+
CONSUL_PID=$!
122
+
sleep 10
123
+
echo "Checking cluster status..."
124
+
if consul operator raft list-peers; then
125
+
echo "Bootstrap successful! Restarting with normal configuration..."
126
+
kill $CONSUL_PID
127
+
systemctl start consul
128
+
else
129
+
echo "Bootstrap failed!"
130
+
kill $CONSUL_PID 2>/dev/null || true
131
+
exit 1
132
+
fi
133
+
else
134
+
echo "Cancelled."
135
+
fi
136
+
;;
137
+
138
+
status)
139
+
echo "Consul Cluster Status:"
140
+
echo "====================="
141
+
echo ""
142
+
echo "Members:"
143
+
consul members || echo "Failed to get members (no leader?)"
144
+
echo ""
145
+
echo "Raft Peers:"
146
+
consul operator raft list-peers || echo "Failed to get raft peers (no leader?)"
147
+
echo ""
148
+
echo "Service Status:"
149
+
systemctl status consul --no-pager | head -20
150
+
;;
151
+
152
+
*)
153
+
echo "Usage: consul-recover <command>"
154
+
echo ""
155
+
echo "Commands:"
156
+
echo " status - Show cluster status"
157
+
echo " force-leave <node> - Force remove a node from the cluster"
158
+
echo " reset-cluster - Completely reset local consul data"
159
+
echo " bootstrap-single - Emergency: Bootstrap as single-node cluster"
160
+
;;
161
+
esac
162
+
'')
163
+
];
22
164
23
165
environment.etc."consul.d/testservice.json".text = builtins.toJSON {
24
166
service = {