This repository has no description
configs.consul: remove
author
Hermes
committer
lassulus
date
(Oct 14, 2026, 12:28 PM CEST)
commit
d3415c8c
d3415c8c80c01e582136aadf43133872b8a275a9
parent
19f499bc
19f499bc2d55663f63f4067e5cd93bf31050233e
-213
-208
configs/consul.nix
-208
configs/consul.nix
···
1
-
{
2
-
self,
3
-
config,
4
-
lib,
5
-
pkgs,
6
-
...
7
-
}:
8
-
let
9
-
inherit (self.inputs.kartei) hosts;
10
-
11
-
# consul server peers; stockholm policy (krebs/3modules/kartei/overlay.nix),
12
-
# not kartei data
13
-
consulHosts = [
14
-
"aergia"
15
-
"blue"
16
-
"coaxmetal"
17
-
"daedalus"
18
-
"green"
19
-
"icarus"
20
-
"ignavia"
21
-
"littleT"
22
-
"massulus"
23
-
"neoprism"
24
-
"orange"
25
-
"radio"
26
-
"shodan"
27
-
"skynet"
28
-
"styx"
29
-
"ubik"
30
-
"yellow"
31
-
];
32
-
in
33
-
{
34
-
services.consul = {
35
-
enable = true;
36
-
# dropPrivileges = false;
37
-
webUi = true;
38
-
# interface.bind = "retiolum";
39
-
extraConfig = {
40
-
bind_addr = hosts.${config.networking.hostName}.nets.retiolum.ip4.addr;
41
-
bootstrap_expect = 3;
42
-
server = true;
43
-
# retry_join = config.services.consul.extraConfig.start_join;
44
-
retry_join = map (name: lib.head hosts.${name}.nets.retiolum.aliases) consulHosts;
45
-
rejoin_after_leave = true;
46
-
47
-
# try to fix random lock loss on leader reelection
48
-
retry_interval = "3s";
49
-
50
-
# Autopilot configuration for better automatic failure handling
51
-
autopilot = {
52
-
# Automatically clean up dead servers
53
-
cleanup_dead_servers = true;
54
-
# How long a server must be stable before promoting
55
-
server_stabilization_time = "10s";
56
-
# Allow cluster to operate with 2 of 3 nodes (majority quorum)
57
-
min_quorum = 2;
58
-
};
59
-
60
-
# Performance tuning - slightly relaxed for network tolerance
61
-
performance = {
62
-
# 2 = default, gives more time for slow/unstable networks
63
-
# 1 was too aggressive and caused leader flapping
64
-
raft_multiplier = 2;
65
-
};
66
-
67
-
# Reconnect timeout for failed nodes (default 72h, reduce for faster cleanup)
68
-
reconnect_timeout = "24h";
69
-
70
-
# Leave on terminate for cleaner shutdowns
71
-
leave_on_terminate = true;
72
-
};
73
-
};
74
-
75
-
# Add systemd override for easier recovery
76
-
systemd.services.consul = {
77
-
# Add pre-start script to clean up potential issues
78
-
preStart = ''
79
-
# Ensure proper permissions
80
-
chown -R consul:consul /var/lib/consul
81
-
'';
82
-
83
-
# Add recovery commands to service
84
-
serviceConfig = {
85
-
# Allow consul to restart on failure
86
-
Restart = "on-failure";
87
-
RestartSec = "5s";
88
-
StartLimitBurst = 3;
89
-
StartLimitInterval = "60s";
90
-
};
91
-
};
92
-
93
-
# Add recovery helper script
94
-
environment.systemPackages = [
95
-
(pkgs.writeScriptBin "consul-recover" ''
96
-
#!${pkgs.stdenv.shell}
97
-
set -e
98
-
99
-
echo "Consul Recovery Tool"
100
-
echo "==================="
101
-
echo ""
102
-
103
-
case "''${1:-}" in
104
-
force-leave)
105
-
if [ -z "''$2" ]; then
106
-
echo "Usage: consul-recover force-leave <node-name>"
107
-
exit 1
108
-
fi
109
-
echo "Force removing node ''$2 from cluster..."
110
-
consul force-leave "''$2"
111
-
;;
112
-
113
-
reset-cluster)
114
-
echo "WARNING: This will completely reset the local consul cluster data!"
115
-
echo "Only use this if the cluster is completely broken."
116
-
read -p "Are you sure? (yes/no): " confirm
117
-
if [ "''$confirm" = "yes" ]; then
118
-
echo "Stopping consul..."
119
-
systemctl stop consul
120
-
echo "Clearing consul data..."
121
-
rm -rf /var/lib/consul/*
122
-
echo "Starting consul..."
123
-
systemctl start consul
124
-
echo "Reset complete. This node will rejoin or form a new cluster."
125
-
else
126
-
echo "Cancelled."
127
-
fi
128
-
;;
129
-
130
-
bootstrap-single)
131
-
echo "WARNING: This will force this node to bootstrap as a single-node cluster!"
132
-
echo "Only use this in emergency recovery scenarios."
133
-
read -p "Are you sure? (yes/no): " confirm
134
-
if [ "''$confirm" = "yes" ]; then
135
-
echo "Stopping consul..."
136
-
systemctl stop consul
137
-
echo "Clearing consul data..."
138
-
rm -rf /var/lib/consul/*
139
-
echo "Creating temporary bootstrap configuration..."
140
-
cat /etc/consul.json | ${pkgs.jq}/bin/jq '.bootstrap_expect = 1' > /tmp/consul-bootstrap.json
141
-
echo "Starting consul in bootstrap mode..."
142
-
consul agent -server -data-dir=/var/lib/consul -config-file=/tmp/consul-bootstrap.json -config-file=/etc/consul-addrs.json -config-dir=/etc/consul.d &
143
-
CONSUL_PID=$!
144
-
sleep 10
145
-
echo "Checking cluster status..."
146
-
if consul operator raft list-peers; then
147
-
echo "Bootstrap successful! Restarting with normal configuration..."
148
-
kill $CONSUL_PID
149
-
systemctl start consul
150
-
else
151
-
echo "Bootstrap failed!"
152
-
kill $CONSUL_PID 2>/dev/null || true
153
-
exit 1
154
-
fi
155
-
else
156
-
echo "Cancelled."
157
-
fi
158
-
;;
159
-
160
-
status)
161
-
echo "Consul Cluster Status:"
162
-
echo "====================="
163
-
echo ""
164
-
echo "Members:"
165
-
consul members || echo "Failed to get members (no leader?)"
166
-
echo ""
167
-
echo "Raft Peers:"
168
-
consul operator raft list-peers || echo "Failed to get raft peers (no leader?)"
169
-
echo ""
170
-
echo "Service Status:"
171
-
systemctl status consul --no-pager | head -20
172
-
;;
173
-
174
-
*)
175
-
echo "Usage: consul-recover <command>"
176
-
echo ""
177
-
echo "Commands:"
178
-
echo " status - Show cluster status"
179
-
echo " force-leave <node> - Force remove a node from the cluster"
180
-
echo " reset-cluster - Completely reset local consul data"
181
-
echo " bootstrap-single - Emergency: Bootstrap as single-node cluster"
182
-
;;
183
-
esac
184
-
'')
185
-
];
186
-
187
-
environment.etc."consul.d/testservice.json".text = builtins.toJSON {
188
-
service = {
189
-
name = "testing";
190
-
};
191
-
};
192
-
193
-
networking.firewall.interfaces.retiolum = {
194
-
allowedTCPPorts = [
195
-
8300
196
-
8301
197
-
8302
198
-
8400
199
-
8500
200
-
8600
201
-
];
202
-
allowedUDPPorts = [
203
-
8301
204
-
8302
205
-
8500
206
-
];
207
-
};
208
-
}
-1
machines/icarus/config.nix
-1
machines/icarus/config.nix
-1
machines/mors/config.nix
-1
machines/mors/config.nix
-1
machines/neoprism/config.nix
-1
machines/neoprism/config.nix
-1
machines/shodan/config.nix
-1
machines/shodan/config.nix