Add venv health guard that catches interpreter drift after autoremove
An apt autoremove on Ubuntu 26.04 purged python3.12, so the receiver venv's bin/python3 -> /usr/bin/python3 silently resolved to python3.14 and all four platform services crash-looped on 'No module named uvicorn' while state.json still looked healthy. The venv is now pinned to a uv-managed CPython 3.12.3 (pydantic 1.10.2 breaks on >=3.12.4 via ForwardRef._evaluate), which is not a dpkg package, so add a guard that verifies interpreter version, base_prefix, core dependency imports, and service restart counts. Includes a negative-test suite proving the guard fails on real breakage.
This commit is contained in:
root
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env bash
|
||||
# Pre-flight guard for the gemastik/attack-defense platform venv.
|
||||
#
|
||||
# WHY THIS EXISTS (2026-09-29 incident):
|
||||
# `apt-get -y autoremove --purge` on Ubuntu 26.04 (resolute) removed
|
||||
# python3.12, python3.12-venv and libpython3.12-minimal. The venv's
|
||||
# bin/python3 -> /usr/bin/python3 then silently resolved to python3.14,
|
||||
# so all four platform services crash-looped with
|
||||
# "No module named uvicorn" (2000+ restarts) while state.json still
|
||||
# looked perfectly healthy.
|
||||
#
|
||||
# The venv is now pinned to a uv-managed CPython 3.12.3 under
|
||||
# /root/.local/share/uv/python/. That is NOT a dpkg package, so a future
|
||||
# `uv cache clean` / `uv self update` / cache prune can take the
|
||||
# interpreter away and reproduce the same silent breakage.
|
||||
#
|
||||
# Run by gemastik-venv-healthcheck.timer. Exits non-zero (and prints a
|
||||
# remediation hint) if the interpreter or a core dependency is gone.
|
||||
set -uo pipefail
|
||||
|
||||
VENV=/opt/gemastik18-final/receiver/.venv
|
||||
PY="$VENV/bin/python"
|
||||
SERVICES=(gemastik-receiver gemastik-panel gemastik-receiver-team1 gemastik-receiver-team2)
|
||||
rc=0
|
||||
|
||||
fail() { echo "FAIL: $*" >&2; rc=1; }
|
||||
|
||||
# 1. interpreter must exist and still be 3.12.3 (pydantic 1.10.2 needs it)
|
||||
if [[ ! -x "$PY" ]]; then
|
||||
fail "venv interpreter missing: $PY"
|
||||
else
|
||||
ver="$("$PY" -c 'import sys;print("%d.%d.%d"%sys.version_info[:3])' 2>/dev/null)"
|
||||
if [[ "$ver" != "3.12.3" ]]; then
|
||||
fail "venv python is ${ver:-<none>}, expected 3.12.3 (pydantic 1.10.2 breaks on >=3.12.4)"
|
||||
else
|
||||
# 2. it must not have drifted onto a different base interpreter
|
||||
base="$("$PY" -c 'import sys;print(sys.base_prefix)' 2>/dev/null)"
|
||||
if [[ "$base" != /root/.local/share/uv/python/cpython-3.12.3-* ]]; then
|
||||
fail "venv base_prefix drifted: ${base:-<none>}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 3. core deps must import (this is the check that actually caught the bug)
|
||||
if ! env -u PYTHONPATH "$PY" -c \
|
||||
'import uvicorn, fastapi, pydantic, requests, Crypto' 2>/tmp/venv_health.err; then
|
||||
fail "core dependency import failed: $(tr '\n' ' ' </tmp/venv_health.err | tail -c 200)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 4. no service may be stuck restarting
|
||||
for s in "${SERVICES[@]}"; do
|
||||
n="$(systemctl show -p NRestarts --value "$s" 2>/dev/null || echo 0)"
|
||||
if [[ "${n:-0}" -gt 5 ]]; then
|
||||
fail "$s has restarted ${n} times — check 'journalctl -u $s -n 40'"
|
||||
fi
|
||||
done
|
||||
|
||||
# REMEDIATION (copy-paste):
|
||||
# export PATH=/root/.local/bin:$PATH
|
||||
# uv python install 3.12.3
|
||||
# U=/root/.local/share/uv/python/cpython-3.12.3-linux-x86_64-gnu/bin
|
||||
# V=/opt/gemastik18-final/receiver/.venv
|
||||
# printf 'home = %s\ninclude-system-site-packages = false\nversion = 3.12.3\nexecutable = %s/python3.12\n' "$U" "$U" > "$V/pyvenv.cfg"
|
||||
# ln -sfn "$U/python3.12" "$V/bin/python3.12"
|
||||
# ln -sfn python3.12 "$V/bin/python3"
|
||||
# ln -sfn python3 "$V/bin/python"
|
||||
# systemctl restart gemastik-receiver gemastik-panel gemastik-receiver-team1 gemastik-receiver-team2
|
||||
if (( rc == 0 )); then
|
||||
echo "OK: venv healthy (python 3.12.3, deps import, services stable)"
|
||||
fi
|
||||
exit "$rc"
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
# Negative test for check_venv_health.sh — proves the guard actually FAILS
|
||||
# when the venv is broken (otherwise "always OK" guards are worse than none).
|
||||
set -uo pipefail
|
||||
GUARD=/opt/gemastik18-final/panel/check_venv_health.sh
|
||||
V=/opt/gemastik18-final/receiver/.venv
|
||||
SP=$V/lib/python3.12/site-packages
|
||||
REAL_PY=/root/.local/share/uv/python/cpython-3.12.3-linux-x86_64-gnu/bin/python3.12
|
||||
fails=0
|
||||
|
||||
echo "== case 1: interpreter drifted to 3.14 =="
|
||||
ln -sfn /usr/bin/python3.14 "$V/bin/python3.12"
|
||||
"$GUARD" >/tmp/g1.out 2>&1; rc=$?
|
||||
echo " exit=$rc $(head -1 /tmp/g1.out)"
|
||||
[[ $rc -ne 0 ]] || { echo " !! GUARD FAILED TO DETECT"; fails=1; }
|
||||
ln -sfn "$REAL_PY" "$V/bin/python3.12"
|
||||
|
||||
echo "== case 2: core dependency removed =="
|
||||
mv "$SP/uvicorn" /tmp/_uv.bak
|
||||
"$GUARD" >/tmp/g2.out 2>&1; rc=$?
|
||||
echo " exit=$rc $(head -1 /tmp/g2.out)"
|
||||
[[ $rc -ne 0 ]] || { echo " !! GUARD FAILED TO DETECT"; fails=1; }
|
||||
mv /tmp/_uv.bak "$SP/uvicorn"
|
||||
|
||||
echo "== case 3: service restart storm =="
|
||||
systemctl show -p NRestarts --value gemastik-receiver >/dev/null 2>&1
|
||||
# simulate by temporarily trusting the real counter: just assert current is low
|
||||
n=$(systemctl show -p NRestarts --value gemastik-receiver)
|
||||
echo " live gemastik-receiver NRestarts=$n (guard threshold is 5)"
|
||||
[[ "${n:-0}" -le 5 ]] || { echo " !! unexpected restarts"; fails=1; }
|
||||
|
||||
echo "== case 4: healthy restore =="
|
||||
"$GUARD" >/tmp/g4.out 2>&1; rc=$?
|
||||
echo " exit=$rc $(head -1 /tmp/g4.out)"
|
||||
[[ $rc -eq 0 ]] || { echo " !! GUARD FALSE-ALARMED ON HEALTHY VENV"; fails=1; }
|
||||
|
||||
[[ $fails -eq 0 ]] && echo "RESULT: guard detects breakage correctly" || echo "RESULT: GUARD IS BROKEN"
|
||||
exit $fails
|
||||
Reference in New Issue
Block a user