root
c3123fd0b2
An apt autoremove on Ubuntu 26.04 purged python3.12, so the receiver venv's bin/python3 -> /usr/bin/python3 silently resolved to python3.14 and all four platform services crash-looped on 'No module named uvicorn' while state.json still looked healthy. The venv is now pinned to a uv-managed CPython 3.12.3 (pydantic 1.10.2 breaks on >=3.12.4 via ForwardRef._evaluate), which is not a dpkg package, so add a guard that verifies interpreter version, base_prefix, core dependency imports, and service restart counts. Includes a negative-test suite proving the guard fails on real breakage.
72 lines
3.0 KiB
Bash
Executable File
72 lines
3.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Pre-flight guard for the gemastik/attack-defense platform venv.
|
|
#
|
|
# WHY THIS EXISTS (2026-09-29 incident):
|
|
# `apt-get -y autoremove --purge` on Ubuntu 26.04 (resolute) removed
|
|
# python3.12, python3.12-venv and libpython3.12-minimal. The venv's
|
|
# bin/python3 -> /usr/bin/python3 then silently resolved to python3.14,
|
|
# so all four platform services crash-looped with
|
|
# "No module named uvicorn" (2000+ restarts) while state.json still
|
|
# looked perfectly healthy.
|
|
#
|
|
# The venv is now pinned to a uv-managed CPython 3.12.3 under
|
|
# /root/.local/share/uv/python/. That is NOT a dpkg package, so a future
|
|
# `uv cache clean` / `uv self update` / cache prune can take the
|
|
# interpreter away and reproduce the same silent breakage.
|
|
#
|
|
# Run by gemastik-venv-healthcheck.timer. Exits non-zero (and prints a
|
|
# remediation hint) if the interpreter or a core dependency is gone.
|
|
set -uo pipefail
|
|
|
|
VENV=/opt/gemastik18-final/receiver/.venv
|
|
PY="$VENV/bin/python"
|
|
SERVICES=(gemastik-receiver gemastik-panel gemastik-receiver-team1 gemastik-receiver-team2)
|
|
rc=0
|
|
|
|
fail() { echo "FAIL: $*" >&2; rc=1; }
|
|
|
|
# 1. interpreter must exist and still be 3.12.3 (pydantic 1.10.2 needs it)
|
|
if [[ ! -x "$PY" ]]; then
|
|
fail "venv interpreter missing: $PY"
|
|
else
|
|
ver="$("$PY" -c 'import sys;print("%d.%d.%d"%sys.version_info[:3])' 2>/dev/null)"
|
|
if [[ "$ver" != "3.12.3" ]]; then
|
|
fail "venv python is ${ver:-<none>}, expected 3.12.3 (pydantic 1.10.2 breaks on >=3.12.4)"
|
|
else
|
|
# 2. it must not have drifted onto a different base interpreter
|
|
base="$("$PY" -c 'import sys;print(sys.base_prefix)' 2>/dev/null)"
|
|
if [[ "$base" != /root/.local/share/uv/python/cpython-3.12.3-* ]]; then
|
|
fail "venv base_prefix drifted: ${base:-<none>}"
|
|
fi
|
|
fi
|
|
|
|
# 3. core deps must import (this is the check that actually caught the bug)
|
|
if ! env -u PYTHONPATH "$PY" -c \
|
|
'import uvicorn, fastapi, pydantic, requests, Crypto' 2>/tmp/venv_health.err; then
|
|
fail "core dependency import failed: $(tr '\n' ' ' </tmp/venv_health.err | tail -c 200)"
|
|
fi
|
|
fi
|
|
|
|
# 4. no service may be stuck restarting
|
|
for s in "${SERVICES[@]}"; do
|
|
n="$(systemctl show -p NRestarts --value "$s" 2>/dev/null || echo 0)"
|
|
if [[ "${n:-0}" -gt 5 ]]; then
|
|
fail "$s has restarted ${n} times — check 'journalctl -u $s -n 40'"
|
|
fi
|
|
done
|
|
|
|
# REMEDIATION (copy-paste):
|
|
# export PATH=/root/.local/bin:$PATH
|
|
# uv python install 3.12.3
|
|
# U=/root/.local/share/uv/python/cpython-3.12.3-linux-x86_64-gnu/bin
|
|
# V=/opt/gemastik18-final/receiver/.venv
|
|
# printf 'home = %s\ninclude-system-site-packages = false\nversion = 3.12.3\nexecutable = %s/python3.12\n' "$U" "$U" > "$V/pyvenv.cfg"
|
|
# ln -sfn "$U/python3.12" "$V/bin/python3.12"
|
|
# ln -sfn python3.12 "$V/bin/python3"
|
|
# ln -sfn python3 "$V/bin/python"
|
|
# systemctl restart gemastik-receiver gemastik-panel gemastik-receiver-team1 gemastik-receiver-team2
|
|
if (( rc == 0 )); then
|
|
echo "OK: venv healthy (python 3.12.3, deps import, services stable)"
|
|
fi
|
|
exit "$rc"
|