fix: per-team receivers as independent systemd services

Receivers were child Popen processes of gemastik-panel systemd cgroup;
restarting the panel killed all team receivers (SLA -> 0/6, 401 on
/api/team/N/status proxy). Now each team receiver is a systemd service
(gemastik-receiver-teamN.service) generated by gen_receiver_services.py with
per-team env (ports, containers, COMPOSE_LOCATION, .env). Verified: panel
restart no longer kills receivers; 18/18 SLA stays UP.

panel/gen_receiver_services.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Generate + (re)start per-team receiver systemd services.
+
+Each team receiver becomes gemastik-receiver-teamN.service, independent of
+the panel service. This fixes the bug where restarting gemastik-panel killed
+all team receivers (they were child processes of the panel systemd cgroup).
+"""
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+TEAMS_DIR = Path("/opt/gemastik18-final/teams")
+RECEIVER_VENV = "/opt/gemastik18-final/receiver/.venv/bin/python"
+UNIT_DIR = Path("/etc/systemd/system")
+
+def write_unit(idx: int, st: dict):
+    port = st["ports"]["receiver"]
+    recv_dir = TEAMS_DIR / f"team{idx}" / "receiver"
+    env = {}
+    # ports/containers for challenge classes
+    for ch in st["ports"]:
+        if ch in ("receiver", "panel"):
+            continue
+        env[f"CHALLENGE_PORT_{ch.upper()}"] = str(st["ports"][ch]["chall"])
+        env[f"CHALLENGE_CONTAINER_{ch.upper()}"] = f"{ch}_container_team{idx}"
+    env["COMPOSE_LOCATION"] = str(TEAMS_DIR / f"team{idx}" / "services" / "docker-compose.yml")
+    env_lines = "\n".join(f'Environment="{k}={v}"' for k, v in env.items())
+    unit = f"""[Unit]
+Description=Gemastik A/D receiver for team {idx}
+After=docker.service
+Wants=docker.service
+
+[Service]
+Type=simple
+WorkingDirectory={recv_dir}
+EnvironmentFile={recv_dir}/.env
+{env_lines}
+ExecStart={RECEIVER_VENV} -m uvicorn main:app --host 0.0.0.0 --port {port}
+Restart=always
+RestartSec=3
+
+[Install]
+WantedBy=multi-user.target
+"""
+    (UNIT_DIR / f"gemastik-receiver-team{idx}.service").write_text(unit)
+
+def main():
+    action = sys.argv[1] if len(sys.argv) > 1 else "start"
+    for d in sorted(TEAMS_DIR.glob("team*")):
+        sf = d / "state.json"
+        if not sf.exists():
+            continue
+        st = json.loads(sf.read_text())
+        idx = st["index"]
+        write_unit(idx, st)
+        subprocess.run(["systemctl", "daemon-reload"], check=False)
+        subprocess.run(["systemctl", "enable", f"gemastik-receiver-team{idx}.service"], check=False)
+        if action == "stop":
+            subprocess.run(["systemctl", "stop", f"gemastik-receiver-team{idx}.service"], check=False)
+        else:
+            subprocess.run(["systemctl", "restart", f"gemastik-receiver-team{idx}.service"], check=False)
+        print(f"gemastik-receiver-team{idx}: {action} (port {st['ports']['receiver']})")
+
+if __name__ == "__main__":
+    main()

panel/teams.py

@@ -25,6 +25,7 @@ import re
 import secrets
 import shutil
 import subprocess
+import sys
 import time
 import uuid
 from datetime import datetime
@@ -235,40 +236,32 @@ def stop_team(idx: int):
     return st
 
 def _start_receiver(idx: int):
-    """Launch team's receiver as a detached uvicorn process with team env."""
+    """Start team's receiver via systemd service (independent of panel cgroup)."""
     team_dir = TEAMS_DIR / f"team{idx}"
-    recv_dir = team_dir / "receiver"
     st = json.loads((team_dir / "state.json").read_text())
     port = st["ports"]["receiver"]
     pidfile = team_dir / "receiver.pid"
-    # kill existing
-    _stop_receiver(idx)
-    env = dict(os.environ)
-    env["PYTHONPATH"] = str(recv_dir)
-    env["COMPOSE_LOCATION"] = str(team_dir / "services" / "docker-compose.yml")
-    # expose team ports/containers so challenge classes bind the right targets
-    for ch in st["ports"]:
-        if ch in ("receiver", "panel"):
-            continue
-        env[f"CHALLENGE_PORT_{ch.upper()}"] = str(st["ports"][ch]["chall"])
-        env[f"CHALLENGE_CONTAINER_{ch.upper()}"] = f"{ch}_container_team{idx}"
-    proc = subprocess.Popen(
-        [str(RECEIVER_SRC.parent / "receiver" / ".venv" / "bin" / "python"),
-         "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", str(port)],
-        cwd=str(recv_dir), env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
-        start_new_session=True)
-    pidfile.write_text(str(proc.pid))
+    # ensure the per-team systemd unit exists with current env
+    subprocess.run([sys.executable, str(BASE / "panel" / "gen_receiver_services.py"), "start"],
+                   check=False, capture_output=True)
+    subprocess.run(["systemctl", "restart", f"gemastik-receiver-team{idx}.service"],
+                   check=False, capture_output=True)
+    # best-effort pid bookkeeping for ps
+    try:
+        out = subprocess.run(["systemctl", "show", "-p", "MainPID", f"gemastik-receiver-team{idx}.service"],
+                             capture_output=True, text=True).stdout
+        pid = out.strip().split("=")[1]
+        pidfile.write_text(pid)
+    except Exception:
+        pass
 
 def _stop_receiver(idx: int):
+    """Stop team's receiver via systemd service."""
     team_dir = TEAMS_DIR / f"team{idx}"
     pidfile = team_dir / "receiver.pid"
-    if pidfile.exists():
-        try:
-            pid = int(pidfile.read_text().strip())
-            os.kill(pid, 15)
-        except Exception:
-            pass
-        pidfile.unlink(missing_ok=True)
+    subprocess.run(["systemctl", "stop", f"gemastik-receiver-team{idx}.service"],
+                   check=False, capture_output=True)
+    pidfile.unlink(missing_ok=True)
 
 def team_logs(idx: int, service: str = None, tail: int = 100):
     team_dir = TEAMS_DIR / f"team{idx}"