fix(ssh): per-challenge SSH login + phew buffering/leak/timeout
Passwords failed on 10/16 challenges while state.json looked correct:
- only the 6 native GEMASTIK XVIII images provision 'ctfuser'; every imported
XVI/XVII image does 'echo root:${PASSWORD} | chpasswd' and logs in as root.
set_ssh_passwords() hardcoded ctfuser, so chpasswd set a password on an
account nobody uses -> 'Permission denied' everywhere.
Registry gains a per-challenge 'ssh_user'; chpasswd now targets the real
login (and ctfuser/ctf when present) and reports failures loudly.
- phew checker: chall.py block-buffers stdout through the docker exec pipe
(PYTHONUNBUFFERED now set) and leaks chall.py inside the container on
timeout (26 orphans, container saturated) -> reaps the whole exec process
group. Startup does a fresh Pailier keygen (~12 s) so crypto reads need
_CRYPTO_TIMEOUT, not the 5 s prompt default.
Adds panel/verify_ssh_creds.py (proves the state->container binding from
inside via a real login), audit_ssh_users.sh, reset_runtime.sh.
This commit is contained in:
Cyrene
+331
-9
@@ -443,8 +443,15 @@ def create_team(idx: int, label: str = None, domain: str = None):
|
||||
for j, line in enumerate(recv_env):
|
||||
if line.startswith(f"PASSWORD_{10000+coff*1000}="):
|
||||
recv_env[j] = f"PASSWORD_{10000+coff*1000}={state['chall_passwords'][name]}"
|
||||
recv_env.append(f"CHALLENGE_PORT_{name.upper()}={ports[name]['chall']}")
|
||||
recv_env.append(f"CHALLENGE_CONTAINER_{name.upper()}={name}_container_team{idx}")
|
||||
# systemd EnvironmentFile keys must match [A-Za-z_][A-Za-z0-9_]* — a
|
||||
# hyphen (gift-card, bit-canvas, ...) makes systemd log
|
||||
# "Ignoring invalid environment assignment" and DROP the line, so the
|
||||
# checker falls back to a default port/container and reports the
|
||||
# service down. Normalize to underscores on the writer; the reader
|
||||
# (gen_receiver_main._envkey) uses the same normalization.
|
||||
key = name.upper().replace("-", "_")
|
||||
recv_env.append(f"CHALLENGE_PORT_{key}={ports[name]['chall']}")
|
||||
recv_env.append(f"CHALLENGE_CONTAINER_{key}={name}_container_team{idx}")
|
||||
(recv_dir / ".env").write_text("\n".join(recv_env) + "\n")
|
||||
|
||||
# --- flags (randomized per team so each team has unique flags) ---
|
||||
@@ -484,6 +491,47 @@ def update_team(idx: int, label: str = None, domain: str = None) -> dict:
|
||||
ensure_team_domains()
|
||||
return st
|
||||
|
||||
def missing_sidecars(idx: int) -> list[str]:
|
||||
"""Sidecar services in the team's compose that are NOT running.
|
||||
|
||||
A multi-container challenge (anti-alchemy + its postgres, gemas-notes +
|
||||
database/validation, kode-viewer + redis, ...) needs its sidecars to boot:
|
||||
the main service's `create_db_conn()` retries in an infinite loop until the
|
||||
DB hostname resolves, so a missing sidecar leaves the main container
|
||||
"Up" with NO app listening and the checker reports it DOWN. Symptom class:
|
||||
container is running, port is open at the docker level, but the app never
|
||||
starts. Recover with `docker compose -p teamN up -d` (no service name).
|
||||
"""
|
||||
svc_dir = TEAMS_DIR / f"team{idx}" / "services"
|
||||
compose = svc_dir / "docker-compose.yml"
|
||||
if not compose.exists():
|
||||
return []
|
||||
declared = _compose_service_names(compose)
|
||||
if not declared:
|
||||
return []
|
||||
want = {f"team{idx}-{n}-1" for n in declared}
|
||||
running = set(subprocess.run(["docker", "ps", "--format", "{{.Names}}"],
|
||||
capture_output=True, text=True).stdout.split())
|
||||
return sorted(want - running)
|
||||
|
||||
|
||||
def _compose_service_names(compose: Path) -> list[str]:
|
||||
"""Top-level service names from a compose file, without needing PyYAML."""
|
||||
names: list[str] = []
|
||||
in_services = False
|
||||
for line in compose.read_text().splitlines():
|
||||
if not line.strip() or line.lstrip().startswith("#"):
|
||||
continue
|
||||
if not line.startswith((" ", "\t")):
|
||||
in_services = line.rstrip() == "services:"
|
||||
continue
|
||||
if in_services and line.startswith(" ") and not line.startswith(" "):
|
||||
name = line.strip().rstrip(":")
|
||||
if name and not name.startswith("-"):
|
||||
names.append(name)
|
||||
return names
|
||||
|
||||
|
||||
def start_team(idx: int):
|
||||
team_dir = TEAMS_DIR / f"team{idx}"
|
||||
if not (team_dir / "state.json").exists():
|
||||
@@ -491,6 +539,13 @@ def start_team(idx: int):
|
||||
svc_dir = team_dir / "services"
|
||||
subprocess.run(["docker", "compose", "-p", f"team{idx}", "-f", svc_dir / "docker-compose.yml", "up", "-d", "--build"],
|
||||
cwd=str(svc_dir), check=False, capture_output=True)
|
||||
# Self-heal: a per-service `up` (challenge toggle) or a raced start can
|
||||
# leave a sidecar behind while the main container looks fine. One plain
|
||||
# `up -d` reconciles the whole project (already-running ones are no-ops).
|
||||
gone = missing_sidecars(idx)
|
||||
if gone:
|
||||
subprocess.run(["docker", "compose", "-p", f"team{idx}", "-f", svc_dir / "docker-compose.yml", "up", "-d"],
|
||||
cwd=str(svc_dir), check=False, capture_output=True)
|
||||
_start_receiver(idx)
|
||||
st = json.loads((team_dir / "state.json").read_text())
|
||||
st["status"] = "running"
|
||||
@@ -498,19 +553,53 @@ def start_team(idx: int):
|
||||
# Set per-team SSH passwords at runtime (images are shared across teams,
|
||||
# so chpasswd ensures each team's containers have the team's own password).
|
||||
set_ssh_passwords(idx)
|
||||
if gone:
|
||||
print(f"[start_team] team{idx}: started missing sidecar(s): {', '.join(gone)}")
|
||||
return st
|
||||
|
||||
|
||||
def challenge_ssh_users() -> dict:
|
||||
"""{challenge_name: ssh login} from the registry.
|
||||
|
||||
Only the 6 native GEMASTIK XVIII images provision `ctfuser`. Every imported
|
||||
XVI/XVII image does `RUN echo root:${PASSWORD} | chpasswd` and logs in as
|
||||
`root` (some also create an unprivileged `ctf` for the app). Hardcoding
|
||||
`ctfuser` made chpasswd either fail or set a password on an account nobody
|
||||
uses, so SSH returned "Permission denied" for every team on 10 of 16
|
||||
challenges while state.json still looked correct.
|
||||
"""
|
||||
out = {}
|
||||
for c in registry_challenges():
|
||||
out[c["name"]] = c.get("ssh_user") or "root"
|
||||
return out
|
||||
|
||||
|
||||
def set_ssh_passwords(idx: int):
|
||||
"""Set SSH password for ctfuser in each challenge container of a team."""
|
||||
"""Set the SSH password for the CORRECT login of each challenge container.
|
||||
|
||||
The login is per-challenge (registry `ssh_user`), not a global `ctfuser`.
|
||||
Retries because right after `compose up` the container may still be booting.
|
||||
Reports failures loudly instead of writing to a log nobody reads.
|
||||
"""
|
||||
team_dir = TEAMS_DIR / f"team{idx}"
|
||||
st = json.loads((team_dir / "state.json").read_text())
|
||||
users = challenge_ssh_users()
|
||||
failures = []
|
||||
for name, coff, soff in CHALLENGES:
|
||||
cont = f"{name}_container_team{idx}"
|
||||
pw = st["chall_passwords"][name]
|
||||
cmd = f"echo 'ctfuser:{pw}' | chpasswd"
|
||||
# retry a few times — right after `compose up`, container may still be booting
|
||||
user = users.get(name, "root")
|
||||
pw = st["chall_passwords"].get(name)
|
||||
if not pw:
|
||||
failures.append(f"{cont}: no password in state")
|
||||
continue
|
||||
# Set the password for the real login AND for ctfuser when that account
|
||||
# exists, so either convention works during a transition.
|
||||
cmd = (f"id {user} >/dev/null 2>&1 && echo '{user}:{pw}' | chpasswd; "
|
||||
f"id ctfuser >/dev/null 2>&1 && echo 'ctfuser:{pw}' | chpasswd; "
|
||||
f"id ctf >/dev/null 2>&1 && echo 'ctf:{pw}' | chpasswd; "
|
||||
f"id {user} >/dev/null 2>&1")
|
||||
ok = False
|
||||
r = None
|
||||
for attempt in range(5):
|
||||
r = subprocess.run(["docker", "exec", cont, "sh", "-c", cmd],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
@@ -519,9 +608,13 @@ def set_ssh_passwords(idx: int):
|
||||
break
|
||||
time.sleep(3)
|
||||
if not ok:
|
||||
print(f"[set_ssh_passwords] {cont}: FAILED after retries ({r.stderr.strip()[:100]})")
|
||||
failures.append(f"{cont}: {(r.stderr or '').strip()[:100]}")
|
||||
else:
|
||||
print(f"[set_ssh_passwords] {cont}: OK")
|
||||
print(f"[set_ssh_passwords] {cont} (login={user}): OK")
|
||||
if failures:
|
||||
print(f"[set_ssh_passwords] team{idx} FAILED {len(failures)}/{len(CHALLENGES)}: "
|
||||
+ "; ".join(failures))
|
||||
return failures
|
||||
|
||||
def stop_team(idx: int):
|
||||
team_dir = TEAMS_DIR / f"team{idx}"
|
||||
@@ -622,12 +715,241 @@ def ensure_team_domains() -> str:
|
||||
- main: {host}
|
||||
""")
|
||||
if not out:
|
||||
return "no teams to route"
|
||||
# No teams left. The file MUST still be rewritten (to an empty router
|
||||
# set) — returning early leaves the previous content on disk, so a
|
||||
# DELETED team keeps its Traefik router and stays routable to the panel
|
||||
# portal forever. That was the stale-domain bug.
|
||||
(traefik_dir / "attackdefense-teams.yaml").write_text("http:\n routers: {}\n")
|
||||
return "no teams to route (emptied attackdefense-teams.yaml)"
|
||||
# Keep the team domain block in its own file so the main attackdefense.yaml stays untouched
|
||||
(traefik_dir / "attackdefense-teams.yaml").write_text("http:\n routers:\n" + "".join(out))
|
||||
return f"wrote {len(out)} team domain(s) in attackdefense-teams.yaml"
|
||||
|
||||
|
||||
def team_port_block(idx: int) -> list[int]:
|
||||
"""Every host port a team occupies: each challenge's chall+ssh port, the
|
||||
receiver, and the reserved panel slot."""
|
||||
st_path = TEAMS_DIR / f"team{idx}" / "state.json"
|
||||
ports: set[int] = set()
|
||||
if st_path.exists():
|
||||
st = json.loads(st_path.read_text())
|
||||
for name, pv in (st.get("ports") or {}).items():
|
||||
if isinstance(pv, dict):
|
||||
for k in ("chall", "ssh"):
|
||||
if pv.get(k):
|
||||
ports.add(int(pv[k]))
|
||||
elif isinstance(pv, int):
|
||||
ports.add(pv)
|
||||
else:
|
||||
# team dir already gone (mid-delete): derive from the port scheme
|
||||
base = PORT_BASE + idx * STEP
|
||||
for name, coff, soff in CHALLENGES:
|
||||
ports.add(base + coff)
|
||||
ports.add(base + soff)
|
||||
ports.add(base + 80)
|
||||
ports.add(base + 81)
|
||||
return sorted(ports)
|
||||
|
||||
|
||||
def sync_team_ufw(idx: int, remove: bool = False) -> dict:
|
||||
"""Reconcile UFW rules for one team's port block.
|
||||
|
||||
UFW here defaults to deny(incoming), so a port that is not explicitly
|
||||
allowed is blackholed — the team is unreachable from the internet AND from
|
||||
its own containers. Team creation never opened these ports (they were
|
||||
opened by hand earlier), so a new team silently gets no connectivity.
|
||||
|
||||
remove=True DELETES the rules instead of adding them (called from
|
||||
delete_team). The two modes are mutually exclusive: never add-then-delete,
|
||||
that would flap the rules and briefly re-open a dead team's ports.
|
||||
"""
|
||||
ports = team_port_block(idx)
|
||||
if remove:
|
||||
for p in ports:
|
||||
subprocess.run(["ufw", "--force", "delete", "allow", f"{p}/tcp"],
|
||||
check=False, capture_output=True)
|
||||
return {"team": idx, "ports": len(ports), "closed": ports, "failed": []}
|
||||
|
||||
failed = []
|
||||
for p in ports:
|
||||
r = subprocess.run(["ufw", "allow", f"{p}/tcp"], check=False, capture_output=True, text=True)
|
||||
# `ufw allow` on an existing rule prints "Skipping adding existing rule"
|
||||
# and still exits 0; a non-zero rc with a skip message is not a failure.
|
||||
if r.returncode != 0 and "Skipping" not in (r.stdout + r.stderr):
|
||||
failed.append(p)
|
||||
return {"team": idx, "ports": len(ports), "opened": [p for p in ports if p not in failed],
|
||||
"failed": failed}
|
||||
|
||||
|
||||
def _purge_team_records(idx: int) -> dict:
|
||||
"""Drop every ledger row that references a team, so a deleted team leaves
|
||||
no ghost entries on the leaderboard / points / attack topology."""
|
||||
removed = {"leaderboard": 0, "points": 0, "attacks": 0}
|
||||
|
||||
lb_path = TEAMS_DIR / "leaderboard.json"
|
||||
if lb_path.exists():
|
||||
try:
|
||||
lb = json.loads(lb_path.read_text())
|
||||
before = len(lb.get("solves", []))
|
||||
lb["solves"] = [e for e in lb.get("solves", [])
|
||||
if e.get("team") != idx and e.get("target") != idx]
|
||||
removed["leaderboard"] = before - len(lb["solves"])
|
||||
lb_path.write_text(json.dumps(lb, indent=2))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
p_path = TEAMS_DIR / "points.json"
|
||||
if p_path.exists():
|
||||
try:
|
||||
data = json.loads(p_path.read_text())
|
||||
if str(idx) in data.get("teams", {}):
|
||||
data["teams"].pop(str(idx))
|
||||
removed["points"] = 1
|
||||
p_path.write_text(json.dumps(data, indent=2))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
a_path = TEAMS_DIR / "attacks.json"
|
||||
if a_path.exists():
|
||||
try:
|
||||
log = json.loads(a_path.read_text())
|
||||
before = len(log.get("events", []))
|
||||
log["events"] = [e for e in log.get("events", [])
|
||||
if e.get("attacker") != idx and e.get("target") != idx]
|
||||
removed["attacks"] = before - len(log["events"])
|
||||
a_path.write_text(json.dumps(log, indent=2))
|
||||
except Exception:
|
||||
pass
|
||||
return removed
|
||||
|
||||
|
||||
def repair_team_receiver_env(idx: int) -> dict:
|
||||
"""Rewrite a team receiver .env with systemd-legal keys.
|
||||
|
||||
Teams created before the hyphen fix have `CHALLENGE_PORT_GIFT-CARD=` in
|
||||
their .env. systemd logs "Ignoring invalid environment assignment" and drops
|
||||
the line, so every hyphenated challenge (gift-card, bit-canvas, gleam-drive,
|
||||
more-less, anti-alchemy, gift-voucher) reports DOWN even though its
|
||||
container is up. This rewrites the file in place, fixing existing teams
|
||||
without forcing a re-create.
|
||||
"""
|
||||
env_path = TEAMS_DIR / f"team{idx}" / "receiver" / ".env"
|
||||
if not env_path.exists():
|
||||
raise FileNotFoundError(f"team{idx} receiver .env not found")
|
||||
st = json.loads((TEAMS_DIR / f"team{idx}" / "state.json").read_text())
|
||||
lines = env_path.read_text().splitlines()
|
||||
kept, fixed, dropped = [], [], []
|
||||
for line in lines:
|
||||
if line.startswith("CHALLENGE_PORT_") or line.startswith("CHALLENGE_CONTAINER_"):
|
||||
k, _, v = line.partition("=")
|
||||
nk = k.replace("-", "_")
|
||||
if nk != k:
|
||||
fixed.append(f"{k}->{nk}")
|
||||
else:
|
||||
kept.append(line)
|
||||
continue
|
||||
kept.append(line)
|
||||
# re-append authoritative values for every challenge in state
|
||||
for name in (st.get("ports") or {}):
|
||||
if name in ("receiver", "panel"):
|
||||
continue
|
||||
key = name.upper().replace("-", "_")
|
||||
kept.append(f"CHALLENGE_PORT_{key}={st['ports'][name]['chall']}")
|
||||
kept.append(f"CHALLENGE_CONTAINER_{key}={name}_container_team{idx}")
|
||||
env_path.write_text("\n".join(kept) + "\n")
|
||||
# also refresh the shared checker packages (team1 was missing xvi.Art)
|
||||
try:
|
||||
sys.path.insert(0, str(BASE / "panel"))
|
||||
from gen_receiver_main import _sync_checker_packages
|
||||
_sync_checker_packages(TEAMS_DIR / f"team{idx}" / "receiver")
|
||||
except Exception as e:
|
||||
dropped.append(f"checker sync failed: {e}")
|
||||
return {"team": idx, "fixed_keys": fixed, "notes": dropped}
|
||||
|
||||
|
||||
def delete_team(idx: int, purge_scores: bool = True) -> dict:
|
||||
"""Permanently remove ONE team and free everything it owns.
|
||||
|
||||
Teardown order matters — do the teardown against the OLD compose before
|
||||
removing the directory, or `docker compose` has no file to read and the
|
||||
containers survive as orphans:
|
||||
|
||||
1. stop the per-team receiver systemd unit and remove the unit file
|
||||
2. `docker compose -p teamN down -v` against the team compose
|
||||
(also drops the teamN_default network)
|
||||
3. force-remove any surviving <chall>_container_teamN container
|
||||
4. delete the team's UFW rules
|
||||
5. rmtree teams/teamN (compose, receiver, flags, state.json)
|
||||
6. purge leaderboard / points / attacks rows for that team
|
||||
7. rewrite the Traefik team-domain file so the domain stops resolving
|
||||
|
||||
Images (services-*) are SHARED across teams and are never removed here.
|
||||
purge_scores=False keeps the team's leaderboard/points history.
|
||||
"""
|
||||
team_dir = TEAMS_DIR / f"team{idx}"
|
||||
if not (team_dir / "state.json").exists():
|
||||
raise FileNotFoundError(f"Team {idx} not created")
|
||||
st = json.loads((team_dir / "state.json").read_text())
|
||||
label = st.get("label", f"Team {idx}")
|
||||
steps: list[str] = []
|
||||
|
||||
# 1. receiver unit
|
||||
unit = f"gemastik-receiver-team{idx}.service"
|
||||
subprocess.run(["systemctl", "disable", unit], check=False, capture_output=True)
|
||||
subprocess.run(["systemctl", "stop", unit], check=False, capture_output=True)
|
||||
unit_path = Path("/etc/systemd/system") / f"{unit}"
|
||||
if unit_path.exists():
|
||||
unit_path.unlink()
|
||||
steps.append("removed receiver unit")
|
||||
subprocess.run(["systemctl", "daemon-reload"], check=False, capture_output=True)
|
||||
|
||||
# 2. compose down (containers + network) while the compose file still exists
|
||||
svc_dir = team_dir / "services"
|
||||
compose = svc_dir / "docker-compose.yml"
|
||||
if compose.exists():
|
||||
r = subprocess.run(["docker", "compose", "-p", f"team{idx}", "-f", str(compose),
|
||||
"down", "-v", "--remove-orphans"],
|
||||
cwd=str(svc_dir), check=False, capture_output=True, text=True,
|
||||
timeout=600)
|
||||
steps.append("compose down" if r.returncode == 0 else f"compose down rc={r.returncode}")
|
||||
|
||||
# 3. force-remove leftovers (a half-written compose can leave orphans)
|
||||
left = subprocess.run(["docker", "ps", "-aq", "--filter", f"name=_container_team{idx}"],
|
||||
capture_output=True, text=True).stdout.split()
|
||||
if left:
|
||||
subprocess.run(["docker", "rm", "-f", *left], check=False, capture_output=True)
|
||||
steps.append(f"force-removed {len(left)} container(s)")
|
||||
net_rm = subprocess.run(["docker", "network", "rm", f"team{idx}_default"],
|
||||
check=False, capture_output=True, text=True)
|
||||
if net_rm.returncode == 0:
|
||||
steps.append("removed docker network")
|
||||
|
||||
# 4. UFW
|
||||
ufw = sync_team_ufw(idx, remove=True)
|
||||
steps.append(f"closed {len(ufw.get('closed', []))} ufw port(s)")
|
||||
|
||||
# 5. directory
|
||||
shutil.rmtree(team_dir, ignore_errors=True)
|
||||
if team_dir.exists():
|
||||
raise RuntimeError(f"team{idx} directory could not be removed")
|
||||
steps.append("deleted team directory")
|
||||
|
||||
# 6. ledgers
|
||||
purged = _purge_team_records(idx) if purge_scores else {}
|
||||
if purge_scores:
|
||||
steps.append("purged score records")
|
||||
|
||||
# 7. Traefik: drop the dead domain
|
||||
try:
|
||||
ensure_team_domains()
|
||||
steps.append("rewrote team domains")
|
||||
except Exception as e:
|
||||
steps.append(f"traefik rewrite failed: {e}")
|
||||
|
||||
return {"ok": True, "team": idx, "label": label, "domain": st.get("domain", ""),
|
||||
"steps": steps, "purged": purged}
|
||||
|
||||
|
||||
def list_teams() -> list:
|
||||
out = []
|
||||
if not TEAMS_DIR.exists():
|
||||
|
||||
Reference in New Issue
Block a user