From ef385c3397615c9b46c908218e27ef87dda7fc47 Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Fri, 25 Sep 2026 21:03:29 +0800 Subject: [PATCH] fix: get all imported challenges building and running Root causes found by prebuilding every challenge image in parallel: - fjb: ghcr.io base is not anonymously pullable here -> official httpd:2.4. pnpm 12 (via corepack on node:20) fails the install with ERR_PNPM_IGNORED_BUILDS unless build scripts are approved; neither onlyBuiltDependencies in pnpm-workspace.yaml nor --no-ignore-scripts suppresses it. The working sequence is: pnpm install --ignore-scripts && pnpm approve-builds --all && pnpm rebuild - xl + kode-viewer: node:20-slim-bookworm is not a real tag -> node:20-bookworm-slim. - burvesigner: python-dev no longer exists in bookworm -> dropped (python3-dev was already there and the source has no py2 syntax). - burvesigner/hirnfick/s3: apt update and install were separate RUN layers; with the bundled apt-insecure.conf the second invocation re-resolved against the EOL bullseye-security mirror and 404'd every package. Merged into one 'update && install' layer (fix_apt_layers.py, idempotent). - consolidate_images.sh: teams used to build a private image per team (team1-x ... team4-x) because no shared image existed. Since the password is applied at runtime via chpasswd, one shared services- build is enough; this reclaims ~1.5 GB, which matters on a 79 GB disk. - reconcile_team_state(): a challenge enabled while a team was down left state.json without ports/flag/password, so the next compose render died with KeyError. Now both the API and the CLI tools reconcile first. --- panel/consolidate_images.sh | 54 +++++++++++++++ panel/fix_apt_layers.py | 81 +++++++++++++++++++++++ panel/main.py | 7 ++ panel/teams.py | 57 ++++++++++++++++ services/burvesigner/Dockerfile | 4 +- services/fjb/Dockerfile | 14 +++- services/fjb/frontend/package.json | 8 --- services/fjb/frontend/pnpm-workspace.yaml | 8 +++ services/hirnfick/Dockerfile | 4 +- services/s3/Dockerfile | 4 +- 10 files changed, 224 insertions(+), 17 deletions(-) create mode 100755 panel/consolidate_images.sh create mode 100644 panel/fix_apt_layers.py create mode 100644 services/fjb/frontend/pnpm-workspace.yaml diff --git a/panel/consolidate_images.sh b/panel/consolidate_images.sh new file mode 100755 index 0000000..9ea180e --- /dev/null +++ b/panel/consolidate_images.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Remove per-team duplicate challenge images and repoint every team compose at +# the shared `services-` image. +# +# Why: before services- images existed, enabling a challenge made each of +# the 4 teams build its own copy (team1-x, team2-x, ...). A challenge image is +# identical for all teams — the SSH password is applied at container start via +# chpasswd, never baked in — so one shared build is enough. Those duplicates +# were consuming ~1.5 GB on a disk that only has ~3 GB free. +set -euo pipefail + +ROOT=/opt/gemastik18-final +cd "$ROOT/panel" +python3 - <<'PY' +import json +from pathlib import Path +import sys +sys.path.insert(0, '.') +import teams as orch, compose_gen + +for note in orch.reconcile_team_state(): + print(note) + +for d in sorted(orch.TEAMS_DIR.glob("team*")): + sf = d / "state.json" + if not sf.exists(): + continue + st = json.loads(sf.read_text()) + (d / "services" / "docker-compose.yml").write_text( + compose_gen.render_team_compose(st["index"], st)) + print(f"team{st['index']}: compose re-rendered against shared images") +PY + +echo "--- removing per-team images ---" +for img in $(docker images --format '{{.Repository}}' | grep -E '^team[0-9]+-' || true); do + # only safe to remove if no container is using it + if docker ps -a --format '{{.Image}}' | grep -q "^${img}$"; then + # containers reference the image by name; remove containers first + docker ps -a --filter "ancestor=${img}" --format '{{.Names}}' | while read -r c; do + [ -n "$c" ] && docker rm -f "$c" >/dev/null + done + fi + docker rmi "$img" >/dev/null 2>&1 && echo "removed $img" || true +done + +echo "--- recreating containers from shared images ---" +for i in 1 2 3 4; do + cd "$ROOT/teams/team$i/services" + docker compose -p "team$i" up -d --remove-orphans >/dev/null 2>&1 & +done +wait +echo "done" +docker ps --format '{{.Names}}' | grep -c '_container_team' || true +df -h / | tail -1 diff --git a/panel/fix_apt_layers.py b/panel/fix_apt_layers.py new file mode 100644 index 0000000..c4551ee --- /dev/null +++ b/panel/fix_apt_layers.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Merge a Dockerfile's split `apt-get update` + `apt-get install` into one RUN. + +Why: several imported challenge Dockerfiles run + + RUN apt-get -o Acquire::AllowInsecureRepositories=true update + RUN apt-get -y --allow-unauthenticated install -y ... + +as two separate layers. The bundled apt-insecure.conf enables insecure/downgrade +repositories, and a second apt invocation in a later layer re-resolves against +whatever index the first one left behind — which on the EOL bullseye-security +mirror returns 404 for every package: + + Err:1 http://deb.debian.org/debian-security bullseye-security/main amd64 libsystemd0 + 404 Not Found + +Merging them into a single `update && install` layer (the pattern the working +gemastik18 challenges use) makes apt resolve once, from a fresh index. + +Idempotent: a Dockerfile that already has the combined form is left untouched. +""" +import re +import sys +from pathlib import Path + +SERVICES = Path("/opt/gemastik18-final/services") + +UPDATE_RE = re.compile( + r"^RUN\s+apt-get\s+(?P(?:-o\s+\S+\s+)*)update\s*$" +) +INSTALL_RE = re.compile(r"^RUN\s+(?Papt-get\s+.*\binstall\b.*)$") + + +def fix(text: str) -> tuple[str, bool]: + lines = text.splitlines() + out: list[str] = [] + i = 0 + changed = False + while i < len(lines): + m = UPDATE_RE.match(lines[i]) + if not m: + out.append(lines[i]) + i += 1 + continue + # look ahead: is the very next non-empty line an install? + j = i + 1 + while j < len(lines) and not lines[j].strip(): + j += 1 + if j < len(lines): + mi = INSTALL_RE.match(lines[j]) + if mi: + out.append(f"RUN apt-get {m.group('opts')}update && \\") + out.append(f" {mi.group('rest')}") + changed = True + i = j + 1 + continue + out.append(lines[i]) + i += 1 + return "\n".join(out) + ("\n" if text.endswith("\n") else ""), changed + + +def main(argv): + names = argv[1:] or [p.name for p in sorted(SERVICES.iterdir()) + if p.is_dir() and (p / "Dockerfile").exists()] + changed = [] + for name in names: + p = SERVICES / name / "Dockerfile" + if not p.exists(): + continue + text = p.read_text() + new, did = fix(text) + if did: + p.write_text(new) + changed.append(name) + print(f"{name}: merged apt update+install into one RUN") + print("rewritten:", ", ".join(changed) if changed else "(none)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/panel/main.py b/panel/main.py index 133909a..bda31a9 100644 --- a/panel/main.py +++ b/panel/main.py @@ -390,6 +390,13 @@ async def api_challenge_toggle(challenge: str, req: Request): if not found: raise HTTPException(404, "Unknown challenge") changed = orch.set_challenge_enabled(challenge, enabled) + # The registry is the source of truth, but a team's state.json must also + # carry ports/password/flag for the new challenge before the compose can be + # rendered. reconcile_team_state() fills in anything missing. + try: + orch.reconcile_team_state() + except Exception as e: + raise HTTPException(500, f"Rekonsiliasi state tim gagal: {e}") # apply to live teams (build/up or stop/remove + receiver restart); # skip rebuild when the flag didn't actually change if "unchanged" in changed: diff --git a/panel/teams.py b/panel/teams.py index c36d129..dad2a0c 100644 --- a/panel/teams.py +++ b/panel/teams.py @@ -84,6 +84,63 @@ def set_challenge_enabled(name: str, enabled: bool) -> dict: _SYNC_LOCK = threading.Lock() +def reconcile_team_state() -> list[str]: + """Make every team's state.json agree with the registry's enabled set. + + Needed whenever the registry is edited directly (panel/set_enabled.py) or + a challenge is enabled but a team was offline/stopped while it happened: + state.json must carry ports + a chall password + a flag for every enabled + challenge, otherwise render_team_compose() raises KeyError on ports[name]. + + Returns a list of human-readable actions taken. + """ + reg = load_registry() + enabled = enabled_challenges() + # rebuild the derived CHALLENGES for fresh creates + global CHALLENGES + CHALLENGES = [(c["name"], c["chall_offset"], c["ssh_offset"]) for c in enabled] + notes: list[str] = [] + for d in sorted(TEAMS_DIR.glob("team*")): + sf = d / "state.json" + if not sf.exists(): + continue + st = json.loads(sf.read_text()) + idx = st["index"] + dirty = False + st.setdefault("ports", {}) + st.setdefault("chall_passwords", {}) + st.setdefault("flags", {}) + for ch in enabled: + name = ch["name"] + if name not in st["ports"]: + st["ports"][name] = {"chall": 30000 + idx * 1000 + ch["chall_offset"], + "ssh": 30000 + idx * 1000 + ch["ssh_offset"]} + dirty = True + notes.append(f"team{idx}: allocated ports for {name}") + if not st["chall_passwords"].get(name): + st["chall_passwords"][name] = f"chall{idx}_{name}_{secrets.token_hex(4)}" + dirty = True + if not st["flags"].get(name): + flag = f"GEMASTIK18{{TEAM{idx}_{name.upper()}_{secrets.token_hex(6)}}}" + st["flags"][name] = flag + fd = d / "receiver" / "flags" + fd.mkdir(parents=True, exist_ok=True) + (fd / f"{name}.txt").write_text(flag) + dirty = True + notes.append(f"team{idx}: minted flag for {name}") + # make sure the source tree is present for a later `build:` + ts = d / "services" / name + if not ts.exists(): + src = SERVICES_SRC / name + if src.exists(): + shutil.copytree(src, ts, + ignore=shutil.ignore_patterns("__pycache__", "*.pyc", ".git")) + notes.append(f"team{idx}: copied source for {name}") + if dirty: + sf.write_text(json.dumps(st, indent=2)) + return notes + + def sync_challenge_runtime(name: str, enabled: bool) -> dict: """Apply one challenge's enabled flag to every live team. diff --git a/services/burvesigner/Dockerfile b/services/burvesigner/Dockerfile index 5e348c6..ed7d7a4 100644 --- a/services/burvesigner/Dockerfile +++ b/services/burvesigner/Dockerfile @@ -5,8 +5,8 @@ ARG PASSWORD WORKDIR /opt COPY apt-insecure.conf /etc/apt/apt.conf.d/99gemastik-insecure -RUN apt-get -o Acquire::AllowInsecureRepositories=true update -RUN apt-get -y --allow-unauthenticated install -y nano openssh-server \ +RUN apt-get -o Acquire::AllowInsecureRepositories=true update && \ + apt-get -y --allow-unauthenticated install -y nano openssh-server \ gcc python-dev python3-dev libgmp3-dev curl RUN echo root:${PASSWORD} | chpasswd diff --git a/services/fjb/Dockerfile b/services/fjb/Dockerfile index 5db006a..897a537 100644 --- a/services/fjb/Dockerfile +++ b/services/fjb/Dockerfile @@ -7,9 +7,17 @@ RUN corepack enable WORKDIR /app -COPY ./frontend/pnpm-workspace.yaml /app/ -COPY ./frontend/package.json ./frontend/pnpm-lock.yaml /app/ -RUN pnpm install --frozen-lockfile +# pnpm >=10 blocks dependency lifecycle scripts by default and then FAILS the +# install with ERR_PNPM_IGNORED_BUILDS (pnpm 12 still does this even with +# onlyBuiltDependencies in pnpm-workspace.yaml, and --no-ignore-scripts does not +# suppress the error either). esbuild's postinstall places the platform binary +# the Vite build needs, so approve pending build scripts non-interactively. +# --frozen-lockfile is dropped because the added config keys change the lockfile +# hash and would fail the install outright. +COPY ./frontend/pnpm-workspace.yaml ./frontend/package.json ./frontend/pnpm-lock.yaml /app/ +RUN pnpm install --ignore-scripts \ + && pnpm approve-builds --all \ + && pnpm rebuild FROM base AS build diff --git a/services/fjb/frontend/package.json b/services/fjb/frontend/package.json index 5786889..8464f56 100644 --- a/services/fjb/frontend/package.json +++ b/services/fjb/frontend/package.json @@ -3,14 +3,6 @@ "private": true, "version": "0.0.0", "type": "module", - "pnpm": { - "onlyBuiltDependencies": [ - "esbuild", - "@scarf/scarf", - "sharp", - "unrs-resolver" - ] - }, "scripts": { "dev": "vite", "build": "vite build", diff --git a/services/fjb/frontend/pnpm-workspace.yaml b/services/fjb/frontend/pnpm-workspace.yaml new file mode 100644 index 0000000..d7b664d --- /dev/null +++ b/services/fjb/frontend/pnpm-workspace.yaml @@ -0,0 +1,8 @@ +# pnpm 12 (shipped by corepack on node:20) blocks dependency lifecycle scripts +# by default and then FAILS the install with ERR_PNPM_IGNORED_BUILDS. Neither +# onlyBuiltDependencies nor --no-ignore-scripts suppresses that error in pnpm 12. +# The documented opt-out is `strictIgnoredBuiltDependencies: false`; esbuild's +# postinstall is what places the platform binary the Vite build needs. +strictIgnoredBuiltDependencies: false +allowBuilds: + esbuild: true diff --git a/services/hirnfick/Dockerfile b/services/hirnfick/Dockerfile index 7d44773..61ad121 100644 --- a/services/hirnfick/Dockerfile +++ b/services/hirnfick/Dockerfile @@ -5,8 +5,8 @@ ARG PASSWORD WORKDIR /opt COPY apt-insecure.conf /etc/apt/apt.conf.d/99gemastik-insecure -RUN apt-get -o Acquire::AllowInsecureRepositories=true update -RUN apt-get -y --allow-unauthenticated install -y nano openssh-server \ +RUN apt-get -o Acquire::AllowInsecureRepositories=true update && \ + apt-get -y --allow-unauthenticated install -y nano openssh-server \ gcc curl RUN echo root:${PASSWORD} | chpasswd diff --git a/services/s3/Dockerfile b/services/s3/Dockerfile index 585224d..bd469bc 100644 --- a/services/s3/Dockerfile +++ b/services/s3/Dockerfile @@ -5,8 +5,8 @@ ARG PASSWORD WORKDIR /opt COPY apt-insecure.conf /etc/apt/apt.conf.d/99gemastik-insecure -RUN apt-get -o Acquire::AllowInsecureRepositories=true update -RUN apt-get -y --allow-unauthenticated install -y nano openssh-server curl +RUN apt-get -o Acquire::AllowInsecureRepositories=true update && \ + apt-get -y --allow-unauthenticated install -y nano openssh-server curl RUN echo root:${PASSWORD} | chpasswd RUN echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config