fix: get all imported challenges building and running
Root causes found by prebuilding every challenge image in parallel:
- fjb: ghcr.io base is not anonymously pullable here -> official httpd:2.4.
pnpm 12 (via corepack on node:20) fails the install with
ERR_PNPM_IGNORED_BUILDS unless build scripts are approved; neither
onlyBuiltDependencies in pnpm-workspace.yaml nor --no-ignore-scripts
suppresses it. The working sequence is:
pnpm install --ignore-scripts && pnpm approve-builds --all && pnpm rebuild
- xl + kode-viewer: node:20-slim-bookworm is not a real tag -> node:20-bookworm-slim.
- burvesigner: python-dev no longer exists in bookworm -> dropped (python3-dev
was already there and the source has no py2 syntax).
- burvesigner/hirnfick/s3: apt update and install were separate RUN layers;
with the bundled apt-insecure.conf the second invocation re-resolved against
the EOL bullseye-security mirror and 404'd every package. Merged into one
'update && install' layer (fix_apt_layers.py, idempotent).
- consolidate_images.sh: teams used to build a private image per team
(team1-x ... team4-x) because no shared image existed. Since the password is
applied at runtime via chpasswd, one shared services-<name> build is enough;
this reclaims ~1.5 GB, which matters on a 79 GB disk.
- reconcile_team_state(): a challenge enabled while a team was down left
state.json without ports/flag/password, so the next compose render died with
KeyError. Now both the API and the CLI tools reconcile first.
This commit is contained in:
MythEclipse
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
# Remove per-team duplicate challenge images and repoint every team compose at
|
||||
# the shared `services-<name>` image.
|
||||
#
|
||||
# Why: before services-<name> images existed, enabling a challenge made each of
|
||||
# the 4 teams build its own copy (team1-x, team2-x, ...). A challenge image is
|
||||
# identical for all teams — the SSH password is applied at container start via
|
||||
# chpasswd, never baked in — so one shared build is enough. Those duplicates
|
||||
# were consuming ~1.5 GB on a disk that only has ~3 GB free.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT=/opt/gemastik18-final
|
||||
cd "$ROOT/panel"
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import teams as orch, compose_gen
|
||||
|
||||
for note in orch.reconcile_team_state():
|
||||
print(note)
|
||||
|
||||
for d in sorted(orch.TEAMS_DIR.glob("team*")):
|
||||
sf = d / "state.json"
|
||||
if not sf.exists():
|
||||
continue
|
||||
st = json.loads(sf.read_text())
|
||||
(d / "services" / "docker-compose.yml").write_text(
|
||||
compose_gen.render_team_compose(st["index"], st))
|
||||
print(f"team{st['index']}: compose re-rendered against shared images")
|
||||
PY
|
||||
|
||||
echo "--- removing per-team images ---"
|
||||
for img in $(docker images --format '{{.Repository}}' | grep -E '^team[0-9]+-' || true); do
|
||||
# only safe to remove if no container is using it
|
||||
if docker ps -a --format '{{.Image}}' | grep -q "^${img}$"; then
|
||||
# containers reference the image by name; remove containers first
|
||||
docker ps -a --filter "ancestor=${img}" --format '{{.Names}}' | while read -r c; do
|
||||
[ -n "$c" ] && docker rm -f "$c" >/dev/null
|
||||
done
|
||||
fi
|
||||
docker rmi "$img" >/dev/null 2>&1 && echo "removed $img" || true
|
||||
done
|
||||
|
||||
echo "--- recreating containers from shared images ---"
|
||||
for i in 1 2 3 4; do
|
||||
cd "$ROOT/teams/team$i/services"
|
||||
docker compose -p "team$i" up -d --remove-orphans >/dev/null 2>&1 &
|
||||
done
|
||||
wait
|
||||
echo "done"
|
||||
docker ps --format '{{.Names}}' | grep -c '_container_team' || true
|
||||
df -h / | tail -1
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Merge a Dockerfile's split `apt-get update` + `apt-get install` into one RUN.
|
||||
|
||||
Why: several imported challenge Dockerfiles run
|
||||
|
||||
RUN apt-get -o Acquire::AllowInsecureRepositories=true update
|
||||
RUN apt-get -y --allow-unauthenticated install -y ...
|
||||
|
||||
as two separate layers. The bundled apt-insecure.conf enables insecure/downgrade
|
||||
repositories, and a second apt invocation in a later layer re-resolves against
|
||||
whatever index the first one left behind — which on the EOL bullseye-security
|
||||
mirror returns 404 for every package:
|
||||
|
||||
Err:1 http://deb.debian.org/debian-security bullseye-security/main amd64 libsystemd0
|
||||
404 Not Found
|
||||
|
||||
Merging them into a single `update && install` layer (the pattern the working
|
||||
gemastik18 challenges use) makes apt resolve once, from a fresh index.
|
||||
|
||||
Idempotent: a Dockerfile that already has the combined form is left untouched.
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SERVICES = Path("/opt/gemastik18-final/services")
|
||||
|
||||
UPDATE_RE = re.compile(
|
||||
r"^RUN\s+apt-get\s+(?P<opts>(?:-o\s+\S+\s+)*)update\s*$"
|
||||
)
|
||||
INSTALL_RE = re.compile(r"^RUN\s+(?P<rest>apt-get\s+.*\binstall\b.*)$")
|
||||
|
||||
|
||||
def fix(text: str) -> tuple[str, bool]:
|
||||
lines = text.splitlines()
|
||||
out: list[str] = []
|
||||
i = 0
|
||||
changed = False
|
||||
while i < len(lines):
|
||||
m = UPDATE_RE.match(lines[i])
|
||||
if not m:
|
||||
out.append(lines[i])
|
||||
i += 1
|
||||
continue
|
||||
# look ahead: is the very next non-empty line an install?
|
||||
j = i + 1
|
||||
while j < len(lines) and not lines[j].strip():
|
||||
j += 1
|
||||
if j < len(lines):
|
||||
mi = INSTALL_RE.match(lines[j])
|
||||
if mi:
|
||||
out.append(f"RUN apt-get {m.group('opts')}update && \\")
|
||||
out.append(f" {mi.group('rest')}")
|
||||
changed = True
|
||||
i = j + 1
|
||||
continue
|
||||
out.append(lines[i])
|
||||
i += 1
|
||||
return "\n".join(out) + ("\n" if text.endswith("\n") else ""), changed
|
||||
|
||||
|
||||
def main(argv):
|
||||
names = argv[1:] or [p.name for p in sorted(SERVICES.iterdir())
|
||||
if p.is_dir() and (p / "Dockerfile").exists()]
|
||||
changed = []
|
||||
for name in names:
|
||||
p = SERVICES / name / "Dockerfile"
|
||||
if not p.exists():
|
||||
continue
|
||||
text = p.read_text()
|
||||
new, did = fix(text)
|
||||
if did:
|
||||
p.write_text(new)
|
||||
changed.append(name)
|
||||
print(f"{name}: merged apt update+install into one RUN")
|
||||
print("rewritten:", ", ".join(changed) if changed else "(none)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
@@ -390,6 +390,13 @@ async def api_challenge_toggle(challenge: str, req: Request):
|
||||
if not found:
|
||||
raise HTTPException(404, "Unknown challenge")
|
||||
changed = orch.set_challenge_enabled(challenge, enabled)
|
||||
# The registry is the source of truth, but a team's state.json must also
|
||||
# carry ports/password/flag for the new challenge before the compose can be
|
||||
# rendered. reconcile_team_state() fills in anything missing.
|
||||
try:
|
||||
orch.reconcile_team_state()
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"Rekonsiliasi state tim gagal: {e}")
|
||||
# apply to live teams (build/up or stop/remove + receiver restart);
|
||||
# skip rebuild when the flag didn't actually change
|
||||
if "unchanged" in changed:
|
||||
|
||||
@@ -84,6 +84,63 @@ def set_challenge_enabled(name: str, enabled: bool) -> dict:
|
||||
_SYNC_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def reconcile_team_state() -> list[str]:
|
||||
"""Make every team's state.json agree with the registry's enabled set.
|
||||
|
||||
Needed whenever the registry is edited directly (panel/set_enabled.py) or
|
||||
a challenge is enabled but a team was offline/stopped while it happened:
|
||||
state.json must carry ports + a chall password + a flag for every enabled
|
||||
challenge, otherwise render_team_compose() raises KeyError on ports[name].
|
||||
|
||||
Returns a list of human-readable actions taken.
|
||||
"""
|
||||
reg = load_registry()
|
||||
enabled = enabled_challenges()
|
||||
# rebuild the derived CHALLENGES for fresh creates
|
||||
global CHALLENGES
|
||||
CHALLENGES = [(c["name"], c["chall_offset"], c["ssh_offset"]) for c in enabled]
|
||||
notes: list[str] = []
|
||||
for d in sorted(TEAMS_DIR.glob("team*")):
|
||||
sf = d / "state.json"
|
||||
if not sf.exists():
|
||||
continue
|
||||
st = json.loads(sf.read_text())
|
||||
idx = st["index"]
|
||||
dirty = False
|
||||
st.setdefault("ports", {})
|
||||
st.setdefault("chall_passwords", {})
|
||||
st.setdefault("flags", {})
|
||||
for ch in enabled:
|
||||
name = ch["name"]
|
||||
if name not in st["ports"]:
|
||||
st["ports"][name] = {"chall": 30000 + idx * 1000 + ch["chall_offset"],
|
||||
"ssh": 30000 + idx * 1000 + ch["ssh_offset"]}
|
||||
dirty = True
|
||||
notes.append(f"team{idx}: allocated ports for {name}")
|
||||
if not st["chall_passwords"].get(name):
|
||||
st["chall_passwords"][name] = f"chall{idx}_{name}_{secrets.token_hex(4)}"
|
||||
dirty = True
|
||||
if not st["flags"].get(name):
|
||||
flag = f"GEMASTIK18{{TEAM{idx}_{name.upper()}_{secrets.token_hex(6)}}}"
|
||||
st["flags"][name] = flag
|
||||
fd = d / "receiver" / "flags"
|
||||
fd.mkdir(parents=True, exist_ok=True)
|
||||
(fd / f"{name}.txt").write_text(flag)
|
||||
dirty = True
|
||||
notes.append(f"team{idx}: minted flag for {name}")
|
||||
# make sure the source tree is present for a later `build:`
|
||||
ts = d / "services" / name
|
||||
if not ts.exists():
|
||||
src = SERVICES_SRC / name
|
||||
if src.exists():
|
||||
shutil.copytree(src, ts,
|
||||
ignore=shutil.ignore_patterns("__pycache__", "*.pyc", ".git"))
|
||||
notes.append(f"team{idx}: copied source for {name}")
|
||||
if dirty:
|
||||
sf.write_text(json.dumps(st, indent=2))
|
||||
return notes
|
||||
|
||||
|
||||
def sync_challenge_runtime(name: str, enabled: bool) -> dict:
|
||||
"""Apply one challenge's enabled flag to every live team.
|
||||
|
||||
|
||||
@@ -5,8 +5,8 @@ ARG PASSWORD
|
||||
WORKDIR /opt
|
||||
|
||||
COPY apt-insecure.conf /etc/apt/apt.conf.d/99gemastik-insecure
|
||||
RUN apt-get -o Acquire::AllowInsecureRepositories=true update
|
||||
RUN apt-get -y --allow-unauthenticated install -y nano openssh-server \
|
||||
RUN apt-get -o Acquire::AllowInsecureRepositories=true update && \
|
||||
apt-get -y --allow-unauthenticated install -y nano openssh-server \
|
||||
gcc python-dev python3-dev libgmp3-dev curl
|
||||
|
||||
RUN echo root:${PASSWORD} | chpasswd
|
||||
|
||||
+11
-3
@@ -7,9 +7,17 @@ RUN corepack enable
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY ./frontend/pnpm-workspace.yaml /app/
|
||||
COPY ./frontend/package.json ./frontend/pnpm-lock.yaml /app/
|
||||
RUN pnpm install --frozen-lockfile
|
||||
# pnpm >=10 blocks dependency lifecycle scripts by default and then FAILS the
|
||||
# install with ERR_PNPM_IGNORED_BUILDS (pnpm 12 still does this even with
|
||||
# onlyBuiltDependencies in pnpm-workspace.yaml, and --no-ignore-scripts does not
|
||||
# suppress the error either). esbuild's postinstall places the platform binary
|
||||
# the Vite build needs, so approve pending build scripts non-interactively.
|
||||
# --frozen-lockfile is dropped because the added config keys change the lockfile
|
||||
# hash and would fail the install outright.
|
||||
COPY ./frontend/pnpm-workspace.yaml ./frontend/package.json ./frontend/pnpm-lock.yaml /app/
|
||||
RUN pnpm install --ignore-scripts \
|
||||
&& pnpm approve-builds --all \
|
||||
&& pnpm rebuild
|
||||
|
||||
FROM base AS build
|
||||
|
||||
|
||||
@@ -3,14 +3,6 @@
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"pnpm": {
|
||||
"onlyBuiltDependencies": [
|
||||
"esbuild",
|
||||
"@scarf/scarf",
|
||||
"sharp",
|
||||
"unrs-resolver"
|
||||
]
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
# pnpm 12 (shipped by corepack on node:20) blocks dependency lifecycle scripts
|
||||
# by default and then FAILS the install with ERR_PNPM_IGNORED_BUILDS. Neither
|
||||
# onlyBuiltDependencies nor --no-ignore-scripts suppresses that error in pnpm 12.
|
||||
# The documented opt-out is `strictIgnoredBuiltDependencies: false`; esbuild's
|
||||
# postinstall is what places the platform binary the Vite build needs.
|
||||
strictIgnoredBuiltDependencies: false
|
||||
allowBuilds:
|
||||
esbuild: true
|
||||
@@ -5,8 +5,8 @@ ARG PASSWORD
|
||||
WORKDIR /opt
|
||||
|
||||
COPY apt-insecure.conf /etc/apt/apt.conf.d/99gemastik-insecure
|
||||
RUN apt-get -o Acquire::AllowInsecureRepositories=true update
|
||||
RUN apt-get -y --allow-unauthenticated install -y nano openssh-server \
|
||||
RUN apt-get -o Acquire::AllowInsecureRepositories=true update && \
|
||||
apt-get -y --allow-unauthenticated install -y nano openssh-server \
|
||||
gcc curl
|
||||
|
||||
RUN echo root:${PASSWORD} | chpasswd
|
||||
|
||||
@@ -5,8 +5,8 @@ ARG PASSWORD
|
||||
WORKDIR /opt
|
||||
|
||||
COPY apt-insecure.conf /etc/apt/apt.conf.d/99gemastik-insecure
|
||||
RUN apt-get -o Acquire::AllowInsecureRepositories=true update
|
||||
RUN apt-get -y --allow-unauthenticated install -y nano openssh-server curl
|
||||
RUN apt-get -o Acquire::AllowInsecureRepositories=true update && \
|
||||
apt-get -y --allow-unauthenticated install -y nano openssh-server curl
|
||||
|
||||
RUN echo root:${PASSWORD} | chpasswd
|
||||
RUN echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config
|
||||
|
||||
Reference in New Issue
Block a user