From 1360a8fa6567ce8081ebff635956af79111604da Mon Sep 17 00:00:00 2001 From: ginnoir Date: Sun, 7 Jun 2026 20:25:30 -0500 Subject: [PATCH] =?UTF-8?q?feat(romhacks):=20automated=20hand-off=20?= =?UTF-8?q?=E2=80=94=20route=20downloads=20to=20RomM=20+=20library,=20rend?= =?UTF-8?q?er=20notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handoff.py classifies completed downloads (console rom vs RPG-Maker fangame vs patch, via bsdtar archive peek incl. RAR5), copies console roms into Emulation/roms/ for RomM, moves the full folder (rom + guides) into the library archive, and renders note.md + handoff.json per hack for the Obsidian publish step. Wired into the cycle; single /data mount keeps moves as fast renames. Validated on 4 real games. --- stacks/romhacks/docker-compose.yml | 12 +- stacks/romhacks/orchestrator/Dockerfile | 6 +- stacks/romhacks/orchestrator/handoff.py | 247 ++++++++++++++++++++++++ stacks/romhacks/orchestrator/run.sh | 3 + stacks/romhacks/stack.env | 3 + 5 files changed, 268 insertions(+), 3 deletions(-) create mode 100644 stacks/romhacks/orchestrator/handoff.py diff --git a/stacks/romhacks/docker-compose.yml b/stacks/romhacks/docker-compose.yml index b9cf031..b5ae161 100644 --- a/stacks/romhacks/docker-compose.yml +++ b/stacks/romhacks/docker-compose.yml @@ -48,11 +48,21 @@ services: networks: [romhacks] env_file: - stack.env + environment: + # Hand-off paths. /data is the single parent mount so incoming->library + # moves are instant renames (not cross-device copies); /emulation is the + # RomM Structure-A roms parent so console hacks land in roms//. + - METADATA_DIR=/data/metadata + - INCOMING_DIR=/data/incoming + - LIBRARY_DIR=/data/library + - EMULATION_DIR=/emulation + - HANDOFF_STATE=/state/handled.json volumes: - /config/romhacks/exports:/exports - /config/romhacks/state:/state - /config/romhacks/crawljobs:/crawljobs - - /storage1/labdata/romhacks/metadata:/metadata + - /storage1/labdata/romhacks:/data + - /storage1/Emulation/roms:/emulation jdownloader: image: jlesage/jdownloader-2:latest diff --git a/stacks/romhacks/orchestrator/Dockerfile b/stacks/romhacks/orchestrator/Dockerfile index 4073aaa..815eb6d 100644 --- a/stacks/romhacks/orchestrator/Dockerfile +++ b/stacks/romhacks/orchestrator/Dockerfile @@ -6,10 +6,12 @@ FROM tyrrrz/discordchatexporter:stable USER root -RUN apk add --no-cache python3 +# python3 for the scripts; libarchive-tools (bsdtar) to peek inside zip/rar/7z +# archives during hand-off classification (reads RAR5, which 7zip's Alpine build can't). +RUN apk add --no-cache python3 libarchive-tools WORKDIR /app -COPY orchestrate.py run.sh channels.json ./ +COPY orchestrate.py handoff.py run.sh channels.json ./ RUN chmod +x run.sh ENTRYPOINT ["./run.sh"] diff --git a/stacks/romhacks/orchestrator/handoff.py b/stacks/romhacks/orchestrator/handoff.py new file mode 100644 index 0000000..084f48f --- /dev/null +++ b/stacks/romhacks/orchestrator/handoff.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""romhacks hand-off — route completed downloads into the library + RomM, and +render an Obsidian note per hack. + +For each *completed* `incoming//` folder (no in-progress JDownloader files): + 1. Classify it — console rom (by file/archive extension) vs RPG-Maker style + fan-game (Game.exe / rgss / rxdata) vs patch-only. + 2. Route — console roms get the rom artifact copied into + `/storage1/Emulation//` so RomM scans it; fan-games stay out of + RomM. The WHOLE folder (rom + guides + spreadsheets) is moved to + `/storage1/labdata/romhacks/library//` as the permanent archive + (folder contents are a feature, not noise — keep them all). + 3. Render `note.md` + `handoff.json` into the library folder for the Obsidian + publish step (done separately via the Obsidian MCP — livesync's chunked + CouchDB format isn't safe to write from here). + +Idempotent via a handled-slug state file. HANDOFF_DRY=true (default) classifies +and renders to a scratch dir but moves/copies nothing. +""" +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys + +INCOMING = pathlib.Path(os.environ.get("INCOMING_DIR", "/incoming")) +LIBRARY = pathlib.Path(os.environ.get("LIBRARY_DIR", "/library")) +EMU = pathlib.Path(os.environ.get("EMULATION_DIR", "/emulation")) +METADATA = pathlib.Path(os.environ.get("METADATA_DIR", "/metadata")) +HANDLED = pathlib.Path(os.environ.get("HANDOFF_STATE", "/state/handled.json")) +DRY = os.environ.get("HANDOFF_DRY", "true").lower() != "false" + +ROM_EXT_PLATFORM = { + ".gba": "gba", ".gb": "gb", ".gbc": "gbc", ".nds": "nds", + ".3ds": "3ds", ".cia": "3ds", ".nsp": "switch", ".xci": "switch", +} +ARCHIVE_EXTS = {".zip", ".rar", ".7z"} +PATCH_EXTS = {".ips", ".bps", ".ups", ".xdelta"} +INPROGRESS = (".part", ".jdownload", ".jdownload.cfg", ".tmp", ".crdownload") +FANGAME_HINTS = ("game.exe", "game.ini", ".rxdata", ".rvdata", ".rvdata2", ".rgssad", ".rgss2a", ".rgss3a") +ART_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".gif") +# Document / extra files worth keeping (guides, spreadsheets, changelogs, and +# supplementary or alternate-version archives). Deliberately excludes raw game +# assets (png/audio/scripts) so an extracted tree isn't counted as "guides". +GUIDE_EXTS = {".html", ".htm", ".pdf", ".xlsx", ".xls", ".ods", ".csv", ".doc", + ".docx", ".txt", ".md", ".rtf", ".zip", ".rar", ".7z"} + + +def slug(s): + return re.sub(r"[^A-Za-z0-9._-]+", "_", s or "").strip("_")[:120] or "unknown" + + +def load_handled(): + try: + return set(json.loads(HANDLED.read_text())["handled"]) + except Exception: + return set() + + +def save_handled(handled): + HANDLED.parent.mkdir(parents=True, exist_ok=True) + HANDLED.write_text(json.dumps({"handled": sorted(handled)})) + + +def is_complete(folder): + for p in folder.rglob("*"): + if p.is_file() and p.name.lower().endswith(INPROGRESS): + return False + return True + + +def archive_listing(path): + """Return lowercased entry names inside an archive. bsdtar (libarchive) lists + zip/rar/7z uniformly, including RAR5 which the Alpine 7zip build can't read.""" + try: + out = subprocess.run(["bsdtar", "-tf", str(path)], capture_output=True, + text=True, timeout=180) + return out.stdout.lower() + except Exception: + return "" + + +def classify(folder): + """Return (kind, platform, primary_artifact). kind in console|fangame|patch|unknown.""" + files = [p for p in folder.rglob("*") if p.is_file()] + # 1. Loose rom file wins outright. + for p in files: + plat = ROM_EXT_PLATFORM.get(p.suffix.lower()) + if plat: + return "console", plat, p + # 2. Fan-game markers (loose). + for p in files: + name = p.name.lower() + if name in ("game.exe", "game.ini") or p.suffix.lower() in FANGAME_HINTS: + archive = max((a for a in files if a.suffix.lower() in ARCHIVE_EXTS), + key=lambda a: a.stat().st_size, default=None) + return "fangame", "pc", archive + # 3. Peek inside archives (largest first). + archives = sorted((p for p in files if p.suffix.lower() in ARCHIVE_EXTS), + key=lambda a: a.stat().st_size, reverse=True) + for a in archives: + listing = archive_listing(a) + if any(h in listing for h in ("game.exe", "rgssad", ".rxdata", ".rvdata")): + return "fangame", "pc", a + for ext, plat in ROM_EXT_PLATFORM.items(): + if ext in listing: + return "console", plat, a + # 4. Patch-only. + if any(p.suffix.lower() in PATCH_EXTS for p in files): + patch = max((p for p in files if p.suffix.lower() in PATCH_EXTS), + key=lambda p: p.stat().st_size) + return "patch", "patch", patch + # 5. Fall back to the biggest archive as the artifact, platform unknown. + return "unknown", "unknown", (archives[0] if archives else None) + + +def collect_guides(folder, primary): + """Files that aren't the primary artifact or art — the guides/spreadsheets.""" + guides = [] + for p in folder.rglob("*"): + if not p.is_file() or p == primary: + continue + if p.suffix.lower() in GUIDE_EXTS: + guides.append(p) + return guides + + +def render_note(meta, kind, platform, primary, guides, art, folder): + name = meta.get("name", folder.name) + lines = [ + "---", + f"title: {name}", + f"platform: {platform}", + f"kind: {kind}", + "tags: [pokemon-romhack]", + "---", + "", + f"# {name}", + "", + ] + for a in art: + lines.append(f"![[{a.name}]]") + if art: + lines.append("") + desc = (meta.get("description") or "").strip() + if desc: + lines += ["## Description", "", desc, ""] + links = meta.get("links") or [] + if links: + lines += ["## Download links", ""] + lines += [f"- {l}" for l in links] + lines.append("") + lines += ["## Files", "", + f"- **Artifact:** `{primary.name if primary else 'n/a'}` ({kind}/{platform})"] + if guides: + lines.append(f"- **Guides / extras ({len(guides)}):**") + for g in sorted(guides): + lines.append(f" - `{g.relative_to(folder)}`") + lines.append("") + return "\n".join(lines) + + +def route(folder, slug_name, kind, platform, primary): + dest_lib = LIBRARY / slug_name + # console roms become RomM-scannable; fan-games/patches stay in the library only. + if kind == "console" and primary is not None: + plat_dir = EMU / platform + if not DRY: + plat_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(primary, plat_dir / primary.name) + if not DRY: + LIBRARY.mkdir(parents=True, exist_ok=True) + if dest_lib.exists(): + shutil.rmtree(dest_lib) + shutil.move(str(folder), str(dest_lib)) + return dest_lib + + +def process(folder, handled): + slug_name = folder.name + if slug_name in handled: + return None + if not is_complete(folder): + print(f"[handoff] skip (in progress): {slug_name}") + return None + meta = {} + mp = METADATA / slug_name / "metadata.json" + if mp.exists(): + try: + meta = json.loads(mp.read_text()) + except Exception: + pass + art = sorted(p for p in (METADATA / slug_name).glob("*") if p.suffix.lower() in ART_EXTS) \ + if (METADATA / slug_name).exists() else [] + + kind, platform, primary = classify(folder) + guides = collect_guides(folder, primary) + note = render_note(meta, kind, platform, primary, guides, art, folder) + + print(f"[handoff] {slug_name}: kind={kind} platform={platform} " + f"artifact={primary.name if primary else 'NONE'} guides={len(guides)} art={len(art)}") + + if DRY: + scratch = pathlib.Path("/tmp/handoff-preview") / slug_name + scratch.mkdir(parents=True, exist_ok=True) + (scratch / "note.md").write_text(note, encoding="utf-8") + (scratch / "handoff.json").write_text(json.dumps( + {"name": meta.get("name", slug_name), "slug": slug_name, "kind": kind, + "platform": platform, "artifact": primary.name if primary else None, + "guides": [str(g.relative_to(folder)) for g in guides], + "art": [a.name for a in art], "links": meta.get("links", [])}, indent=2), encoding="utf-8") + return None + + # Snapshot guide paths relative to the folder BEFORE moving it; the same + # relative layout holds under dest_lib after the move. + guide_rels = [str(g.relative_to(folder)) for g in guides] + dest_lib = route(folder, slug_name, kind, platform, primary) + for a in art: + shutil.copy2(a, dest_lib / a.name) + (dest_lib / "note.md").write_text(note, encoding="utf-8") + (dest_lib / "handoff.json").write_text(json.dumps( + {"name": meta.get("name", slug_name), "slug": slug_name, "kind": kind, + "platform": platform, "artifact": primary.name if primary else None, + "guides": guide_rels, "art": [a.name for a in art], + "links": meta.get("links", [])}, indent=2), encoding="utf-8") + handled.add(slug_name) + return slug_name + + +def main(): + if not INCOMING.exists(): + print(f"[handoff] no incoming dir {INCOMING}") + return + handled = load_handled() + done = 0 + for folder in sorted(p for p in INCOMING.iterdir() if p.is_dir()): + if process(folder, handled): + done += 1 + if not DRY: + save_handled(handled) + print(f"[handoff] done — {done} game(s) routed; DRY={DRY}") + + +if __name__ == "__main__": + main() diff --git a/stacks/romhacks/orchestrator/run.sh b/stacks/romhacks/orchestrator/run.sh index 1a6bb4b..415e119 100644 --- a/stacks/romhacks/orchestrator/run.sh +++ b/stacks/romhacks/orchestrator/run.sh @@ -6,6 +6,9 @@ set -u POLL_INTERVAL="${POLL_INTERVAL:-21600}" while true; do + # Route any completed downloads first (fast; independent of Discord access), + # then export + enqueue new threads. + python3 /app/handoff.py || echo "[romhacks] handoff failed (exit $?)" if [ -z "${DISCORD_TOKEN:-}" ] || [ -z "${GUILD_ID:-}" ]; then echo "[romhacks] DISCORD_TOKEN/GUILD_ID not set — idle. Sleeping ${POLL_INTERVAL}s." else diff --git a/stacks/romhacks/stack.env b/stacks/romhacks/stack.env index b081ad4..cf49464 100644 --- a/stacks/romhacks/stack.env +++ b/stacks/romhacks/stack.env @@ -17,6 +17,9 @@ GUILD_ID=1474050123347525652 # downloads. Inspect /storage1/labdata/romhacks/metadata first. # DRY_RUN=false => write .crawljob files so JDownloader starts fetching. DRY_RUN=false +# HANDOFF_DRY=false => route completed downloads into the library + RomM and +# render per-hack notes. true => classify/preview only, move nothing. +HANDOFF_DRY=false # Seconds between harvest cycles (21600 = 6h). POLL_INTERVAL=21600 # JDownloader in-container download root (maps to /storage1/labdata/romhacks/incoming).