#!/usr/bin/env python3 """romhacks orchestrator. One cycle: 1. For each configured channel, run DiscordChatExporter (DCE) to dump the channel + all its threads to JSON, downloading inline media (box art). 2. Parse every exported JSON. For each NOT-yet-seen message, pull download links (filtered to known file hosts) and any image attachments. 3. Stage thread name + blurb + art + links as metadata (for a later RomM custom-cover hand-off), and — unless DRY_RUN — write a JDownloader .crawljob so JD fetches the links. Idempotent: the full thread JSON is re-parsed each cycle; a message-id state file ensures each post is only enqueued once. INERT guards live in run.sh (token/guild) and here (DRY_RUN). """ import glob import hashlib import json import os import pathlib import re import shutil import subprocess import sys TOKEN = os.environ.get("DISCORD_TOKEN", "").strip() DRY_RUN = os.environ.get("DRY_RUN", "true").lower() != "false" JD_DOWNLOAD_ROOT = os.environ.get("JD_DOWNLOAD_ROOT", "/output") # Paths default to the in-container mounts; overridable via env for local testing. EXPORTS = pathlib.Path(os.environ.get("EXPORTS_DIR", "/exports")) STATE = pathlib.Path(os.environ.get("STATE_FILE", "/state/processed.json")) CRAWLJOBS = pathlib.Path(os.environ.get("CRAWLJOBS_DIR", "/crawljobs")) METADATA = pathlib.Path(os.environ.get("METADATA_DIR", "/metadata")) CHANNELS_FILE = pathlib.Path(os.environ.get("CHANNELS_FILE", "/app/channels.json")) # Fallback host list; overridden by channels.json -> "hosts". DEFAULT_HOSTS = [ "mega.nz", "mediafire.com", "drive.google.com", "pixeldrain.com", "gofile.io", "1fichier.com", "workupload.com", "krakenfiles.com", "bunkr", "anonfiles", ] URL_RE = re.compile(r"https?://[^\s<>()\[\]]+", re.I) IMG_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".gif") def load_json(p): try: return json.loads(pathlib.Path(p).read_text(encoding="utf-8")) except Exception: return None def slug(s): return re.sub(r"[^A-Za-z0-9._-]+", "_", s or "").strip("_")[:120] or "unknown" def find_dce(): # Self-contained single-file executable in the Alpine DCE image (no `dotnet`). direct = "/opt/app/DiscordChatExporter.Cli" if os.path.exists(direct): return direct hits = glob.glob("/opt/**/DiscordChatExporter.Cli", recursive=True) return hits[0] if hits else None def run_dce(channel_id): dce = find_dce() if not dce: print("[romhacks] DiscordChatExporter.Cli not found in image", file=sys.stderr) return False out = EXPORTS / channel_id out.mkdir(parents=True, exist_ok=True) cmd = [ dce, "export", "-t", TOKEN, "-c", channel_id, "-f", "Json", "--include-threads", "all", "--media", "True", "--reuse-media", "True", "-o", str(out / "%c.json"), ] print(f"[romhacks] exporting channel {channel_id}") return subprocess.run(cmd).returncode == 0 def load_state(): return set((load_json(STATE) or {}).get("processed", [])) def save_state(processed): STATE.parent.mkdir(parents=True, exist_ok=True) STATE.write_text(json.dumps({"processed": sorted(processed)}), encoding="utf-8") def host_match(url, hosts): u = url.lower() return any(h in u for h in hosts) def write_crawljob(game, links): CRAWLJOBS.mkdir(parents=True, exist_ok=True) job = [{ "text": "\n".join(links), "packageName": game, "downloadFolder": f"{JD_DOWNLOAD_ROOT}/{slug(game)}", "enabled": "TRUE", "autoStart": "TRUE", "autoConfirm": "TRUE", }] digest = hashlib.sha1("|".join(links).encode()).hexdigest()[:8] f = CRAWLJOBS / f"{slug(game)}-{digest}.crawljob" f.write_text(json.dumps(job, indent=2), encoding="utf-8") print(f"[romhacks] enqueued {len(links)} link(s) for '{game}' -> {f.name}") def stage_metadata(game, description, links, art_files): d = METADATA / slug(game) d.mkdir(parents=True, exist_ok=True) meta = { "name": game, "description": description, "links": links, "art": [a.name for a in art_files], } (d / "metadata.json").write_text(json.dumps(meta, indent=2), encoding="utf-8") for a in art_files: try: shutil.copy2(a, d / a.name) except Exception as e: print(f"[romhacks] art copy failed {a}: {e}", file=sys.stderr) def process_file(path, hosts, processed): data = load_json(path) if not data: return 0 channel = data.get("channel", {}) game = channel.get("name") or "unknown" msgs = data.get("messages", []) description = next((m.get("content", "") for m in msgs if (m.get("content") or "").strip()), "") links, art, new = [], [], 0 for m in msgs: mid = m.get("id") if not mid or mid in processed: continue # Links usually live in the message body as markdown links, but some posts # only surface them via an auto-embed — scan both. blobs = [m.get("content") or ""] for emb in m.get("embeds", []): blobs.append(emb.get("url") or "") blobs.append(emb.get("description") or "") for fld in emb.get("fields", []): blobs.append(fld.get("value") or "") for blob in blobs: for u in URL_RE.findall(blob): if host_match(u, hosts): links.append(u) for att in m.get("attachments", []): if att.get("fileName", "").lower().endswith(IMG_EXTS): # DCE --media rewrites attachment "url" to a path relative to the export local = pathlib.Path(path).parent / att.get("url", "") if local.exists(): art.append(local) processed.add(mid) new += 1 links = list(dict.fromkeys(links)) if links: if DRY_RUN: print(f"[romhacks][DRY_RUN] would enqueue {len(links)} link(s) for '{game}'") else: write_crawljob(game, links) if links or art: stage_metadata(game, description, links, art) return new def main(): cfg = load_json(CHANNELS_FILE) or {} channels = cfg.get("channels", []) hosts = cfg.get("hosts", DEFAULT_HOSTS) if not channels: print("[romhacks] no channels configured in channels.json — nothing to do") return processed = load_state() for ch in channels: cid = str(ch.get("id", "")).strip() if not cid: continue if not run_dce(cid): print(f"[romhacks] export failed for {cid}, skipping", file=sys.stderr) total_new = 0 for jf in EXPORTS.glob("**/*.json"): total_new += process_file(jf, hosts, processed) save_state(processed) print(f"[romhacks] cycle done — {total_new} new message(s); DRY_RUN={DRY_RUN}") if __name__ == "__main__": main()