feat(romhacks): add Discord rom-hack ingestion stack (DCE -> JDownloader -> RomM)
Inert by default: empty token keeps the orchestrator idle and DRY_RUN=true suppresses downloads. Orchestrator exports per-generation forum threads via DiscordChatExporter, extracts download links + art, stages metadata, and writes JDownloader crawljobs. Armed here with an alt account token and all 11 romhack forums, still in DRY_RUN pending first-cycle validation.
This commit is contained in:
ginnoir
@@ -0,0 +1,2 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
@@ -0,0 +1,15 @@
|
||||
# Orchestrator = DiscordChatExporter (has .NET + the DCE CLI) + Python for parsing.
|
||||
# Baking scripts + channels.json into the image avoids Portainer's relative
|
||||
# FILE-bind quirk (see docker-compose.yml header).
|
||||
FROM tyrrrz/discordchatexporter:stable
|
||||
|
||||
USER root
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends python3 python3-requests \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
COPY orchestrate.py run.sh channels.json ./
|
||||
RUN chmod +x run.sh
|
||||
|
||||
ENTRYPOINT ["./run.sh"]
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"_comment": "Forum channels to harvest (type 15 — each thread is one rom-hack). DCE's --include-threads all captures the threads beneath each forum. 'platform' is a hint for the later RomM hand-off and is not used yet. Editing this file requires a push + Portainer rebuild (it is baked into the orchestrator image). Cheat-code forums are intentionally omitted.",
|
||||
"hosts": [
|
||||
"mega.nz",
|
||||
"mediafire.com",
|
||||
"drive.google.com",
|
||||
"pixeldrain.com",
|
||||
"gofile.io",
|
||||
"1fichier.com",
|
||||
"workupload.com",
|
||||
"krakenfiles.com"
|
||||
],
|
||||
"channels": [
|
||||
{ "id": "1474174360926290105", "label": "gen-1-romhacks", "platform": "gb" },
|
||||
{ "id": "1474174532196761802", "label": "gen-2-romhacks", "platform": "gbc" },
|
||||
{ "id": "1474174608587489416", "label": "gen-3-romhacks", "platform": "gba" },
|
||||
{ "id": "1474174672210755714", "label": "gen-4-romhacks", "platform": "nds" },
|
||||
{ "id": "1474174736438132756", "label": "gen-5-romhacks", "platform": "nds" },
|
||||
{ "id": "1474174789680632084", "label": "gen-6-romhacks", "platform": "3ds" },
|
||||
{ "id": "1474174859671240898", "label": "gen-7-romhacks", "platform": "3ds" },
|
||||
{ "id": "1474174950138183880", "label": "gen-8-romhacks", "platform": "switch" },
|
||||
{ "id": "1474175004248903935", "label": "gen-9-romhacks", "platform": "switch" },
|
||||
{ "id": "1474175244209098986", "label": "joiplay-rpgxp-games", "platform": "pc" },
|
||||
{ "id": "1475605095805747351", "label": "unique-romhacks", "platform": "" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
"""romhacks orchestrator.
|
||||
|
||||
One cycle:
|
||||
1. For each configured channel, run DiscordChatExporter (DCE) to dump the
|
||||
channel + all its threads to JSON, downloading inline media (box art).
|
||||
2. Parse every exported JSON. For each NOT-yet-seen message, pull download
|
||||
links (filtered to known file hosts) and any image attachments.
|
||||
3. Stage thread name + blurb + art + links as metadata (for a later RomM
|
||||
custom-cover hand-off), and — unless DRY_RUN — write a JDownloader
|
||||
.crawljob so JD fetches the links.
|
||||
|
||||
Idempotent: the full thread JSON is re-parsed each cycle; a message-id state file
|
||||
ensures each post is only enqueued once. INERT guards live in run.sh (token/guild)
|
||||
and here (DRY_RUN).
|
||||
"""
|
||||
import glob
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
TOKEN = os.environ.get("DISCORD_TOKEN", "").strip()
|
||||
DRY_RUN = os.environ.get("DRY_RUN", "true").lower() != "false"
|
||||
JD_DOWNLOAD_ROOT = os.environ.get("JD_DOWNLOAD_ROOT", "/output")
|
||||
|
||||
# Paths default to the in-container mounts; overridable via env for local testing.
|
||||
EXPORTS = pathlib.Path(os.environ.get("EXPORTS_DIR", "/exports"))
|
||||
STATE = pathlib.Path(os.environ.get("STATE_FILE", "/state/processed.json"))
|
||||
CRAWLJOBS = pathlib.Path(os.environ.get("CRAWLJOBS_DIR", "/crawljobs"))
|
||||
METADATA = pathlib.Path(os.environ.get("METADATA_DIR", "/metadata"))
|
||||
CHANNELS_FILE = pathlib.Path(os.environ.get("CHANNELS_FILE", "/app/channels.json"))
|
||||
|
||||
# Fallback host list; overridden by channels.json -> "hosts".
|
||||
DEFAULT_HOSTS = [
|
||||
"mega.nz", "mediafire.com", "drive.google.com", "pixeldrain.com",
|
||||
"gofile.io", "1fichier.com", "workupload.com", "krakenfiles.com",
|
||||
"bunkr", "anonfiles",
|
||||
]
|
||||
URL_RE = re.compile(r"https?://[^\s<>()\[\]]+", re.I)
|
||||
IMG_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".gif")
|
||||
|
||||
|
||||
def load_json(p):
|
||||
try:
|
||||
return json.loads(pathlib.Path(p).read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def slug(s):
|
||||
return re.sub(r"[^A-Za-z0-9._-]+", "_", s or "").strip("_")[:120] or "unknown"
|
||||
|
||||
|
||||
def find_dce():
|
||||
direct = "/opt/app/DiscordChatExporter.Cli.dll"
|
||||
if os.path.exists(direct):
|
||||
return direct
|
||||
hits = glob.glob("/opt/**/DiscordChatExporter.Cli.dll", recursive=True)
|
||||
return hits[0] if hits else None
|
||||
|
||||
|
||||
def run_dce(channel_id):
|
||||
dll = find_dce()
|
||||
if not dll:
|
||||
print("[romhacks] DiscordChatExporter.Cli.dll not found in image", file=sys.stderr)
|
||||
return False
|
||||
out = EXPORTS / channel_id
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
cmd = [
|
||||
"dotnet", dll, "export",
|
||||
"-t", TOKEN,
|
||||
"-c", channel_id,
|
||||
"-f", "Json",
|
||||
"--include-threads", "all",
|
||||
"--media", "True",
|
||||
"--reuse-media", "True",
|
||||
"-o", str(out / "%c.json"),
|
||||
]
|
||||
print(f"[romhacks] exporting channel {channel_id}")
|
||||
return subprocess.run(cmd).returncode == 0
|
||||
|
||||
|
||||
def load_state():
|
||||
return set((load_json(STATE) or {}).get("processed", []))
|
||||
|
||||
|
||||
def save_state(processed):
|
||||
STATE.parent.mkdir(parents=True, exist_ok=True)
|
||||
STATE.write_text(json.dumps({"processed": sorted(processed)}), encoding="utf-8")
|
||||
|
||||
|
||||
def host_match(url, hosts):
|
||||
u = url.lower()
|
||||
return any(h in u for h in hosts)
|
||||
|
||||
|
||||
def write_crawljob(game, links):
|
||||
CRAWLJOBS.mkdir(parents=True, exist_ok=True)
|
||||
job = [{
|
||||
"text": "\n".join(links),
|
||||
"packageName": game,
|
||||
"downloadFolder": f"{JD_DOWNLOAD_ROOT}/{slug(game)}",
|
||||
"enabled": "TRUE",
|
||||
"autoStart": "TRUE",
|
||||
"autoConfirm": "TRUE",
|
||||
}]
|
||||
digest = hashlib.sha1("|".join(links).encode()).hexdigest()[:8]
|
||||
f = CRAWLJOBS / f"{slug(game)}-{digest}.crawljob"
|
||||
f.write_text(json.dumps(job, indent=2), encoding="utf-8")
|
||||
print(f"[romhacks] enqueued {len(links)} link(s) for '{game}' -> {f.name}")
|
||||
|
||||
|
||||
def stage_metadata(game, description, links, art_files):
|
||||
d = METADATA / slug(game)
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
meta = {
|
||||
"name": game,
|
||||
"description": description,
|
||||
"links": links,
|
||||
"art": [a.name for a in art_files],
|
||||
}
|
||||
(d / "metadata.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
|
||||
for a in art_files:
|
||||
try:
|
||||
shutil.copy2(a, d / a.name)
|
||||
except Exception as e:
|
||||
print(f"[romhacks] art copy failed {a}: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
def process_file(path, hosts, processed):
|
||||
data = load_json(path)
|
||||
if not data:
|
||||
return 0
|
||||
channel = data.get("channel", {})
|
||||
game = channel.get("name") or "unknown"
|
||||
msgs = data.get("messages", [])
|
||||
description = next((m.get("content", "") for m in msgs if (m.get("content") or "").strip()), "")
|
||||
|
||||
links, art, new = [], [], 0
|
||||
for m in msgs:
|
||||
mid = m.get("id")
|
||||
if not mid or mid in processed:
|
||||
continue
|
||||
# Links usually live in the message body as markdown links, but some posts
|
||||
# only surface them via an auto-embed — scan both.
|
||||
blobs = [m.get("content") or ""]
|
||||
for emb in m.get("embeds", []):
|
||||
blobs.append(emb.get("url") or "")
|
||||
blobs.append(emb.get("description") or "")
|
||||
for fld in emb.get("fields", []):
|
||||
blobs.append(fld.get("value") or "")
|
||||
for blob in blobs:
|
||||
for u in URL_RE.findall(blob):
|
||||
if host_match(u, hosts):
|
||||
links.append(u)
|
||||
for att in m.get("attachments", []):
|
||||
if att.get("fileName", "").lower().endswith(IMG_EXTS):
|
||||
# DCE --media rewrites attachment "url" to a path relative to the export
|
||||
local = pathlib.Path(path).parent / att.get("url", "")
|
||||
if local.exists():
|
||||
art.append(local)
|
||||
processed.add(mid)
|
||||
new += 1
|
||||
|
||||
links = list(dict.fromkeys(links))
|
||||
if links:
|
||||
if DRY_RUN:
|
||||
print(f"[romhacks][DRY_RUN] would enqueue {len(links)} link(s) for '{game}'")
|
||||
else:
|
||||
write_crawljob(game, links)
|
||||
if links or art:
|
||||
stage_metadata(game, description, links, art)
|
||||
return new
|
||||
|
||||
|
||||
def main():
|
||||
cfg = load_json(CHANNELS_FILE) or {}
|
||||
channels = cfg.get("channels", [])
|
||||
hosts = cfg.get("hosts", DEFAULT_HOSTS)
|
||||
if not channels:
|
||||
print("[romhacks] no channels configured in channels.json — nothing to do")
|
||||
return
|
||||
|
||||
processed = load_state()
|
||||
for ch in channels:
|
||||
cid = str(ch.get("id", "")).strip()
|
||||
if not cid:
|
||||
continue
|
||||
if not run_dce(cid):
|
||||
print(f"[romhacks] export failed for {cid}, skipping", file=sys.stderr)
|
||||
|
||||
total_new = 0
|
||||
for jf in EXPORTS.glob("**/*.json"):
|
||||
total_new += process_file(jf, hosts, processed)
|
||||
save_state(processed)
|
||||
print(f"[romhacks] cycle done — {total_new} new message(s); DRY_RUN={DRY_RUN}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
# Harvest loop. Stays idle (no export, no enqueue) until the alt-account token and
|
||||
# guild id are configured, so deploying the stack unconfigured is harmless.
|
||||
set -u
|
||||
|
||||
POLL_INTERVAL="${POLL_INTERVAL:-21600}"
|
||||
|
||||
while true; do
|
||||
if [ -z "${DISCORD_TOKEN:-}" ] || [ -z "${GUILD_ID:-}" ]; then
|
||||
echo "[romhacks] DISCORD_TOKEN/GUILD_ID not set — idle. Sleeping ${POLL_INTERVAL}s."
|
||||
else
|
||||
python3 /app/orchestrate.py || echo "[romhacks] cycle failed (exit $?)"
|
||||
fi
|
||||
sleep "${POLL_INTERVAL}"
|
||||
done
|
||||
Reference in New Issue
Block a user