ginnoir
328b5df776
process_file now marks messages processed only after a real crawljob write, so DRY_RUN no longer poisons the state and link-less posts get re-checked after the host filter widens. State file reset alongside this so the arm cycle re-enqueues all threads.
212 lines
7.1 KiB
Python
212 lines
7.1 KiB
Python
#!/usr/bin/env python3
|
|
"""romhacks orchestrator.
|
|
|
|
One cycle:
|
|
1. For each configured channel, run DiscordChatExporter (DCE) to dump the
|
|
channel + all its threads to JSON, downloading inline media (box art).
|
|
2. Parse every exported JSON. For each NOT-yet-seen message, pull download
|
|
links (filtered to known file hosts) and any image attachments.
|
|
3. Stage thread name + blurb + art + links as metadata (for a later RomM
|
|
custom-cover hand-off), and — unless DRY_RUN — write a JDownloader
|
|
.crawljob so JD fetches the links.
|
|
|
|
Idempotent: the full thread JSON is re-parsed each cycle; a message-id state file
|
|
ensures each post is only enqueued once. INERT guards live in run.sh (token/guild)
|
|
and here (DRY_RUN).
|
|
"""
|
|
import glob
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
TOKEN = os.environ.get("DISCORD_TOKEN", "").strip()
|
|
DRY_RUN = os.environ.get("DRY_RUN", "true").lower() != "false"
|
|
JD_DOWNLOAD_ROOT = os.environ.get("JD_DOWNLOAD_ROOT", "/output")
|
|
|
|
# Paths default to the in-container mounts; overridable via env for local testing.
|
|
EXPORTS = pathlib.Path(os.environ.get("EXPORTS_DIR", "/exports"))
|
|
STATE = pathlib.Path(os.environ.get("STATE_FILE", "/state/processed.json"))
|
|
CRAWLJOBS = pathlib.Path(os.environ.get("CRAWLJOBS_DIR", "/crawljobs"))
|
|
METADATA = pathlib.Path(os.environ.get("METADATA_DIR", "/metadata"))
|
|
CHANNELS_FILE = pathlib.Path(os.environ.get("CHANNELS_FILE", "/app/channels.json"))
|
|
|
|
# Fallback host list; overridden by channels.json -> "hosts".
|
|
DEFAULT_HOSTS = [
|
|
"mega.nz", "mediafire.com", "drive.google.com", "pixeldrain.com",
|
|
"gofile.io", "1fichier.com", "workupload.com", "krakenfiles.com",
|
|
"bunkr", "anonfiles",
|
|
]
|
|
URL_RE = re.compile(r"https?://[^\s<>()\[\]]+", re.I)
|
|
IMG_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".gif")
|
|
|
|
|
|
def load_json(p):
|
|
try:
|
|
return json.loads(pathlib.Path(p).read_text(encoding="utf-8"))
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def slug(s):
|
|
return re.sub(r"[^A-Za-z0-9._-]+", "_", s or "").strip("_")[:120] or "unknown"
|
|
|
|
|
|
def find_dce():
|
|
# Self-contained single-file executable in the Alpine DCE image (no `dotnet`).
|
|
direct = "/opt/app/DiscordChatExporter.Cli"
|
|
if os.path.exists(direct):
|
|
return direct
|
|
hits = glob.glob("/opt/**/DiscordChatExporter.Cli", recursive=True)
|
|
return hits[0] if hits else None
|
|
|
|
|
|
def run_dce(channel_id):
|
|
dce = find_dce()
|
|
if not dce:
|
|
print("[romhacks] DiscordChatExporter.Cli not found in image", file=sys.stderr)
|
|
return False
|
|
out = EXPORTS / channel_id
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
cmd = [
|
|
dce, "export",
|
|
"-t", TOKEN,
|
|
"-c", channel_id,
|
|
"-f", "Json",
|
|
"--include-threads", "all",
|
|
"--media", "True",
|
|
"--reuse-media", "True",
|
|
"-o", str(out / "%c.json"),
|
|
]
|
|
print(f"[romhacks] exporting channel {channel_id}")
|
|
return subprocess.run(cmd).returncode == 0
|
|
|
|
|
|
def load_state():
|
|
return set((load_json(STATE) or {}).get("processed", []))
|
|
|
|
|
|
def save_state(processed):
|
|
STATE.parent.mkdir(parents=True, exist_ok=True)
|
|
STATE.write_text(json.dumps({"processed": sorted(processed)}), encoding="utf-8")
|
|
|
|
|
|
def host_match(url, hosts):
|
|
u = url.lower()
|
|
return any(h in u for h in hosts)
|
|
|
|
|
|
def write_crawljob(game, links):
|
|
CRAWLJOBS.mkdir(parents=True, exist_ok=True)
|
|
job = [{
|
|
"text": "\n".join(links),
|
|
"packageName": game,
|
|
"downloadFolder": f"{JD_DOWNLOAD_ROOT}/{slug(game)}",
|
|
"enabled": "TRUE",
|
|
"autoStart": "TRUE",
|
|
"autoConfirm": "TRUE",
|
|
}]
|
|
digest = hashlib.sha1("|".join(links).encode()).hexdigest()[:8]
|
|
f = CRAWLJOBS / f"{slug(game)}-{digest}.crawljob"
|
|
f.write_text(json.dumps(job, indent=2), encoding="utf-8")
|
|
print(f"[romhacks] enqueued {len(links)} link(s) for '{game}' -> {f.name}")
|
|
|
|
|
|
def stage_metadata(game, description, links, art_files):
|
|
d = METADATA / slug(game)
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
meta = {
|
|
"name": game,
|
|
"description": description,
|
|
"links": links,
|
|
"art": [a.name for a in art_files],
|
|
}
|
|
(d / "metadata.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
|
|
for a in art_files:
|
|
try:
|
|
shutil.copy2(a, d / a.name)
|
|
except Exception as e:
|
|
print(f"[romhacks] art copy failed {a}: {e}", file=sys.stderr)
|
|
|
|
|
|
def process_file(path, hosts, processed):
|
|
data = load_json(path)
|
|
if not data:
|
|
return 0
|
|
channel = data.get("channel", {})
|
|
game = channel.get("name") or "unknown"
|
|
msgs = data.get("messages", [])
|
|
description = next((m.get("content", "") for m in msgs if (m.get("content") or "").strip()), "")
|
|
|
|
links, art, fresh_mids = [], [], []
|
|
for m in msgs:
|
|
mid = m.get("id")
|
|
if not mid or mid in processed:
|
|
continue
|
|
# Links usually live in the message body as markdown links, but some posts
|
|
# only surface them via an auto-embed — scan both.
|
|
blobs = [m.get("content") or ""]
|
|
for emb in m.get("embeds", []):
|
|
blobs.append(emb.get("url") or "")
|
|
blobs.append(emb.get("description") or "")
|
|
for fld in emb.get("fields", []):
|
|
blobs.append(fld.get("value") or "")
|
|
for blob in blobs:
|
|
for u in URL_RE.findall(blob):
|
|
if host_match(u, hosts):
|
|
links.append(u)
|
|
for att in m.get("attachments", []):
|
|
if att.get("fileName", "").lower().endswith(IMG_EXTS):
|
|
# DCE --media rewrites attachment "url" to a path relative to the export
|
|
local = pathlib.Path(path).parent / att.get("url", "")
|
|
if local.exists():
|
|
art.append(local)
|
|
fresh_mids.append(mid)
|
|
|
|
links = list(dict.fromkeys(links))
|
|
enqueued = 0
|
|
if links:
|
|
if DRY_RUN:
|
|
print(f"[romhacks][DRY_RUN] would enqueue {len(links)} link(s) for '{game}'")
|
|
else:
|
|
write_crawljob(game, links)
|
|
# Mark this thread's new messages processed ONLY after a real enqueue.
|
|
# This keeps DRY_RUN from burning the state, and leaves link-less posts
|
|
# un-marked so a later host-filter widening re-checks them.
|
|
processed.update(fresh_mids)
|
|
enqueued = len(fresh_mids)
|
|
if links or art:
|
|
stage_metadata(game, description, links, art)
|
|
return enqueued
|
|
|
|
|
|
def main():
|
|
cfg = load_json(CHANNELS_FILE) or {}
|
|
channels = cfg.get("channels", [])
|
|
hosts = cfg.get("hosts", DEFAULT_HOSTS)
|
|
if not channels:
|
|
print("[romhacks] no channels configured in channels.json — nothing to do")
|
|
return
|
|
|
|
processed = load_state()
|
|
for ch in channels:
|
|
cid = str(ch.get("id", "")).strip()
|
|
if not cid:
|
|
continue
|
|
if not run_dce(cid):
|
|
print(f"[romhacks] export failed for {cid}, skipping", file=sys.stderr)
|
|
|
|
total_new = 0
|
|
for jf in EXPORTS.glob("**/*.json"):
|
|
total_new += process_file(jf, hosts, processed)
|
|
save_state(processed)
|
|
print(f"[romhacks] cycle done — {total_new} message(s) enqueued; DRY_RUN={DRY_RUN}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|