ginnoir/homelabstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(romhacks): arm downloads (DRY_RUN=false) + fix dry-run state burn

Browse Source 
process_file now marks messages processed only after a real crawljob write, so
DRY_RUN no longer poisons the state and link-less posts get re-checked after the
host filter widens. State file reset alongside this so the arm cycle re-enqueues
all threads.
This commit is contained in:
ginnoir
2026-06-07 19:55:00 -05:00
parent 4fda9a3b3e
commit 328b5df776
2 changed files with 11 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
stacks/romhacks/orchestrator/orchestrate.py
+10 -5
View File
@@ -142,7 +142,7 @@ def process_file(path, hosts, processed):
msgs = data.get("messages", []) msgs = data.get("messages", [])
description = next((m.get("content", "") for m in msgs if (m.get("content") or "").strip()), "") description = next((m.get("content", "") for m in msgs if (m.get("content") or "").strip()), "")
links, art, new = [], [], 0 links, art, fresh_mids = [], [], []
for m in msgs: for m in msgs:
mid = m.get("id") mid = m.get("id")
if not mid or mid in processed: if not mid or mid in processed:
@@ -165,18 +165,23 @@ def process_file(path, hosts, processed):
local = pathlib.Path(path).parent / att.get("url", "") local = pathlib.Path(path).parent / att.get("url", "")
if local.exists(): if local.exists():
art.append(local) art.append(local)
processed.add(mid) fresh_mids.append(mid)
new += 1
links = list(dict.fromkeys(links)) links = list(dict.fromkeys(links))
enqueued = 0
if links: if links:
if DRY_RUN: if DRY_RUN:
print(f"[romhacks][DRY_RUN] would enqueue {len(links)} link(s) for '{game}'") print(f"[romhacks][DRY_RUN] would enqueue {len(links)} link(s) for '{game}'")
else: else:
write_crawljob(game, links) write_crawljob(game, links)
# Mark this thread's new messages processed ONLY after a real enqueue.
# This keeps DRY_RUN from burning the state, and leaves link-less posts
# un-marked so a later host-filter widening re-checks them.
processed.update(fresh_mids)
enqueued = len(fresh_mids)
if links or art: if links or art:
stage_metadata(game, description, links, art) stage_metadata(game, description, links, art)
return new return enqueued
def main(): def main():
@@ -199,7 +204,7 @@ def main():
for jf in EXPORTS.glob("**/*.json"): for jf in EXPORTS.glob("**/*.json"):
total_new += process_file(jf, hosts, processed) total_new += process_file(jf, hosts, processed)
save_state(processed) save_state(processed)
print(f"[romhacks] cycle done — {total_new} new message(s); DRY_RUN={DRY_RUN}") print(f"[romhacks] cycle done — {total_new} message(s) enqueued; DRY_RUN={DRY_RUN}")
if __name__ == "__main__": if __name__ == "__main__":
stacks/romhacks/stack.env
+1 -1
View File
@@ -16,7 +16,7 @@ GUILD_ID=1474050123347525652
# DRY_RUN=true => export, parse, and stage metadata, but DO NOT enqueue any # DRY_RUN=true => export, parse, and stage metadata, but DO NOT enqueue any
# downloads. Inspect /storage1/labdata/romhacks/metadata first. # downloads. Inspect /storage1/labdata/romhacks/metadata first.
# DRY_RUN=false => write .crawljob files so JDownloader starts fetching. # DRY_RUN=false => write .crawljob files so JDownloader starts fetching.
DRY_RUN=true DRY_RUN=false
# Seconds between harvest cycles (21600 = 6h). # Seconds between harvest cycles (21600 = 6h).
POLL_INTERVAL=21600 POLL_INTERVAL=21600
# JDownloader in-container download root (maps to /storage1/labdata/romhacks/incoming). # JDownloader in-container download root (maps to /storage1/labdata/romhacks/incoming).