diff --git a/stacks/romhacks/orchestrator/orchestrate.py b/stacks/romhacks/orchestrator/orchestrate.py index f30f557..b1fb59a 100644 --- a/stacks/romhacks/orchestrator/orchestrate.py +++ b/stacks/romhacks/orchestrator/orchestrate.py @@ -142,7 +142,7 @@ def process_file(path, hosts, processed): msgs = data.get("messages", []) description = next((m.get("content", "") for m in msgs if (m.get("content") or "").strip()), "") - links, art, new = [], [], 0 + links, art, fresh_mids = [], [], [] for m in msgs: mid = m.get("id") if not mid or mid in processed: @@ -165,18 +165,23 @@ def process_file(path, hosts, processed): local = pathlib.Path(path).parent / att.get("url", "") if local.exists(): art.append(local) - processed.add(mid) - new += 1 + fresh_mids.append(mid) links = list(dict.fromkeys(links)) + enqueued = 0 if links: if DRY_RUN: print(f"[romhacks][DRY_RUN] would enqueue {len(links)} link(s) for '{game}'") else: write_crawljob(game, links) + # Mark this thread's new messages processed ONLY after a real enqueue. + # This keeps DRY_RUN from burning the state, and leaves link-less posts + # un-marked so a later host-filter widening re-checks them. + processed.update(fresh_mids) + enqueued = len(fresh_mids) if links or art: stage_metadata(game, description, links, art) - return new + return enqueued def main(): @@ -199,7 +204,7 @@ def main(): for jf in EXPORTS.glob("**/*.json"): total_new += process_file(jf, hosts, processed) save_state(processed) - print(f"[romhacks] cycle done — {total_new} new message(s); DRY_RUN={DRY_RUN}") + print(f"[romhacks] cycle done — {total_new} message(s) enqueued; DRY_RUN={DRY_RUN}") if __name__ == "__main__": diff --git a/stacks/romhacks/stack.env b/stacks/romhacks/stack.env index 69ab9ed..b081ad4 100644 --- a/stacks/romhacks/stack.env +++ b/stacks/romhacks/stack.env @@ -16,7 +16,7 @@ GUILD_ID=1474050123347525652 # DRY_RUN=true => export, parse, and stage metadata, but DO NOT enqueue any # downloads. Inspect /storage1/labdata/romhacks/metadata first. # DRY_RUN=false => write .crawljob files so JDownloader starts fetching. -DRY_RUN=true +DRY_RUN=false # Seconds between harvest cycles (21600 = 6h). POLL_INTERVAL=21600 # JDownloader in-container download root (maps to /storage1/labdata/romhacks/incoming).