pokemon/scripts/list-acquire-from-urls.py

#!/usr/bin/env python3
"""Diff URL-list hacks against placed library + attempt direct downloads."""
import json, os, re, shutil, subprocess, sys, tempfile, unicodedata, urllib.request
from pathlib import Path

CATALOG = Path(__file__).resolve().parents[1] / "wiki" / "catalog.json"
INCOMING = "/storage1/igir/romhacks/incoming/list-import"
UA = "Mozilla/5.0 (homelab personal archival)"

# Unique games from the 7 source lists with best-known download URL (None = manual only).
ACQUIRE = [
    # --- direct-download candidates ---
    ("Coral", "rom", "https://raw.githubusercontent.com/vbaemulator/GBC-Roms/main/Pokemon%20Coral%20Version%20v1.2.zip", "pokecommunity"),
    ("Poketale", "zip", "https://www.mediafire.com/file/n6yzxnclf2uys7c/Poketale.zip/file", "mediafire"),
    ("Vanguard", "archive", "https://www.mediafire.com/file/w9gy3qldi29imhk/Vanguard+3.0.12.7z/file", "mediafire"),
    ("Bizarre", "zip", "https://www.mediafire.com/file/lox59f4nubxflw6/Pokemon_Bizarre_ENGLISH.zip/file", "mediafire"),
    ("Spades and Clubs", "page", "https://senta-storage-system.neocities.org/main", "neocities"),
    ("Dreamstone Mysteries", "github", "https://github.com/dsmyst/dreamstone-mysteries", "github"),
    ("Giratina Strikes Back", "staging", None, "pokecommunity"),
    # --- catalog-only / gated (reported, not fetched) ---
    ("Abstract", "manual", "https://www.pokecommunity.com/threads/pokémon-abstract-version.527324/", "pokecommunity"),
    ("Alchemist", "manual", "https://www.pokecommunity.com/threads/pokémon-alchemist.378280/", "pokecommunity"),
    ("Awakening", "manual", "https://mega.nz/221fb646-0a04-4316-a163-f9b532c89197", "mega"),
    ("AfricanVs", "have", None, "mega"),
    ("Bushido", "have", None, "eeveeexpo"),
    ("Consonancia", "manual", "https://www.pokecommunity.com/threads/pokémon-consonancia-full-game.537329/", "pokecommunity"),
    ("Concealed", "manual", "https://www.pokecommunity.com/threads/concealed.527325/", "pokecommunity"),
    ("Crown", "manual", "https://www.pokecommunity.com/threads/pokémon-crown.463821", "pokecommunity"),
    ("Decay", "have", None, "eeveeexpo"),
    ("Deserted", "have", None, "eeveeexpo"),
    ("Eon Guardians", "manual", "https://www.pokecommunity.com/threads/pokémon-eon-guardians.527913/", "pokecommunity"),
    ("Fire Ash", "manual", "https://www.pokecommunity.com/threads/pokémon-fire-ash.399096/", "pokecommunity"),
    ("Floral Tempus", "manual", "https://www.pokecommunity.com/threads/pokémon-floral-tempus-ex-1-9-4-released-12-gym-badges-elite-4-champion.409175/", "pokecommunity"),
    ("Hero Legacy", "manual", "https://www.pokecommunity.com/threads/pokémon-hero-legacy-eng-de-now-with-new-game-mode.538518/", "pokecommunity"),
    ("HGSS Sevii", "manual", "https://www.pokecommunity.com/threads/pokémon-hgss-sevii-islands-hoenn-postgame.434636/", "pokecommunity"),
    ("Itinerant", "manual", "https://www.pokecommunity.com/threads/pokemon-itinerant.528218/", "pokecommunity"),
    ("Jam Festival", "manual", "https://www.pokecommunity.com/threads/pkmn-jam-festival.536702/", "pokecommunity"),
    ("Legends of the Arena", "manual", "https://www.pokecommunity.com/threads/pokémon-legends-of-the-arena.298738/", "pokecommunity"),
    ("Mega Adventures", "manual", "https://www.pokecommunity.com/threads/pokemon-mega-adventure.362058/", "pokecommunity"),
    ("Nightmare", "manual", "https://www.pokecommunity.com/threads/pokémon-nightmare.504856/", "pokecommunity"),
    ("Odyssey II", "discord", None, "discord"),
    ("Legacy Edition", "discord", None, "discord"),
    ("Iridium", "discord", None, "discord"),
    ("Light Platinum DS", "manual", None, "twitter"),
    ("Prisme", "manual", "https://pokemonprisme.com", "website"),
    ("Relict", "manual", "https://phiongames.blogspot.com/2025/12/descarga-pokemon-relict.html", "blogspot"),
    ("Reminiscencia", "manual", "https://mega.nz/file/5oNDGI5R#0b_7aQ3fMlUCo6Gwo8dVLXEXnW-seV-o_OanYqxVdRg", "mega"),
    ("Reloaded", "manual", "https://pokemon-reloaded.blogspot.com", "blogspot"),
    ("Shattered Light", "manual", "https://eeveeexpo.com/shattered-light/", "eeveeexpo"),
    ("Sienna", "manual", "https://www.pokecommunity.com/threads/hack-of-the-year-2010-pokémon-sienna-complete-version-released.202372/", "pokecommunity"),
    ("Tectonic", "manual", "https://www.tectonic-game.com", "website"),
    ("Uranium", "manual", "https://mega.nz/file/gh82ST6S#makZr_hsZAcMOIKSEPUwejVJ_N4OW7Mk9wEBlWIoIG0", "mega"),
    ("Vega Fairy EX", "manual", "https://www.pokecommunity.com/threads/pokemon-vega-fairy-edition-ex-minus-v1-4-balance-updates-and-skarmory-evo.475538/", "pokecommunity"),
    ("Xenoverse", "staging", None, "mega"),
    ("Zeta", "staging", None, "website"),
    ("Insurgence", "have", None, "website"),
    ("Infinite Fusion", "have", None, "pokecommunity"),
    ("Reborn", "have", None, "website"),
    ("Rejuvenation", "have", None, "website"),
    ("Pokerogue", "web", "https://pokerogue.net", "browser"),
    ("Emerald Seaglass", "manual", "https://ko-fi.com/s/4a1535f351", "ko-fi"),
    ("Lazarus", "have", "https://ko-fi.com/nemo622", "ko-fi"),
]


def norm(s):
    s = unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode().lower()
    s = re.sub(r"^pokemon\s+", "", s)
    s = re.sub(r"[^a-z0-9]+", " ", s).strip()
    return s


def load_placed():
  """Read placed library basenames from stdin (piped ssh find output)."""
  placed = set()
  for line in sys.stdin:
      line = line.strip()
      if not line:
          continue
      m = re.match(r"Pokemon - (.+?) \(Hack\)", line, re.I)
      if m:
          placed.add(norm(m.group(1)))
  return placed


def download(url, dest):
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=120) as r, open(dest, "wb") as f:
        shutil.copyfileobj(r, f)
    return os.path.getsize(dest)


def main():
    placed = load_placed()
    catalog = {norm(h.get("title", h.get("stem", ""))): h for h in json.loads(CATALOG.read_text(encoding="utf-8"))["hacks"]}
    os.makedirs(INCOMING, exist_ok=True)
    rows = []
    for name, kind, url, host in ACQUIRE:
        key = norm(name)
        in_cat = key in catalog or any(key in ck or ck in key for ck in catalog)
        in_lib = any(key in p or p in key for p in placed)
        status = "HAVE" if in_lib or kind == "have" else "MISSING"
        if kind == "staging" and not in_lib:
            status = "STAGING"
        dl = ""
        if status in ("MISSING", "STAGING") and kind in ("rom", "zip", "archive", "github", "page") and url:
            dest = os.path.join(INCOMING, re.sub(r"[^\w.\- ]", "_", name))
            try:
                if kind == "github":
                    # release asset lookup would need gh api; skip for now
                    dl = "SKIP(github-release)"
                elif kind == "page":
                    dl = "SKIP(scrape-neocities)"
                else:
                    ext = ".zip" if kind in ("rom", "zip") else ".7z"
                    dest += ext
                    sz = download(url, dest)
                    dl = f"OK {sz:,}b -> {dest}"
                    status = "DOWNLOADED"
            except Exception as e:
                dl = f"FAIL: {e}"
        rows.append((status, name, "catalog" if in_cat else "NEW", host, url or "—", dl))
    print(f"Placed library entries: {len(placed)}")
    print(f"\n{'STATUS':12} {'NAME':28} {'CAT':5} {'HOST':12} URL")
    for st, name, cat, host, url, dl in sorted(rows, key=lambda r: (r[0], r[1])):
        print(f"{st:12} {name:28} {cat:5} {host:12} {url[:60]}")
        if dl:
            print(f"{'':12}   {dl}")


if __name__ == "__main__":
    main()