diff --git a/scripts/romhack-apply.py b/scripts/romhack-apply.py new file mode 100644 index 0000000..1c078a6 --- /dev/null +++ b/scripts/romhack-apply.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Apply a Pokémon ROM-hack patch to a base ROM (IPS / BPS / UPS). + +Format is auto-detected by magic bytes. BPS embeds source+target CRC32, so a +wrong base ROM (or a buggy apply) fails loudly here rather than producing a +silently-corrupt ROM — that is the correctness guarantee for the whole pipeline. + +Usage: romhack-apply.py +Exit: 0 ok; 2 bad/mismatched input; 3 unsupported format. +xdelta patches are handled by the caller via the `xdelta3` binary, not here. +""" +import sys, zlib, struct + +def die(code, msg): + sys.stderr.write(msg + "\n"); sys.exit(code) + +def read(p): + with open(p, "rb") as f: return f.read() + +# ---------- IPS ---------- +def apply_ips(patch, base): + if patch[:5] != b"PATCH": die(2, "IPS: bad header") + out = bytearray(base) + i = 5 + while True: + rec = patch[i:i+3] + if rec == b"EOF": break + if len(rec) < 3: die(2, "IPS: truncated") + off = (rec[0] << 16) | (rec[1] << 8) | rec[2]; i += 3 + size = (patch[i] << 8) | patch[i+1]; i += 2 + if size == 0: # RLE run + rle = (patch[i] << 8) | patch[i+1]; i += 2 + val = patch[i:i+1]; i += 1 + chunk = val * rle + off2 = off + rle + else: + chunk = patch[i:i+size]; i += size + off2 = off + size + if off2 > len(out): out.extend(b"\x00" * (off2 - len(out))) + out[off:off2] = chunk + # optional truncate extension (3-byte length after EOF) + tail = patch[i+3:i+6] if patch[i:i+3] == b"EOF" else b"" + if len(tail) == 3: + newlen = (tail[0] << 16) | (tail[1] << 8) | tail[2] + out = out[:newlen] + return bytes(out) + +# ---------- UPS ---------- +def _ups_num(buf, i): + val = 0; shift = 1 + while True: + x = buf[i]; i += 1 + val += (x & 0x7f) * shift + if x & 0x80: break + shift <<= 7; val += shift + return val, i + +def apply_ups(patch, base): + if patch[:4] != b"UPS1": die(2, "UPS: bad header") + i = 4 + src_size, i = _ups_num(patch, i) + dst_size, i = _ups_num(patch, i) + if len(base) != src_size: + sys.stderr.write(f"UPS: base size {len(base)} != expected {src_size} (continuing)\n") + out = bytearray(base) + b"\x00" * max(0, dst_size - len(base)) + out = out[:dst_size] if dst_size < len(out) else out + pos = 0 + body_end = len(patch) - 12 # trailing 3 CRC32s (src,dst,patch) + while i < body_end: + rel, i = _ups_num(patch, i) + pos += rel + while True: + b = patch[i]; i += 1 + if pos < len(out): out[pos] ^= b + pos += 1 + if b == 0: break + src_crc, dst_crc, _ = struct.unpack("> 2) + 1 + if action == 0: # SourceRead + out[out_pos:out_pos+length] = base[out_pos:out_pos+length]; out_pos += length + elif action == 1: # TargetRead + out[out_pos:out_pos+length] = patch[i:i+length]; i += length; out_pos += length + elif action == 2: # SourceCopy + off, i = _ups_num(patch, i) + src_rel += (-(off >> 1) if (off & 1) else (off >> 1)) + for _ in range(length): + out[out_pos] = base[src_rel]; out_pos += 1; src_rel += 1 + else: # TargetCopy + off, i = _ups_num(patch, i) + dst_rel += (-(off >> 1) if (off & 1) else (off >> 1)) + for _ in range(length): + out[out_pos] = out[dst_rel]; out_pos += 1; dst_rel += 1 + got = zlib.crc32(bytes(out)) & 0xffffffff + if got != dst_crc: + die(2, f"BPS: output CRC {got:08x} != expected {dst_crc:08x}") + return bytes(out) + +def main(): + if len(sys.argv) != 4: + die(2, "usage: romhack-apply.py ") + patch, base_path, out_path = sys.argv[1], sys.argv[2], sys.argv[3] + pb = read(patch); base = read(base_path) + magic = pb[:5] + if magic[:4] == b"BPS1": out = apply_bps(pb, base) + elif magic[:4] == b"UPS1": out = apply_ups(pb, base) + elif magic == b"PATCH": out = apply_ips(pb, base) + else: die(3, f"unsupported patch magic {magic!r}") + with open(out_path, "wb") as f: f.write(out) + print(f"OK {out_path} ({len(out)} bytes, crc32={zlib.crc32(out)&0xffffffff:08x})") + +if __name__ == "__main__": + main() diff --git a/scripts/romhack-fetch.py b/scripts/romhack-fetch.py new file mode 100644 index 0000000..edf8953 --- /dev/null +++ b/scripts/romhack-fetch.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""Acquire, patch, validate, and place worthwhile Pokémon ROM-hacks into the +EmuDeck library at /storage1/Emulation/roms//Hacks/ on valhalla. + +Each hack is one MANIFEST entry naming its source. Two source kinds: + - "rom": a pre-patched full ROM is downloaded and placed as-is (validated). + - "patch": a patch (BPS/UPS/IPS) is downloaded and applied to a base ROM the + user already owns; BPS/UPS self-verify via embedded CRC32. + +Sources are direct, no-anti-bot URLs only (GitHub release assets, archive.org). +hackdex/Discord/Drive-gated hacks are intentionally NOT here — they are reported +as identify-only because their hosts gate or token-protect downloads. + +Output names follow the chosen scheme: "Pokemon - (Hack)." +(parallels igir's `Title (Region)` No-Intro style, with a (Hack) tag). + +Run on valhalla: python3 romhack-fetch.py [--place] [--only ] +Without --place it downloads + validates but does not touch the library. +""" +import sys, os, json, zlib, shutil, subprocess, zipfile, re, urllib.request, urllib.parse + +ROMS = "/storage1/Emulation/roms" +WORK = "/storage1/igir/romhacks/work" +PATCHES = "/storage1/igir/romhacks/patches" +APPLY = "/storage1/igir/romhacks/tools/apply.py" +UA = "Mozilla/5.0 (homelab personal archival)" + +BASES = { + "crystal_rev1": f"{ROMS}/gbc/Pokemon - Crystal Version (USA, Europe) (Rev 1).gbc", + "gold": f"{ROMS}/gbc/Pokemon - Gold Version (USA, Europe) (SGB Enhanced) (GB Compatible).gbc", + "silver": f"{ROMS}/gbc/Pokemon - Silver Version (USA, Europe) (SGB Enhanced) (GB Compatible).gbc", + "ruby": f"{ROMS}/gba/Pokemon - Ruby Version (USA, Europe).gba", + "sapphire": f"{ROMS}/gba/Pokemon - Sapphire Version (USA, Europe).gba", + "emerald": f"{ROMS}/gba/Pokemon - Emerald Version (USA, Europe).gba", + "firered": f"{ROMS}/gba/Pokemon - FireRed Version (USA, Europe).gba", + "leafgreen": f"{ROMS}/gba/Pokemon - LeafGreen Version (USA, Europe).gba", + "red": f"{ROMS}/gb/Pokemon - Red Version (USA, Europe) (SGB Enhanced).gb", + "heartgold": f"{ROMS}/nds/Pokemon HeartGold Version.nds", + "soulsilver": f"{ROMS}/nds/Pokemon SoulSilver Version.nds", +} + +def arc(idn, fn): + return f"https://archive.org/download/{idn}/{urllib.parse.quote(fn)}" + +# --- the curated, source-resolved batch (directly downloadable only) ---------- +MANIFEST = [ + # GBC + {"hack": "Polished Crystal", "platform": "gbc", "ext": "gbc", "vfmt": "gbc", + "kind": "rom", "url": "https://github.com/Rangi42/polishedcrystal/releases/download/v3.2.3/polishedcrystal-3.2.3.gbc", + "note": "v3.2.3, official GitHub release ROM (author distributes full .gbc)"}, + {"hack": "Crystal Clear", "platform": "gbc", "ext": "gbc", "vfmt": "gbc", + "kind": "patch", "base": "crystal_rev1", + "url": arc("pokemon_crystal_clear_v2.5.7", "2_5_7_Standard_1_1.bps"), + "note": "v2.5.7 Standard, BPS vs Crystal v1.1 (self-verifying)"}, + # GBA + {"hack": "Light Platinum", "platform": "gba", "ext": "gba", "vfmt": "gba", + "kind": "patch", "base": "ruby", + "url": arc("pokemon-light-platinum-u-1.1", "Pokémon_Light_Platinum (U) 1.1.ips"), + "note": "v1.1 IPS vs Ruby (USA) — IPS has no checksum, header-validated after"}, + {"hack": "Gaia", "platform": "gba", "ext": "gba", "vfmt": "gba", + "kind": "rom", "url": arc("pokemon-gaia", "GaiaB3.2.gba"), + "note": "v3.2 (final), pre-patched ROM"}, + {"hack": "Radical Red", "platform": "gba", "ext": "gba", "vfmt": "gba", + "kind": "rom", "url": arc("pokemon-radical-red-v-4.1", "Pokemon Radical Red v4.1.gba"), + "note": "v4.1, pre-patched ROM"}, + {"hack": "Inclement Emerald", "platform": "gba", "ext": "gba", "vfmt": "gba", + "kind": "rom", "url": arc("inclement-emerald_202310", "Inclement Emerald.gba"), + "note": "Oct-2023 build, pre-patched ROM"}, + {"hack": "Glazed", "platform": "gba", "ext": "gba", "vfmt": "gba", + "kind": "rom", "url": arc("pokemon-glazed", "pokemon glazed.gba"), + "note": "pre-patched ROM; version unverified (Glazed has buggy early builds)"}, + # NDS + {"hack": "Volt White", "platform": "nds", "ext": "nds", "vfmt": "nds", + "kind": "rom", "url": arc("pokemon-volt-white-v-3.1-complete", "Pokemon Volt White v3.1 - Complete.nds"), + "note": "v3.1 Complete (Drayano, base Black/White), pre-patched ROM"}, + {"hack": "Renegade Platinum", "platform": "nds", "ext": "nds", "vfmt": "nds", + "kind": "rom", "url": arc("pokemon-renegade-platinum-v-1.3.0-complete-normal-shiny", + "Pokemon Renegade Platinum v1.3.0 Complete Normal Shiny.nds"), + "note": "v1.3.0 Complete (Drayano, base Platinum), pre-patched ROM"}, +] + +# --- batch 2: more direct-downloadable hacks (some zip-wrapped) --------------- +MANIFEST2 = [ + {"hack": "Prism", "platform": "gbc", "ext": "gbc", "vfmt": "gbc", + "kind": "rom", "url": arc("pokeprism_202301", "pokeprism.gbc"), + "note": "Pokémon Prism (complete, Crystal-based), pre-patched ROM"}, + {"hack": "Liquid Crystal", "platform": "gba", "ext": "gba", "vfmt": "gba", + "kind": "rom", "url": arc("pokemon-liquid-crystal-v-3.3.00512", "Pokemon - Liquid Crystal (v3.3.00512).gba"), + "note": "v3.3.00512 (final), Johto remake, pre-patched ROM"}, + {"hack": "Theta Emerald EX", "platform": "gba", "ext": "gba", "vfmt": "gba", + "kind": "rom", "url": arc("pokemon-theta-emerald-ex_202407", "theta-emerald-ex-02-27-17.gba"), + "note": "EX build (all 800+ mons), pre-patched ROM"}, + {"hack": "Snakewood", "platform": "gba", "ext": "gba", "vfmt": "gba", + "kind": "rom", "url": arc("pokemon-snakewood", "Snakewood.gba"), + "note": "zombie/horror hack (base Ruby), pre-patched ROM"}, + {"hack": "Brown", "platform": "gb", "ext": "gb", "vfmt": "gbc", + "kind": "patch", "base": "red", "zip_member": r"\.(ips|bps|ups)$", + "url": arc("brown_20250526", "brown.zip"), + "note": "v1.1 (Koolboyman, base Red) — patch inside zip"}, + {"hack": "Blaze Black 2 Redux", "platform": "nds", "ext": "nds", "vfmt": "nds", + "kind": "rom", "zip_member": r"\.nds$", + "url": arc("pokemon-blaze-black-2-redux-complete-v-1.3.0", "Pokemon Blaze Black 2 Redux (Complete v1.3.0).zip"), + "note": "Redux fork v1.3.0 Complete (base Black 2), ROM inside zip"}, + {"hack": "Volt White 2 Redux", "platform": "nds", "ext": "nds", "vfmt": "nds", + "kind": "rom", "zip_member": r"\.nds$", + "url": arc("pokemon-volt-white-2-redux-complete-v-1.4.1", "Pokemon Volt White 2 Redux Complete (v1.4.1).zip"), + "note": "Redux fork v1.4.1 Complete (base White 2), ROM inside zip"}, +] + +# --- batch 3: additional well-regarded complete hacks (direct ROMs) ---------- +MANIFEST3 = [ + {"hack": "Blaze Black", "platform": "nds", "ext": "nds", "vfmt": "nds", + "kind": "rom", "url": arc("pokemon-blaze-black", "Blaze Black.nds"), + "note": "Drayano (base Black), pairs with Volt White, pre-patched ROM"}, + {"hack": "Mega Power", "platform": "gba", "ext": "gba", "vfmt": "gba", + "kind": "rom", "url": arc("pokemon-mega-power-v-5.62", "Pokemon Mega Power (v5.62).gba"), + "note": "v5.62 (complete), pre-patched ROM"}, + {"hack": "Resolute", "platform": "gba", "ext": "gba", "vfmt": "gba", + "kind": "rom", "url": arc("pokemon-resolute", "Resolute.gba"), + "note": "complete story hack, pre-patched ROM"}, +] + +# Original multi-file archive packs handled by one-off steps (not single-artifact +# manifest entries). Listed here so --retain can store the true source archives. +SPECIAL_SOURCES = [ + ("gba", arc("pokemon-emerald-kaizo", "Pokemon Emerald Kaizo.rar"), + "Emerald Kaizo source (rar -> IPS inside)"), + ("gba", arc("pokemon-dark-rising-complete-pack", "Pokemon Dark Rising Complete Pack.zip"), + "Dark Rising 1/2/Order Destroyed/Kaizo (nested zips inside)"), + ("nds", arc("pokemon-sacred-gold-and-storm-silver-1.5-fairy", "Pokemon Sacred Gold and Storm Silver 1.5.zip"), + "Sacred Gold + Storm Silver V1.05 xdelta patch pack"), +] + + +def basename_from_url(url): + return os.path.basename(urllib.parse.unquote(urllib.parse.urlparse(url).path)) + + +def retain_all(): + """Re-download every source artifact and store it persistently under PATCHES//.""" + rows = [] + items = [(m["platform"], m["url"], m["hack"]) for m in (MANIFEST + MANIFEST2 + MANIFEST3)] + items += [(plat, url, note) for plat, url, note in SPECIAL_SOURCES] + for plat, url, label in items: + dst_dir = os.path.join(PATCHES, plat) + os.makedirs(dst_dir, exist_ok=True) + dst = os.path.join(dst_dir, basename_from_url(url)) + try: + sz = download(url, dst) + rows.append((label, plat, "OK", f"{sz:,}b", os.path.basename(dst))) + except Exception as e: + rows.append((label, plat, "FAIL", str(e)[:60], basename_from_url(url))) + print("\n=== RETAIN SOURCE ARTIFACTS ===") + for lbl, pl, st, dt, nm in rows: + print(f"[{st:4}] {pl:4} {dt:>16} {nm}") + ok = sum(1 for r in rows if r[2] == "OK") + print(f"\n{ok}/{len(rows)} stored under {PATCHES}//") + + +def extract_member(zip_path, pattern, dest): + with zipfile.ZipFile(zip_path) as z: + members = [n for n in z.namelist() if not n.endswith("/") and re.search(pattern, n, re.I)] + if not members: + raise RuntimeError(f"no zip member matches {pattern} (have: {z.namelist()[:5]})") + # pick the largest matching member (avoids readme/junk) + members.sort(key=lambda n: z.getinfo(n).file_size, reverse=True) + with z.open(members[0]) as src, open(dest, "wb") as out: + shutil.copyfileobj(src, out) + return members[0] + +def download(url, dest): + req = urllib.request.Request(url, headers={"User-Agent": UA}) + with urllib.request.urlopen(req, timeout=120) as r, open(dest, "wb") as f: + shutil.copyfileobj(r, f) + return os.path.getsize(dest) + +def validate(path, vfmt): + """Cheap structural sanity check so a mislabeled/corrupt file is caught.""" + with open(path, "rb") as f: + data = f.read() + n = len(data) + if vfmt == "gba": + if n % (1024 * 1024) != 0 or not (4*1024*1024 <= n <= 32*1024*1024): + return f"bad GBA size {n}" + if data[0xB2] != 0x96: + return "GBA fixed byte 0xB2!=0x96 (not a GBA ROM?)" + elif vfmt == "gbc": + logo = bytes.fromhex("ceed6666cc0d") + if data[0x104:0x10A] != logo: + return "GBC Nintendo logo missing at 0x104" + elif vfmt == "nds": + if not (8*1024*1024 <= n <= 512*1024*1024): + return f"bad NDS size {n}" + if data[0x15C:0x15E] != b"\x56\xCF": # standard logo CRC16 0xCF56 + return "NDS logo CRC16 != 0xCF56" + return None # ok + +def main(): + place = "--place" in sys.argv + only = None + if "--only" in sys.argv: + only = sys.argv[sys.argv.index("--only") + 1].lower() + if "--retain" in sys.argv: + retain_all() + return + batch = 1 + if "--batch" in sys.argv: + batch = int(sys.argv[sys.argv.index("--batch") + 1]) + manifest = {1: MANIFEST, 2: MANIFEST2, 3: MANIFEST3}[batch] + os.makedirs(WORK, exist_ok=True) + rows = [] + for m in manifest: + if only and only not in m["hack"].lower(): + continue + hack = m["hack"]; plat = m["platform"]; ext = m["ext"] + out_name = f"Pokemon - {hack} (Hack).{ext}" + tmp = os.path.join(WORK, f"dl_{hack.replace(' ', '_')}") + status = "OK"; detail = m.get("note", "") + try: + sz = download(m["url"], tmp) + if m.get("zip_member"): + inner = os.path.join(WORK, f"zx_{hack.replace(' ', '_')}") + picked = extract_member(tmp, m["zip_member"], inner) + detail += f" | zip:{os.path.basename(picked)}" + tmp = inner + if m["kind"] == "patch": + base = BASES[m["base"]] + if not os.path.exists(base): + raise RuntimeError(f"base missing: {base}") + final = os.path.join(WORK, f"out_{hack.replace(' ', '_')}.{ext}") + r = subprocess.run([sys.executable, APPLY, tmp, base, final], + capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError(f"apply failed: {r.stderr.strip()}") + detail = r.stdout.strip().split("crc32=")[-1].rstrip(")") + detail = f"applied, crc32={detail}" + src = final + else: + src = tmp + verr = validate(src, m["vfmt"]) + if verr: + raise RuntimeError(f"validation: {verr}") + crc = zlib.crc32(open(src, "rb").read()) & 0xffffffff + if place: + dst_dir = os.path.join(ROMS, plat, "Hacks") + os.makedirs(dst_dir, exist_ok=True) + shutil.copy2(src, os.path.join(dst_dir, out_name)) + detail += " | PLACED" + rows.append((hack, plat, "OK", f"{os.path.getsize(src):,}b crc={crc:08x}", out_name)) + except Exception as e: + rows.append((hack, plat, "FAIL", str(e)[:80], out_name)) + print("\n=== ROMHACK FETCH SUMMARY ===") + for hk, pl, st, dt, nm in rows: + print(f"[{st:4}] {pl:4} {hk:<20} {dt}") + if st == "OK": + print(f" -> {pl}/Hacks/{nm}") + print(f"\nplaced: {'YES (--place)' if place else 'NO (dry run; pass --place)'}") + +if __name__ == "__main__": + main()