feat(vault): synthesize Discord scrape + web research into the ROM-hack catalog
Enrich the Obsidian ROM-hack vault from the valhalla Discord scrape
(/storage1/labdata/romhacks/metadata) plus web research, growing the catalog
from 285 to 368 notes and preparing a data export for a wiki site.
Tooling (all dry-run by default, --apply to write; .scrape/ is gitignored):
- scrape_match.py reconcile scrape entries vs notes (43 overlap / 211 match / 83 new)
- enrich_vault.py backfill frontmatter, merge scrape into curated notes,
create 83 new notes, rebuild import bodies to one layout
- enrich_web.py apply hand/web-verified facts from .scrape/web_facts*.json
- build-wiki-data.py export wiki/catalog.json (the site data source)
Results: 368 notes, 327 with served banner art, 332 rich summaries, 200 with
feature lists; new frontmatter developer/release_date/banner/homepage/scrape_dir;
12 flagship hacks web-verified. wiki/SPEC.md describes the site build.
build-vault-mocs.py Index callout updated for the new counts.
This commit is contained in:
ginnoir
@@ -0,0 +1,751 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Enrich the Pokémon ROM-hack vault from the Discord scrape (.scrape/*/metadata.json).
|
||||
|
||||
Three subcommands, each --dry-run by default (writes previews to .scrape/_preview/)
|
||||
and only touches the live vault with --apply:
|
||||
|
||||
backfill 211 Discord-import notes: fill missing frontmatter (base, version,
|
||||
status, type, developer, release_date, banner, scrape_dir) parsed
|
||||
from the scrape. Body is preserved verbatim.
|
||||
merge 43 curated-overlap notes: keep the hand-written note, add a banner,
|
||||
append Story/Features/community Links parsed from the scrape, and
|
||||
add developer/release_date/banner/scrape_dir frontmatter. The curated
|
||||
Summary and existing frontmatter are preserved.
|
||||
create 83 unmatched scraped hacks: write brand-new wiki-schema notes.
|
||||
|
||||
Source of truth for matching = .scrape/_match_report.json (scrape_match.py).
|
||||
Art is served at https://romhacks-files.ginnoir.com/_meta/<scrape_dir>/<file>.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json, os, re, sys, glob, unicodedata
|
||||
from datetime import date
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
VAULT = r"C:\Users\MattC\Documents\Obsidian Vault"
|
||||
HACKS = os.path.join(VAULT, "Pokémon ROM Hacks", "Hacks")
|
||||
SCRAPE = os.path.join(ROOT, ".scrape")
|
||||
PREVIEW = os.path.join(SCRAPE, "_preview")
|
||||
ART_BASE = "https://romhacks-files.ginnoir.com/_meta"
|
||||
TODAY = date.today().isoformat()
|
||||
|
||||
# ---------------------------------------------------------------- text cleaning
|
||||
|
||||
def clean(s: str) -> str:
|
||||
if not s:
|
||||
return ""
|
||||
s = s.replace("", "").replace("", "")
|
||||
s = s.replace("\xa0", " ").replace("’", "'").replace("‘", "'")
|
||||
s = s.replace("“", '"').replace("”", '"').replace("–", "-")
|
||||
s = re.sub(r"[ \t]+", " ", s)
|
||||
s = re.sub(r"\n{3,}", "\n\n", s)
|
||||
return s.strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- parsing
|
||||
|
||||
MONTHS = {m: i for i, m in enumerate(
|
||||
["january", "february", "march", "april", "may", "june", "july",
|
||||
"august", "september", "october", "november", "december"], 1)}
|
||||
|
||||
PLATFORM_HINTS = [
|
||||
(r"\bGBA\b|game boy advance", ("GBA", "console")),
|
||||
(r"\bGBC\b|game boy color", ("GBC", "console")),
|
||||
(r"\bNDS\b|nintendo ds\b|\bDS ROM", ("NDS", "console")),
|
||||
(r"\b3DS\b", ("3DS", "console")),
|
||||
(r"\bGB\b|game boy(?! advance| color)", ("GB", "console")),
|
||||
(r"RPGXP|RPG ?Maker|Essentials|FanGame|fan game|GameMaker", ("PC", "fangame")),
|
||||
]
|
||||
|
||||
BASE_CANON = {
|
||||
"fire red": "FireRed", "firered": "FireRed", "fire-red": "FireRed",
|
||||
"leaf green": "LeafGreen", "leafgreen": "LeafGreen",
|
||||
"fire red/leaf green": "FireRed", "firered/leafgreen": "FireRed",
|
||||
"emerald": "Emerald", "ruby": "Ruby", "sapphire": "Sapphire",
|
||||
"ruby/sapphire": "Ruby",
|
||||
"crystal": "Crystal", "gold": "Gold", "silver": "Silver",
|
||||
"gold/silver": "Gold", "red": "Red", "blue": "Blue", "yellow": "Yellow",
|
||||
"diamond": "Diamond", "pearl": "Pearl", "platinum": "Platinum",
|
||||
"heart gold": "HeartGold", "heartgold": "HeartGold",
|
||||
"soul silver": "SoulSilver", "soulsilver": "SoulSilver",
|
||||
"black": "Black", "white": "White", "black 2": "Black 2",
|
||||
"white 2": "White 2", "black2": "Black 2", "white2": "White 2",
|
||||
}
|
||||
|
||||
SECTION_ALIASES = {
|
||||
"description": "Description", "about": "Description", "overview": "Description",
|
||||
"plot": "Story", "story": "Story", "synopsis": "Story",
|
||||
"features": "Features", "feature": "Features", "key features": "Features",
|
||||
"download": "Download", "downloads": "Download", "rom link": "Download",
|
||||
"changelog": "Changelog", "credits": "Credits",
|
||||
}
|
||||
|
||||
|
||||
def norm_key(name: str) -> str:
|
||||
name = unicodedata.normalize("NFKD", name)
|
||||
name = "".join(c for c in name if not unicodedata.combining(c)).lower()
|
||||
name = re.sub(r"\bpok[e]?mon\b", " ", name)
|
||||
return re.sub(r"[^a-z0-9]+", "", name)
|
||||
|
||||
|
||||
def parse_release(text: str):
|
||||
m = re.search(r"(?:last updated on|released on|updated on|released)\s+"
|
||||
r"([A-Za-z]+)\s+(\d{1,2}),?\s*(\d{4})", text, re.I)
|
||||
if m:
|
||||
mo = MONTHS.get(m.group(1).lower())
|
||||
if mo:
|
||||
return f"{m.group(3)}-{mo:02d}-{int(m.group(2)):02d}"
|
||||
m = re.search(r"\b(20\d{2})\b", text)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def parse_intro(text: str) -> dict:
|
||||
out: dict = {}
|
||||
head = text[:400]
|
||||
m = re.search(r"\bby\s+(.+?)(?:\s+based on|\s+made using|\s+using|\s+in English|"
|
||||
r"\.\s|\,\s|\s+and it| \(|\n)", head)
|
||||
if m:
|
||||
dev = m.group(1).strip(" .,-")
|
||||
if 1 < len(dev) < 60 and not dev.lower().startswith(("the ", "a ")):
|
||||
out["developer"] = dev
|
||||
m = re.search(r"based on\s+(?:Pok[eé]mon\s+)?([A-Za-z0-9][A-Za-z0-9 /&]+?)"
|
||||
r"(?:\.|,|\n| in | with |!)", head, re.I)
|
||||
if m:
|
||||
raw = m.group(1).strip().lower()
|
||||
raw = re.sub(r"\bversions?\b", "", raw).strip()
|
||||
out["base"] = BASE_CANON.get(raw, m.group(1).strip())
|
||||
for pat, (plat, kind) in PLATFORM_HINTS:
|
||||
if re.search(pat, head, re.I):
|
||||
out["platform"], out["kind"] = plat, kind
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
# inline / line key-value metadata blocks, e.g. "**Creator:** X **Type:** GBA
|
||||
# **Hack of:** FireRed **Version:** Final **Last Updated:** May 26, 2020".
|
||||
KV_KEYS = (r"Creator|Developer|Dev|Author|Made by|Type of ROM|Type|Platform|"
|
||||
r"System|Hack of|Base ROM|Base game|Base|Based on|Version|Status|"
|
||||
r"Last Updated|Updated|Release Date|Released|Date|Language|Genre")
|
||||
KV_RE = re.compile(rf"(?i)\b(?P<key>{KV_KEYS})\s*[:\-]\s*(?P<val>.+?)"
|
||||
rf"(?=\s{{2,}}|\s*\b(?:{KV_KEYS})\b\s*[:\-]|$)", re.M)
|
||||
KV_LINE = re.compile(rf"(?i)^\s*(?:{KV_KEYS})\s*[:\-]")
|
||||
|
||||
|
||||
def parse_kv(text: str) -> dict:
|
||||
"""Extract dev/platform/base/version/status/release from key-value blocks
|
||||
(bold or plain). De-bold first so '**Creator:** X' -> 'Creator: X'."""
|
||||
flat = text.replace("*", "")
|
||||
got: dict[str, str] = {}
|
||||
for m in KV_RE.finditer(flat):
|
||||
k = m.group("key").lower().strip()
|
||||
v = m.group("val").strip(" .,-")
|
||||
if not v or len(v) > 80:
|
||||
continue
|
||||
got.setdefault(k, v)
|
||||
out = {}
|
||||
def first(*keys):
|
||||
for k in keys:
|
||||
if k in got:
|
||||
return got[k]
|
||||
return None
|
||||
dev = first("creator", "developer", "dev", "author", "made by")
|
||||
if dev:
|
||||
out["developer"] = dev
|
||||
typ = first("type of rom", "type", "platform", "system")
|
||||
if typ:
|
||||
for pat, (plat, kind) in PLATFORM_HINTS:
|
||||
if re.search(pat, typ, re.I):
|
||||
out["platform"], out["kind"] = plat, kind
|
||||
break
|
||||
base = first("hack of", "base rom", "base game", "base", "based on")
|
||||
if base:
|
||||
bl = re.sub(r"\bversions?\b", "", base.lower()).strip()
|
||||
out["base"] = BASE_CANON.get(bl, base.strip())
|
||||
if first("version"):
|
||||
out["version"] = first("version")
|
||||
if first("status"):
|
||||
out["status"] = first("status")
|
||||
if first("language"):
|
||||
out["language"] = first("language")
|
||||
rel = first("last updated", "updated", "release date", "released", "date")
|
||||
if rel:
|
||||
pr = parse_release(rel) or parse_release("released " + rel)
|
||||
if pr:
|
||||
out["release_date"] = pr
|
||||
return out
|
||||
|
||||
|
||||
def split_sections(text: str, name: str = ""):
|
||||
lines = text.split("\n")
|
||||
sections: dict[str, list[str]] = {}
|
||||
lead: list[str] = []
|
||||
cur = None
|
||||
name_key = norm_key(name)
|
||||
for raw in lines:
|
||||
line = re.sub(r"^>\s?", "", raw.strip())
|
||||
hkey = re.sub(r"[*#:_~`]", "", line).strip().lower()
|
||||
if hkey in SECTION_ALIASES and len(line) < 40:
|
||||
cur = SECTION_ALIASES[hkey]
|
||||
sections.setdefault(cur, [])
|
||||
continue
|
||||
flat = line.replace("*", "")
|
||||
low = flat.lower()
|
||||
is_meta = (re.match(r"^pok[e]?mon", low) and (" is a" in low or " is an" in low)) \
|
||||
or bool(KV_LINE.match(flat)) \
|
||||
or low.startswith(("rom link", "- rom link", "it was last updated",
|
||||
"made using", "by ")) \
|
||||
or (name_key and norm_key(line) == name_key) # bare title repeat
|
||||
if cur:
|
||||
sections[cur].append(line)
|
||||
elif not is_meta and line:
|
||||
lead.append(line)
|
||||
feats = []
|
||||
for ln in sections.get("Features", []):
|
||||
s = ln.strip().lstrip("-*•").strip()
|
||||
if s and not s.lower().startswith("download"):
|
||||
feats.append(s)
|
||||
lead_text = " ".join(lead).strip()
|
||||
return lead_text, {k: clean("\n".join(v)) for k, v in sections.items()}, feats
|
||||
|
||||
|
||||
def collect_links(meta: dict, text: str):
|
||||
urls = list(meta.get("links") or [])
|
||||
for m in re.finditer(r"https?://[^\s)\]>'\"]+", text):
|
||||
u = m.group(0).rstrip(".,)")
|
||||
if u not in urls:
|
||||
urls.append(u)
|
||||
seen, out = set(), []
|
||||
for u in urls:
|
||||
if u not in seen:
|
||||
seen.add(u); out.append(u)
|
||||
return out
|
||||
|
||||
|
||||
TYPE_RULES = [
|
||||
("Difficulty", r"kaizo|difficulty hack|harder|increased difficulty|"
|
||||
r"competitive|nuzlocke|challenging|boosted ai|smart ai"),
|
||||
("Expansion", r"mega evolution|national dex|gen 8|gen 9|generation 8|"
|
||||
r"generation 9|physical/special split|fairy type|decomp|"
|
||||
r"expansion|all 8\d\d"),
|
||||
("New Experience", r"new region|new story|original story|fakemon|"
|
||||
r"new pok[e]?mon|custom region|brand new"),
|
||||
("Roguelite", r"roguelite|roguelike|run-based|permadeath|randomized run"),
|
||||
("Demake", r"demake|de-make|in gen ?[12] style|on gbc|1-bit"),
|
||||
("Cosmetic", r"moemon|sprite pack|tileset|cosmetic|visual overhaul|reskin"),
|
||||
("QoL", r"quality of life|quality-of-life|\bqol\b|reusable tms|exp share"),
|
||||
]
|
||||
|
||||
|
||||
def infer_types(text: str) -> list[str]:
|
||||
low = text.lower()
|
||||
hits = [name for name, pat in TYPE_RULES if re.search(pat, low)]
|
||||
return hits[:2]
|
||||
|
||||
|
||||
# base game -> (platform, canonical base) for name/desc heuristic fallback
|
||||
GAME_PLATFORM = {
|
||||
"emerald": ("GBA", "Emerald"), "fire red": ("GBA", "FireRed"),
|
||||
"firered": ("GBA", "FireRed"), "leaf green": ("GBA", "LeafGreen"),
|
||||
"leafgreen": ("GBA", "LeafGreen"), "ruby": ("GBA", "Ruby"),
|
||||
"sapphire": ("GBA", "Sapphire"),
|
||||
"crystal": ("GBC", "Crystal"), "gold": ("GBC", "Gold"),
|
||||
"silver": ("GBC", "Silver"),
|
||||
"diamond": ("NDS", "Diamond"), "pearl": ("NDS", "Pearl"),
|
||||
"platinum": ("NDS", "Platinum"), "heartgold": ("NDS", "HeartGold"),
|
||||
"soulsilver": ("NDS", "SoulSilver"), "heart gold": ("NDS", "HeartGold"),
|
||||
"soul silver": ("NDS", "SoulSilver"), "black": ("NDS", "Black"),
|
||||
"white": ("NDS", "White"), "black 2": ("NDS", "Black 2"),
|
||||
"white 2": ("NDS", "White 2"),
|
||||
}
|
||||
|
||||
|
||||
def infer_platform_base(name, desc, platform, base):
|
||||
"""Fallback: if platform unknown, infer GBA/GBC/GB/NDS from a base game word
|
||||
in the name or first part of the description. Never overrides a known value."""
|
||||
if platform and base:
|
||||
return platform, base
|
||||
hay = (name + " " + desc[:300]).lower()
|
||||
# if we already know base, set platform from it
|
||||
if base and base != "—" and not platform:
|
||||
bp = GAME_PLATFORM.get(base.lower())
|
||||
if bp:
|
||||
return bp[0], base
|
||||
if not platform:
|
||||
for word, (plat, canon) in GAME_PLATFORM.items():
|
||||
if re.search(rf"\b{re.escape(word)}\b", hay):
|
||||
return plat, (base if base and base != "—" else canon)
|
||||
return platform, base
|
||||
|
||||
|
||||
LEAD_PREFIX = re.compile(
|
||||
r"^\s*(?:version|status|update[d]?)\b\s*[:\-]?\s*"
|
||||
r"(?:completed|complete|beta|ongoing|final|released?)?\s*"
|
||||
r"(?:v?\d[\d.\w]*)?\s*[:\-]?\s*", re.I)
|
||||
|
||||
|
||||
def clean_lead(lead: str):
|
||||
"""Strip a leading 'Version <x>'/'Completed' label off a lead line and
|
||||
return (cleaned_lead, version, status) harvested from it."""
|
||||
ver = status = None
|
||||
mv = re.match(r"^\s*(?:version|ver)\b\s*[:\-]?\s*(?:completed|complete|"
|
||||
r"beta|ongoing|final)?\s*(v?\d[\d.\w]*)", lead, re.I)
|
||||
if mv:
|
||||
ver = mv.group(1)
|
||||
ms = re.match(r"^\s*(?:status\b\s*[:\-]?\s*)?(completed|complete|beta|ongoing)",
|
||||
lead, re.I)
|
||||
if ms:
|
||||
status = ms.group(1)
|
||||
new = LEAD_PREFIX.sub("", lead)
|
||||
# a second pass for 'Version completed 1.3.1' style double-prefix
|
||||
new = LEAD_PREFIX.sub("", new).strip(" :-")
|
||||
if len(new) < 25:
|
||||
new = ""
|
||||
return new, ver, status
|
||||
|
||||
|
||||
def build_record(meta: dict, libidx: dict) -> dict:
|
||||
name = clean(meta.get("name", ""))
|
||||
desc = clean(meta.get("description", ""))
|
||||
scrape_dir = meta["_dir"]
|
||||
li = libidx.get(scrape_dir, {})
|
||||
intro = parse_intro(desc)
|
||||
kv = parse_kv(desc)
|
||||
lead, sections, feats = split_sections(desc, name)
|
||||
rec = {
|
||||
"name": name,
|
||||
"scrape_dir": scrape_dir,
|
||||
"platform": (li.get("platform") or intro.get("platform")
|
||||
or kv.get("platform") or "").upper() or None,
|
||||
"kind": li.get("kind") or intro.get("kind") or kv.get("kind"),
|
||||
"base": intro.get("base") or kv.get("base"),
|
||||
"developer": intro.get("developer") or kv.get("developer"),
|
||||
"release_date": kv.get("release_date") or parse_release(desc),
|
||||
"version": kv.get("version"),
|
||||
"status": kv.get("status"),
|
||||
"lead": lead,
|
||||
"story": sections.get("Story", ""),
|
||||
"description_body": sections.get("Description", ""),
|
||||
"features": feats,
|
||||
"links": [u for u in collect_links(meta, desc) if not u.startswith(ART_BASE)],
|
||||
"art": (meta.get("art") or [None])[0],
|
||||
"types": infer_types(desc),
|
||||
}
|
||||
# tidy the lead: strip leading 'Version x'/'Completed' labels, fall back to
|
||||
# the Description section, harvest version/status if still unknown.
|
||||
cl, lv, ls = clean_lead(rec["lead"])
|
||||
rec["lead"] = cl or sections.get("Description", "")[:600]
|
||||
if not rec["version"] and lv:
|
||||
rec["version"] = lv
|
||||
if not rec["status"] and ls:
|
||||
rec["status"] = ls
|
||||
if rec["platform"] == "PC" and not rec["base"]:
|
||||
rec["base"] = "—"
|
||||
if rec["platform"] != "PC":
|
||||
p, b = infer_platform_base(name, desc, rec["platform"], rec["base"])
|
||||
rec["platform"], rec["base"] = p, b
|
||||
return rec
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- frontmatter io
|
||||
|
||||
def read_note(path):
|
||||
text = open(path, encoding="utf-8").read()
|
||||
m = re.match(r"^---\n(.*?)\n---\n?(.*)$", text, re.S)
|
||||
if not m:
|
||||
return "", text, text
|
||||
return m.group(1), m.group(2), text
|
||||
|
||||
|
||||
def fm_get(fm_text, key):
|
||||
m = re.search(rf"(?im)^{re.escape(key)}:\s*(.*)$", fm_text)
|
||||
return m.group(1).strip().strip('"') if m else None
|
||||
|
||||
|
||||
def fm_set(fm_text, key, value, quote_it=True):
|
||||
v = f'"{value}"' if quote_it and not str(value).startswith(("[", '"')) else value
|
||||
line = f"{key}: {v}"
|
||||
if re.search(rf"(?im)^{re.escape(key)}:\s*.*$", fm_text):
|
||||
return re.sub(rf"(?im)^{re.escape(key)}:\s*.*$", line, fm_text, count=1)
|
||||
return fm_text.rstrip() + "\n" + line
|
||||
|
||||
|
||||
def art_url(rec):
|
||||
if rec.get("art"):
|
||||
return f"{ART_BASE}/{rec['scrape_dir']}/{quote(rec['art'])}"
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- loaders
|
||||
|
||||
def load_scrape():
|
||||
out = {}
|
||||
for jf in glob.glob(os.path.join(SCRAPE, "*", "metadata.json")):
|
||||
m = json.load(open(jf, encoding="utf-8"))
|
||||
m["_dir"] = os.path.basename(os.path.dirname(jf))
|
||||
out[m["_dir"]] = m
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- frontmatter (de)serialize
|
||||
|
||||
FM_ORDER = ["title", "platform", "base", "version", "status", "type",
|
||||
"generation", "developer", "release_date", "banner",
|
||||
"library_path", "source", "homepage", "scrape_dir",
|
||||
"added", "play_status", "rating", "tags"]
|
||||
LIST_KEYS = {"type", "tags"}
|
||||
|
||||
|
||||
def parse_fm(fm_text):
|
||||
"""Parse simple block-style frontmatter into an ordered dict.
|
||||
Scalars -> str; block/flow lists -> list[str]."""
|
||||
fm, lines, i = {}, fm_text.split("\n"), 0
|
||||
while i < len(lines):
|
||||
km = re.match(r"^([\w]+):\s*(.*)$", lines[i])
|
||||
if km:
|
||||
k, v = km.group(1), km.group(2)
|
||||
if v == "" and i + 1 < len(lines) and re.match(r"^\s*-\s", lines[i + 1]):
|
||||
items = []
|
||||
while i + 1 < len(lines) and re.match(r"^\s*-\s", lines[i + 1]):
|
||||
items.append(lines[i + 1].strip()[2:].strip().strip('"'))
|
||||
i += 1
|
||||
fm[k] = items
|
||||
elif v.strip().startswith("[") and v.strip().endswith("]"):
|
||||
inner = v.strip()[1:-1].strip()
|
||||
fm[k] = [x.strip().strip('"') for x in inner.split(",")] if inner else []
|
||||
else:
|
||||
fm[k] = v.strip().strip('"')
|
||||
i += 1
|
||||
return fm
|
||||
|
||||
|
||||
def emit_fm(fm):
|
||||
out = []
|
||||
keys = [k for k in FM_ORDER if k in fm] + [k for k in fm if k not in FM_ORDER]
|
||||
for k in keys:
|
||||
v = fm[k]
|
||||
if k in LIST_KEYS or isinstance(v, list):
|
||||
v = v or []
|
||||
if not v:
|
||||
out.append(f"{k}: []")
|
||||
else:
|
||||
out.append(f"{k}:")
|
||||
out.extend(f" - {x}" for x in v)
|
||||
elif k == "rating":
|
||||
out.append(f"{k}: {v}")
|
||||
elif k == "added" and re.match(r"^\d{4}-\d{2}-\d{2}$", str(v)):
|
||||
out.append(f"{k}: {v}")
|
||||
elif k in ("version", "release_date", "generation"):
|
||||
# always strings — avoid YAML number coercion (4.10 -> 4.1, 2024 -> int)
|
||||
out.append(f'{k}: "{v}"')
|
||||
else:
|
||||
sv = str(v)
|
||||
if sv == "" or re.search(r'[:#\[\]"\']', sv) or sv == "—":
|
||||
out.append(f'{k}: "{sv}"')
|
||||
else:
|
||||
out.append(f"{k}: {sv}")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def canon_status(s):
|
||||
if not s:
|
||||
return None
|
||||
low = s.lower()
|
||||
if "beta" in low:
|
||||
return "Beta"
|
||||
if "complete" in low or "finished" in low or "full" in low:
|
||||
return "Complete"
|
||||
if any(w in low for w in ("ongoing", "wip", "development", "in progress",
|
||||
"updates", "demo", "unfinished")):
|
||||
return "Ongoing"
|
||||
return None
|
||||
|
||||
|
||||
PLATFORM_LABEL = {"GB": "Game Boy", "GBC": "Game Boy Color",
|
||||
"GBA": "Game Boy Advance", "NDS": "Nintendo DS",
|
||||
"3DS": "Nintendo 3DS", "PC": "PC / Joiplay", "—": "Unknown"}
|
||||
|
||||
|
||||
def derive_tags(fm):
|
||||
tags = ["hack"]
|
||||
p = (fm.get("platform") or "").lower()
|
||||
if p and p != "—":
|
||||
tags.append(f"platform/{p}")
|
||||
b = (fm.get("base") or "")
|
||||
if b and b != "—":
|
||||
tags.append(f"base/{b.lower().replace(' ', '')}")
|
||||
st = (fm.get("status") or "")
|
||||
if st and st != "—":
|
||||
tags.append(f"status/{st.lower().replace(' ', '')}")
|
||||
for t in (fm.get("type") or []):
|
||||
tags.append(f"type/{t.lower().replace(' ', '').replace('+', 'plus')}")
|
||||
if fm.get("scrape_dir"):
|
||||
tags.append("source/discord")
|
||||
return tags
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- body builders
|
||||
|
||||
def features_block(feats, limit=24):
|
||||
return "\n".join(f"- {clean(f)}" for f in feats[:limit])
|
||||
|
||||
|
||||
def links_block(links, limit=8):
|
||||
out = []
|
||||
for u in links[:limit]:
|
||||
label = "Download" if re.search(r"mediafire|drive\.google|mega\.nz|"
|
||||
r"github|\.zip|\.rar|/file", u) else "Link"
|
||||
if "docs.google" in u:
|
||||
label = "Guide / docs"
|
||||
if "pokecommunity" in u or "pokeharbor" in u or "hackdex" in u:
|
||||
label = "Info / thread"
|
||||
out.append(f"- {label}: {u}")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def type_links(types):
|
||||
return " ".join(f"[[Types#{t}|{t}]]" for t in types) if types else ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- writers
|
||||
|
||||
def out_write(path, content, apply):
|
||||
if apply:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
open(path, "w", encoding="utf-8").write(content)
|
||||
else:
|
||||
pp = os.path.join(PREVIEW, os.path.basename(path))
|
||||
os.makedirs(PREVIEW, exist_ok=True)
|
||||
open(pp, "w", encoding="utf-8").write(content)
|
||||
|
||||
|
||||
def cmd_backfill(report, scrape, libidx, apply):
|
||||
"""211 Discord notes: fill missing frontmatter only; body preserved."""
|
||||
changed = 0
|
||||
log = []
|
||||
for e in report["results"]["discord_match"]:
|
||||
stem = e["note_stem"]
|
||||
path = os.path.join(HACKS, stem + ".md")
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
rec = build_record(scrape[e["scrape_dir"]], libidx)
|
||||
fm_text, body, _ = read_note(path)
|
||||
fm = parse_fm(fm_text)
|
||||
before = dict(fm)
|
||||
# fill only missing / dash
|
||||
def missing(k):
|
||||
v = fm.get(k)
|
||||
return v in (None, "", "—") or (k == "type" and not v)
|
||||
if missing("base") and rec["base"] and rec["base"] != "—":
|
||||
fm["base"] = rec["base"]
|
||||
if missing("version") and rec["version"]:
|
||||
fm["version"] = rec["version"]
|
||||
if missing("status"):
|
||||
cs = canon_status(rec["status"]) or canon_status(rec["lead"])
|
||||
if cs:
|
||||
fm["status"] = cs
|
||||
if missing("type") and rec["types"]:
|
||||
fm["type"] = rec["types"]
|
||||
if missing("developer") and rec["developer"]:
|
||||
fm["developer"] = rec["developer"]
|
||||
if missing("release_date") and rec["release_date"]:
|
||||
fm["release_date"] = rec["release_date"]
|
||||
if missing("banner"):
|
||||
au = art_url(rec)
|
||||
if au:
|
||||
fm["banner"] = au
|
||||
fm["scrape_dir"] = e["scrape_dir"]
|
||||
fm["tags"] = derive_tags(fm)
|
||||
if fm != before:
|
||||
changed += 1
|
||||
new = f"---\n{emit_fm(fm)}\n---\n{body}"
|
||||
out_write(path, new, apply)
|
||||
log.append(f" {stem}: +{sorted(set(fm)-set(k for k in before if before[k] not in (None,'','—')))}")
|
||||
print(f"backfill: {changed} notes updated"
|
||||
f"{' (APPLIED)' if apply else ' (dry-run -> .scrape/_preview)'}")
|
||||
for l in log[:30]:
|
||||
print(l)
|
||||
|
||||
|
||||
def cmd_merge(report, scrape, libidx, apply):
|
||||
"""43 curated overlaps: keep note, add banner + Story/Features/community links."""
|
||||
changed = 0
|
||||
for e in report["results"]["curated_overlap"]:
|
||||
stem = e["note_stem"]
|
||||
path = os.path.join(HACKS, stem + ".md")
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
rec = build_record(scrape[e["scrape_dir"]], libidx)
|
||||
fm_text, body, _ = read_note(path)
|
||||
fm = parse_fm(fm_text)
|
||||
if rec["developer"] and not fm.get("developer"):
|
||||
fm["developer"] = rec["developer"]
|
||||
if rec["release_date"] and not fm.get("release_date"):
|
||||
fm["release_date"] = rec["release_date"]
|
||||
au = art_url(rec)
|
||||
if au and not fm.get("banner"):
|
||||
fm["banner"] = au
|
||||
fm["scrape_dir"] = e["scrape_dir"]
|
||||
|
||||
# body: insert banner image after callout blockquote (if no image yet)
|
||||
if au and "![" not in body:
|
||||
m = re.search(r"((?:^>.*\n)+)", body, re.M)
|
||||
if m:
|
||||
ins = m.end()
|
||||
body = body[:ins] + f"\n\n" + body[ins:]
|
||||
# build append sections before the [[Index footer
|
||||
add = []
|
||||
if rec["story"] and "## Story" not in body:
|
||||
add.append(f"## Story\n\n{rec['story']}")
|
||||
if rec["features"] and "## Features" not in body:
|
||||
add.append(f"## Features\n\n{features_block(rec['features'])}")
|
||||
new_links = [u for u in rec["links"] if u not in body]
|
||||
if new_links and "## Community links" not in body:
|
||||
add.append(f"## Community links\n\n{links_block(new_links)}")
|
||||
if add:
|
||||
block = "\n" + "\n\n".join(add) + "\n\n"
|
||||
# insert after Summary: before the first of Type/Links/In the library,
|
||||
# else before the [[Index footer.
|
||||
anchor = None
|
||||
for h in (r"\n## Type", r"\n## Links", r"\n## In the library",
|
||||
r"\n## Files", r"\n\[\[Index"):
|
||||
m2 = re.search(h, body)
|
||||
if m2:
|
||||
anchor = m2.start()
|
||||
break
|
||||
if anchor is not None:
|
||||
body = body[:anchor] + "\n" + block.rstrip() + "\n" + body[anchor:]
|
||||
else:
|
||||
body = body.rstrip() + "\n\n" + block
|
||||
new = f"---\n{emit_fm(fm)}\n---\n{body}"
|
||||
out_write(path, new, apply)
|
||||
changed += 1
|
||||
print(f"merge: {changed} curated notes enriched"
|
||||
f"{' (APPLIED)' if apply else ' (dry-run -> .scrape/_preview)'}")
|
||||
|
||||
|
||||
def safe_stem(name):
|
||||
s = re.sub(r"^\s*pok[eé]mon\s+", "", name, flags=re.I).strip()
|
||||
s = s.replace("/", " ").replace(":", " -").replace("|", " ")
|
||||
s = re.sub(r'[<>"\\?*]', "", s)
|
||||
s = re.sub(r"\s+", " ", s).strip()
|
||||
return s or name
|
||||
|
||||
|
||||
def build_body(rec, fm, *, owned, library_note=None):
|
||||
"""Clean, uniform wiki body shared by create + rebuild. owned=True writes an
|
||||
'In the library' (RomM/served) block; owned=False writes a catalog-entry note."""
|
||||
title = fm["title"]
|
||||
plat, base = fm.get("platform", "—"), fm.get("base", "—")
|
||||
status = fm.get("status", "—")
|
||||
ver = fm.get("version", "—")
|
||||
au = fm.get("banner")
|
||||
kindline = {"console": "ROM hack", "fangame": "fan-game",
|
||||
"patch": "patch"}.get(rec["kind"], "ROM hack / fan-game")
|
||||
callout = (f"> [!info] {', '.join(fm.get('type') or []) or kindline} — **{status}**\n"
|
||||
f"> {PLATFORM_LABEL.get(plat, plat)} · base **{base}** · "
|
||||
f"version **{ver}**"
|
||||
+ (f" · by **{fm['developer']}**" if fm.get("developer") else ""))
|
||||
parts = [f"# {title}", "", callout, ""]
|
||||
if au:
|
||||
parts += [f"", ""]
|
||||
parts += ["## Summary", "",
|
||||
rec["lead"] or rec["description_body"]
|
||||
or "Catalogued from the Discord ROM-hack archive. See links below.", ""]
|
||||
if rec["story"]:
|
||||
parts += ["## Story", "", rec["story"], ""]
|
||||
if rec["features"]:
|
||||
parts += ["## Features", "", features_block(rec["features"]), ""]
|
||||
if fm.get("type"):
|
||||
parts += ["## Type", type_links(fm["type"]), ""]
|
||||
if rec["links"]:
|
||||
parts += ["## Links", "", links_block(rec["links"]), ""]
|
||||
if owned:
|
||||
slug = library_note or rec["scrape_dir"]
|
||||
parts += ["## In the library", "",
|
||||
f"- Served files: [browse]({ART_BASE.replace('/_meta','')}/{slug}/)",
|
||||
f"- Library path: `/storage1/labdata/romhacks/library/{slug}/`", ""]
|
||||
else:
|
||||
parts += ["## In the library", "",
|
||||
"> [!note] Catalog entry imported from the Discord archive on "
|
||||
f"{TODAY}. Not yet placed in the RomM library.", ""]
|
||||
parts += ["[[Index|← back to directory]]", ""]
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def cmd_create(report, scrape, libidx, apply):
|
||||
"""83 unmatched: brand-new wiki-schema notes."""
|
||||
created = skipped = 0
|
||||
existing = {os.path.splitext(f)[0].lower() for f in os.listdir(HACKS)}
|
||||
for e in report["results"]["unmatched"]:
|
||||
rec = build_record(scrape[e["scrape_dir"]], libidx)
|
||||
stem = safe_stem(rec["name"])
|
||||
if stem.lower() in existing:
|
||||
skipped += 1
|
||||
continue
|
||||
plat = rec["platform"] or "—"
|
||||
base = rec["base"] or "—"
|
||||
status = canon_status(rec["status"]) or canon_status(rec["lead"]) or "—"
|
||||
au = art_url(rec)
|
||||
title = rec["name"] if rec["name"].lower().startswith(("pokémon", "pokemon")) \
|
||||
else f"Pokémon {stem}"
|
||||
title = title.replace("Pokemon", "Pokémon")
|
||||
fm = {"title": title, "platform": plat, "base": base,
|
||||
"version": rec["version"] or "—", "status": status,
|
||||
"type": rec["types"], "generation": "—"}
|
||||
if rec["developer"]:
|
||||
fm["developer"] = rec["developer"]
|
||||
if rec["release_date"]:
|
||||
fm["release_date"] = rec["release_date"]
|
||||
if au:
|
||||
fm["banner"] = au
|
||||
fm["library_path"] = "—"
|
||||
if rec["links"]:
|
||||
fm["source"] = rec["links"][0]
|
||||
fm["scrape_dir"] = e["scrape_dir"]
|
||||
fm["added"] = TODAY
|
||||
fm["play_status"] = "Unplayed"
|
||||
fm["tags"] = derive_tags(fm)
|
||||
content = f"---\n{emit_fm(fm)}\n---\n" + build_body(rec, fm, owned=False)
|
||||
out_write(os.path.join(HACKS, stem + ".md"), content, apply)
|
||||
created += 1
|
||||
print(f"create: {created} new notes, {skipped} skipped (name collision)"
|
||||
f"{' (APPLIED)' if apply else ' (dry-run -> .scrape/_preview)'}")
|
||||
|
||||
|
||||
def cmd_rebuild(report, scrape, libidx, apply):
|
||||
"""211 Discord notes: regenerate the BODY into the clean uniform schema,
|
||||
preserving (already-backfilled) frontmatter. Removes the messy auto-import
|
||||
callout / blockquoted Description / duplicated Download artifacts."""
|
||||
done = 0
|
||||
owned_slugs = set(libidx)
|
||||
for e in report["results"]["discord_match"]:
|
||||
stem = e["note_stem"]
|
||||
path = os.path.join(HACKS, stem + ".md")
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
rec = build_record(scrape[e["scrape_dir"]], libidx)
|
||||
fm_text, _, _ = read_note(path)
|
||||
fm = parse_fm(fm_text)
|
||||
owned = e["scrape_dir"] in owned_slugs
|
||||
body = build_body(rec, fm, owned=owned, library_note=e["scrape_dir"])
|
||||
content = f"---\n{emit_fm(fm)}\n---\n{body}"
|
||||
out_write(path, content, apply)
|
||||
done += 1
|
||||
print(f"rebuild: {done} Discord note bodies normalized"
|
||||
f"{' (APPLIED)' if apply else ' (dry-run -> .scrape/_preview)'}")
|
||||
|
||||
|
||||
def main():
|
||||
cmds = {"backfill": cmd_backfill, "merge": cmd_merge,
|
||||
"create": cmd_create, "rebuild": cmd_rebuild}
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in cmds:
|
||||
sys.exit(f"usage: enrich_vault.py {{{'|'.join(cmds)}}} [--apply]")
|
||||
apply = "--apply" in sys.argv
|
||||
report = json.load(open(os.path.join(SCRAPE, "_match_report.json"), encoding="utf-8"))
|
||||
scrape = load_scrape()
|
||||
libidx = json.load(open(os.path.join(SCRAPE, "_library_index.json"), encoding="utf-8"))
|
||||
cmds[sys.argv[1]](report, scrape, libidx, apply)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user