feat(vault): synthesize Discord scrape + web research into the ROM-hack catalog
Enrich the Obsidian ROM-hack vault from the valhalla Discord scrape
(/storage1/labdata/romhacks/metadata) plus web research, growing the catalog
from 285 to 368 notes and preparing a data export for a wiki site.
Tooling (all dry-run by default, --apply to write; .scrape/ is gitignored):
- scrape_match.py reconcile scrape entries vs notes (43 overlap / 211 match / 83 new)
- enrich_vault.py backfill frontmatter, merge scrape into curated notes,
create 83 new notes, rebuild import bodies to one layout
- enrich_web.py apply hand/web-verified facts from .scrape/web_facts*.json
- build-wiki-data.py export wiki/catalog.json (the site data source)
Results: 368 notes, 327 with served banner art, 332 rich summaries, 200 with
feature lists; new frontmatter developer/release_date/banner/homepage/scrape_dir;
12 flagship hacks web-verified. wiki/SPEC.md describes the site build.
build-vault-mocs.py Index callout updated for the new counts.
This commit is contained in:
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export the Pokémon ROM-hack vault to a single JSON catalog for the website.
|
||||
|
||||
Reads every <VAULT>/Pokémon ROM Hacks/Hacks/*.md note (frontmatter + the
|
||||
Summary / Features / Story / Why-it-stands-out / Links body sections) and emits
|
||||
wiki/catalog.json — the data source a static wiki site can render directly.
|
||||
|
||||
This is READ-ONLY against the vault. Re-run after any enrichment pass.
|
||||
Run: python scripts/build-wiki-data.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json, os, re, glob, sys
|
||||
from collections import Counter
|
||||
from datetime import date
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import enrich_vault as E
|
||||
|
||||
OUT_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "wiki")
|
||||
OUT = os.path.join(OUT_DIR, "catalog.json")
|
||||
|
||||
SCALARS = ["title", "platform", "base", "version", "status", "generation",
|
||||
"developer", "release_date", "banner", "homepage", "source",
|
||||
"library_path", "play_status", "scrape_dir"]
|
||||
|
||||
|
||||
def section(body, heading):
|
||||
m = re.search(rf"(?ms)^## {re.escape(heading)}\s*\n+(.+?)(?=^## |\n\[\[Index|\Z)", body)
|
||||
return m.group(1).strip() if m else ""
|
||||
|
||||
|
||||
def bullets(text):
|
||||
return [re.sub(r"^[-*]\s*", "", ln).strip()
|
||||
for ln in text.splitlines() if ln.strip().startswith(("-", "*"))]
|
||||
|
||||
|
||||
def links(text):
|
||||
out = []
|
||||
for ln in text.splitlines():
|
||||
m = re.search(r"\[?([\w /]+?)\]?:?\s*(https?://\S+)", ln)
|
||||
if m:
|
||||
out.append({"label": m.group(1).strip(" -:"), "url": m.group(2).rstrip(").")})
|
||||
else:
|
||||
m2 = re.search(r"(https?://\S+)", ln)
|
||||
if m2:
|
||||
out.append({"label": "Link", "url": m2.group(1).rstrip(").")})
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
hacks = []
|
||||
for f in sorted(glob.glob(os.path.join(E.HACKS, "*.md"))):
|
||||
fm_text, body, _ = E.read_note(f)
|
||||
fm = E.parse_fm(fm_text)
|
||||
stem = os.path.splitext(os.path.basename(f))[0]
|
||||
rec = {"stem": stem}
|
||||
for k in SCALARS:
|
||||
if fm.get(k) not in (None, ""):
|
||||
rec[k] = fm[k]
|
||||
rec["type"] = fm.get("type") or []
|
||||
rec["tags"] = fm.get("tags") or []
|
||||
if fm.get("rating") not in (None, ""):
|
||||
try:
|
||||
rec["rating"] = int(str(fm["rating"]))
|
||||
except ValueError:
|
||||
pass
|
||||
rec["summary"] = section(body, "Summary")
|
||||
rec["notability"] = section(body, "Why it stands out")
|
||||
rec["story"] = section(body, "Story")
|
||||
rec["features"] = bullets(section(body, "Features"))
|
||||
rec["links"] = links(section(body, "Links"))
|
||||
for k in ("base", "version", "status", "platform", "generation"):
|
||||
if rec.get(k) == "—":
|
||||
rec[k] = None
|
||||
hacks.append(rec)
|
||||
|
||||
def facet(key):
|
||||
c = Counter(h.get(key) or "Unknown" for h in hacks)
|
||||
return dict(sorted(c.items(), key=lambda kv: (-kv[1], kv[0])))
|
||||
|
||||
type_counts = Counter(t for h in hacks for t in h["type"])
|
||||
out = {
|
||||
"generated": date.today().isoformat(),
|
||||
"count": len(hacks),
|
||||
"facets": {
|
||||
"platform": facet("platform"),
|
||||
"base": facet("base"),
|
||||
"status": facet("status"),
|
||||
"type": dict(sorted(type_counts.items(), key=lambda kv: (-kv[1], kv[0]))),
|
||||
"play_status": facet("play_status"),
|
||||
},
|
||||
"image_base": "https://romhacks-files.ginnoir.com",
|
||||
"hacks": hacks,
|
||||
}
|
||||
os.makedirs(OUT_DIR, exist_ok=True)
|
||||
json.dump(out, open(OUT, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
|
||||
|
||||
enriched = sum(1 for h in hacks if len(h["summary"]) > 60)
|
||||
withimg = sum(1 for h in hacks if h.get("banner"))
|
||||
withfeat = sum(1 for h in hacks if h["features"])
|
||||
print(f"wrote {OUT}")
|
||||
print(f" hacks={len(hacks)} with-image={withimg} with-features={withfeat} "
|
||||
f"rich-summary={enriched}")
|
||||
print(f" platforms: {out['facets']['platform']}")
|
||||
print(f" types: {out['facets']['type']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user