ginnoir
dd26aafc42
Enrich the Obsidian ROM-hack vault from the valhalla Discord scrape
(/storage1/labdata/romhacks/metadata) plus web research, growing the catalog
from 285 to 368 notes and preparing a data export for a wiki site.
Tooling (all dry-run by default, --apply to write; .scrape/ is gitignored):
- scrape_match.py reconcile scrape entries vs notes (43 overlap / 211 match / 83 new)
- enrich_vault.py backfill frontmatter, merge scrape into curated notes,
create 83 new notes, rebuild import bodies to one layout
- enrich_web.py apply hand/web-verified facts from .scrape/web_facts*.json
- build-wiki-data.py export wiki/catalog.json (the site data source)
Results: 368 notes, 327 with served banner art, 332 rich summaries, 200 with
feature lists; new frontmatter developer/release_date/banner/homepage/scrape_dir;
12 flagship hacks web-verified. wiki/SPEC.md describes the site build.
build-vault-mocs.py Index callout updated for the new counts.
107 lines
4.2 KiB
Python
107 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Apply web-researched facts onto vault notes, idempotently.
|
|
|
|
Reads .scrape/web_facts.json:
|
|
{ "<note stem>": {
|
|
"tagline": str, "summary": str, "developer": str, "version": str,
|
|
"status": "Complete|Ongoing|Beta", "release_date": "YYYY-MM-DD"|"YYYY",
|
|
"base": str, "platform": str, "generation": str, "homepage": str,
|
|
"type": [str], "features": [str], "notability": str } }
|
|
All keys optional. Provided scalars OVERRIDE existing frontmatter (web is
|
|
authoritative, fixes bad scrape values). Body sections Summary / Features /
|
|
Why it stands out are replaced (or inserted after Summary) — re-runnable.
|
|
|
|
Default dry-run -> .scrape/_preview/. Pass --apply to write the live vault.
|
|
"""
|
|
from __future__ import annotations
|
|
import json, os, re, sys, glob
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import enrich_vault as E
|
|
|
|
FACTS_GLOB = os.path.join(E.SCRAPE, "web_facts*.json")
|
|
SCALAR_FIELDS = ["tagline", "developer", "version", "status", "release_date",
|
|
"base", "platform", "generation", "homepage"]
|
|
|
|
|
|
def set_section(body, heading, content):
|
|
"""Replace the body under '## heading' (up to next '## ' or footer), or
|
|
insert it. Summary near the top; others after Summary; falling back to
|
|
before Type/Links/In the library/footer."""
|
|
pat = re.compile(rf"(?ms)^## {re.escape(heading)}\s*\n.*?(?=^## |\n\[\[Index|\Z)")
|
|
block = f"## {heading}\n\n{content.rstrip()}\n\n"
|
|
if pat.search(body):
|
|
return pat.sub(block, body, count=1)
|
|
if heading == "Summary":
|
|
m = re.search(r"(?m)^## ", body)
|
|
idx = m.start() if m else len(body)
|
|
return body[:idx] + block + body[idx:]
|
|
msum = re.search(r"(?ms)^## Summary\s*\n.*?(?=^## |\n\[\[Index|\Z)", body)
|
|
if msum:
|
|
idx = msum.end()
|
|
return body[:idx] + block + body[idx:]
|
|
for h in (r"(?m)^## Type", r"(?m)^## Links", r"(?m)^## In the library",
|
|
r"\n\[\[Index"):
|
|
m = re.search(h, body)
|
|
if m:
|
|
return body[:m.start()] + block + body[m.start():]
|
|
return body.rstrip() + "\n\n" + block
|
|
|
|
|
|
def apply_one(stem, facts):
|
|
path = os.path.join(E.HACKS, stem + ".md")
|
|
if not os.path.exists(path):
|
|
return None, "missing"
|
|
fm_text, body, _ = E.read_note(path)
|
|
fm = E.parse_fm(fm_text)
|
|
|
|
for k in SCALAR_FIELDS:
|
|
if facts.get(k):
|
|
fm[k] = facts[k]
|
|
if facts.get("type"):
|
|
fm["type"] = facts["type"]
|
|
fm["tags"] = E.derive_tags(fm)
|
|
|
|
# keep the body's info callout in sync with the (now-updated) frontmatter
|
|
plat = fm.get("platform", "—")
|
|
callout = (f"> [!info] {', '.join(fm.get('type') or []) or 'ROM hack'} — "
|
|
f"**{fm.get('status', '—')}**\n"
|
|
f"> {E.PLATFORM_LABEL.get(plat, plat)} · base **{fm.get('base', '—')}** · "
|
|
f"version **{fm.get('version', '—')}**"
|
|
+ (f" · by **{fm['developer']}**" if fm.get("developer") else ""))
|
|
body = re.sub(r"(?m)^> \[!info\].*(?:\n> .*)*", callout, body, count=1)
|
|
|
|
if facts.get("summary"):
|
|
body = set_section(body, "Summary", facts["summary"].strip())
|
|
if facts.get("features"):
|
|
feats = "\n".join(f"- {x.strip()}" for x in facts["features"])
|
|
body = set_section(body, "Features", feats)
|
|
if facts.get("notability"):
|
|
body = set_section(body, "Why it stands out", facts["notability"].strip())
|
|
|
|
return f"---\n{E.emit_fm(fm)}\n---\n{body}", "ok"
|
|
|
|
|
|
def main():
|
|
apply = "--apply" in sys.argv
|
|
facts = {}
|
|
for fp in sorted(glob.glob(FACTS_GLOB)):
|
|
facts.update(json.load(open(fp, encoding="utf-8")))
|
|
ok = miss = 0
|
|
for stem, f in facts.items():
|
|
content, st = apply_one(stem, f)
|
|
if st == "missing":
|
|
miss += 1
|
|
print(f" MISSING note: {stem}")
|
|
continue
|
|
if apply:
|
|
open(os.path.join(E.HACKS, stem + ".md"), "w", encoding="utf-8").write(content)
|
|
else:
|
|
os.makedirs(E.PREVIEW, exist_ok=True)
|
|
open(os.path.join(E.PREVIEW, stem + ".md"), "w", encoding="utf-8").write(content)
|
|
ok += 1
|
|
print(f"enrich_web: {ok} notes {'APPLIED' if apply else 'previewed'}, {miss} missing")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|