feat(vault): synthesize Discord scrape + web research into the ROM-hack catalog
Enrich the Obsidian ROM-hack vault from the valhalla Discord scrape
(/storage1/labdata/romhacks/metadata) plus web research, growing the catalog
from 285 to 368 notes and preparing a data export for a wiki site.
Tooling (all dry-run by default, --apply to write; .scrape/ is gitignored):
- scrape_match.py reconcile scrape entries vs notes (43 overlap / 211 match / 83 new)
- enrich_vault.py backfill frontmatter, merge scrape into curated notes,
create 83 new notes, rebuild import bodies to one layout
- enrich_web.py apply hand/web-verified facts from .scrape/web_facts*.json
- build-wiki-data.py export wiki/catalog.json (the site data source)
Results: 368 notes, 327 with served banner art, 332 rich summaries, 200 with
feature lists; new frontmatter developer/release_date/banner/homepage/scrape_dir;
12 flagship hacks web-verified. wiki/SPEC.md describes the site build.
build-vault-mocs.py Index callout updated for the new counts.
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply web-researched facts onto vault notes, idempotently.
|
||||
|
||||
Reads .scrape/web_facts.json:
|
||||
{ "<note stem>": {
|
||||
"tagline": str, "summary": str, "developer": str, "version": str,
|
||||
"status": "Complete|Ongoing|Beta", "release_date": "YYYY-MM-DD"|"YYYY",
|
||||
"base": str, "platform": str, "generation": str, "homepage": str,
|
||||
"type": [str], "features": [str], "notability": str } }
|
||||
All keys optional. Provided scalars OVERRIDE existing frontmatter (web is
|
||||
authoritative, fixes bad scrape values). Body sections Summary / Features /
|
||||
Why it stands out are replaced (or inserted after Summary) — re-runnable.
|
||||
|
||||
Default dry-run -> .scrape/_preview/. Pass --apply to write the live vault.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json, os, re, sys, glob
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import enrich_vault as E
|
||||
|
||||
FACTS_GLOB = os.path.join(E.SCRAPE, "web_facts*.json")
|
||||
SCALAR_FIELDS = ["tagline", "developer", "version", "status", "release_date",
|
||||
"base", "platform", "generation", "homepage"]
|
||||
|
||||
|
||||
def set_section(body, heading, content):
|
||||
"""Replace the body under '## heading' (up to next '## ' or footer), or
|
||||
insert it. Summary near the top; others after Summary; falling back to
|
||||
before Type/Links/In the library/footer."""
|
||||
pat = re.compile(rf"(?ms)^## {re.escape(heading)}\s*\n.*?(?=^## |\n\[\[Index|\Z)")
|
||||
block = f"## {heading}\n\n{content.rstrip()}\n\n"
|
||||
if pat.search(body):
|
||||
return pat.sub(block, body, count=1)
|
||||
if heading == "Summary":
|
||||
m = re.search(r"(?m)^## ", body)
|
||||
idx = m.start() if m else len(body)
|
||||
return body[:idx] + block + body[idx:]
|
||||
msum = re.search(r"(?ms)^## Summary\s*\n.*?(?=^## |\n\[\[Index|\Z)", body)
|
||||
if msum:
|
||||
idx = msum.end()
|
||||
return body[:idx] + block + body[idx:]
|
||||
for h in (r"(?m)^## Type", r"(?m)^## Links", r"(?m)^## In the library",
|
||||
r"\n\[\[Index"):
|
||||
m = re.search(h, body)
|
||||
if m:
|
||||
return body[:m.start()] + block + body[m.start():]
|
||||
return body.rstrip() + "\n\n" + block
|
||||
|
||||
|
||||
def apply_one(stem, facts):
|
||||
path = os.path.join(E.HACKS, stem + ".md")
|
||||
if not os.path.exists(path):
|
||||
return None, "missing"
|
||||
fm_text, body, _ = E.read_note(path)
|
||||
fm = E.parse_fm(fm_text)
|
||||
|
||||
for k in SCALAR_FIELDS:
|
||||
if facts.get(k):
|
||||
fm[k] = facts[k]
|
||||
if facts.get("type"):
|
||||
fm["type"] = facts["type"]
|
||||
fm["tags"] = E.derive_tags(fm)
|
||||
|
||||
# keep the body's info callout in sync with the (now-updated) frontmatter
|
||||
plat = fm.get("platform", "—")
|
||||
callout = (f"> [!info] {', '.join(fm.get('type') or []) or 'ROM hack'} — "
|
||||
f"**{fm.get('status', '—')}**\n"
|
||||
f"> {E.PLATFORM_LABEL.get(plat, plat)} · base **{fm.get('base', '—')}** · "
|
||||
f"version **{fm.get('version', '—')}**"
|
||||
+ (f" · by **{fm['developer']}**" if fm.get("developer") else ""))
|
||||
body = re.sub(r"(?m)^> \[!info\].*(?:\n> .*)*", callout, body, count=1)
|
||||
|
||||
if facts.get("summary"):
|
||||
body = set_section(body, "Summary", facts["summary"].strip())
|
||||
if facts.get("features"):
|
||||
feats = "\n".join(f"- {x.strip()}" for x in facts["features"])
|
||||
body = set_section(body, "Features", feats)
|
||||
if facts.get("notability"):
|
||||
body = set_section(body, "Why it stands out", facts["notability"].strip())
|
||||
|
||||
return f"---\n{E.emit_fm(fm)}\n---\n{body}", "ok"
|
||||
|
||||
|
||||
def main():
|
||||
apply = "--apply" in sys.argv
|
||||
facts = {}
|
||||
for fp in sorted(glob.glob(FACTS_GLOB)):
|
||||
facts.update(json.load(open(fp, encoding="utf-8")))
|
||||
ok = miss = 0
|
||||
for stem, f in facts.items():
|
||||
content, st = apply_one(stem, f)
|
||||
if st == "missing":
|
||||
miss += 1
|
||||
print(f" MISSING note: {stem}")
|
||||
continue
|
||||
if apply:
|
||||
open(os.path.join(E.HACKS, stem + ".md"), "w", encoding="utf-8").write(content)
|
||||
else:
|
||||
os.makedirs(E.PREVIEW, exist_ok=True)
|
||||
open(os.path.join(E.PREVIEW, stem + ".md"), "w", encoding="utf-8").write(content)
|
||||
ok += 1
|
||||
print(f"enrich_web: {ok} notes {'APPLIED' if apply else 'previewed'}, {miss} missing")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user