pokemon/scripts/enrich_web.py

#!/usr/bin/env python3
"""Apply web-researched facts onto vault notes, idempotently.

Reads .scrape/web_facts.json:
  { "<note stem>": {
      "tagline": str, "summary": str, "developer": str, "version": str,
      "status": "Complete|Ongoing|Beta", "release_date": "YYYY-MM-DD"|"YYYY",
      "base": str, "platform": str, "generation": str, "homepage": str,
      "type": [str], "features": [str], "notability": str } }
All keys optional. Provided scalars OVERRIDE existing frontmatter (web is
authoritative, fixes bad scrape values). Body sections Summary / Features /
Why it stands out are replaced (or inserted after Summary) — re-runnable.

Default dry-run -> .scrape/_preview/. Pass --apply to write the live vault.
"""
from __future__ import annotations
import json, os, re, sys, glob
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import enrich_vault as E

FACTS_GLOB = os.path.join(E.SCRAPE, "web_facts*.json")
SCALAR_FIELDS = ["tagline", "developer", "version", "status", "release_date",
                 "base", "platform", "generation", "homepage", "source", "fakemon",
                 "banner"]
LIST_FIELDS = ["generations"]


def normalize_link_fact(link):
    if isinstance(link, str):
        return "Link", link
    if isinstance(link, dict) and link.get("url"):
        return link.get("label") or "Link", link["url"]
    return None


def merge_links_section(body, links):
    normalized = [x for x in (normalize_link_fact(link) for link in links) if x]
    if not normalized:
        return body
    existing = ""
    match = re.search(r"(?ms)^## Links\s*\n(.*?)(?=^## |\n\[\[Index|\Z)", body)
    if match:
        existing = match.group(1).strip()
    pairs = []
    for line in existing.splitlines():
        url_match = re.search(r"(https?://\S+)", line)
        if not url_match:
            continue
        label_match = re.match(r"^\s*-\s*([^:]+):", line)
        pairs.append(((label_match.group(1).strip() if label_match else "Link"), url_match.group(1).rstrip(".,)")))
    pairs.extend(normalized)
    seen = set()
    lines = []
    for label, url in pairs:
        if url in seen:
            continue
        seen.add(url)
        lines.append(f"- {label}: {url}")
    return set_section(body, "Links", "\n".join(lines))


def facts_sort_key(path: str) -> int:
    match = re.search(r"web_facts(\d*)\.json$", os.path.basename(path))
    if not match or not match.group(1):
        return 0
    return int(match.group(1))


def set_section(body, heading, content):
    """Replace the body under '## heading' (up to next '## ' or footer), or
    insert it. Summary near the top; others after Summary; falling back to
    before Type/Links/In the library/footer."""
    pat = re.compile(rf"(?ms)^## {re.escape(heading)}\s*\n.*?(?=^## |\n\[\[Index|\Z)")
    block = f"## {heading}\n\n{content.rstrip()}\n\n"
    if pat.search(body):
        return pat.sub(block, body, count=1)
    if heading == "Summary":
        m = re.search(r"(?m)^## ", body)
        idx = m.start() if m else len(body)
        return body[:idx] + block + body[idx:]
    msum = re.search(r"(?ms)^## Summary\s*\n.*?(?=^## |\n\[\[Index|\Z)", body)
    if msum:
        idx = msum.end()
        return body[:idx] + block + body[idx:]
    for h in (r"(?m)^## Type", r"(?m)^## Links", r"(?m)^## In the library",
              r"\n\[\[Index"):
        m = re.search(h, body)
        if m:
            return body[:m.start()] + block + body[m.start():]
    return body.rstrip() + "\n\n" + block


def apply_one(stem, facts):
    path = os.path.join(E.HACKS, stem + ".md")
    if not os.path.exists(path):
        return None, "missing"
    fm_text, body, _ = E.read_note(path)
    fm = E.parse_fm(fm_text)

    for k in SCALAR_FIELDS:
        if facts.get(k):
            fm[k] = facts[k]
    for k in LIST_FIELDS:
        if k in facts:
            fm[k] = facts[k]
    if facts.get("type"):
        fm["type"] = facts["type"]
    fm["tags"] = E.derive_tags(fm)

    # keep the body's info callout in sync with the (now-updated) frontmatter
    plat = fm.get("platform", "—")
    callout = (f"> [!info] {', '.join(fm.get('type') or []) or 'ROM hack'} — "
               f"**{fm.get('status', '—')}**\n"
               f"> {E.PLATFORM_LABEL.get(plat, plat)} · base **{fm.get('base', '—')}** · "
               f"version **{fm.get('version', '—')}**"
               + (f" · by **{fm['developer']}**" if fm.get("developer") else ""))
    body = re.sub(r"(?m)^> \[!info\].*(?:\n> .*)*", callout, body, count=1)

    if facts.get("summary"):
        body = set_section(body, "Summary", facts["summary"].strip())
    if facts.get("features"):
        feats = "\n".join(f"- {x.strip()}" for x in facts["features"])
        body = set_section(body, "Features", feats)
    if facts.get("notability"):
        body = set_section(body, "Why it stands out", facts["notability"].strip())
    if facts.get("links"):
        body = merge_links_section(body, facts["links"])

    return f"---\n{E.emit_fm(fm)}\n---\n{body}", "ok"


def main():
    apply = "--apply" in sys.argv
    facts = {}
    for fp in sorted(glob.glob(FACTS_GLOB), key=facts_sort_key):
        facts.update(json.load(open(fp, encoding="utf-8")))
    ok = miss = 0
    for stem, f in facts.items():
        content, st = apply_one(stem, f)
        if st == "missing":
            miss += 1
            print(f"  MISSING note: {stem}")
            continue
        if apply:
            open(os.path.join(E.HACKS, stem + ".md"), "w", encoding="utf-8").write(content)
        else:
            os.makedirs(E.PREVIEW, exist_ok=True)
            open(os.path.join(E.PREVIEW, stem + ".md"), "w", encoding="utf-8").write(content)
        ok += 1
    print(f"enrich_web: {ok} notes {'APPLIED' if apply else 'previewed'}, {miss} missing")


if __name__ == "__main__":
    main()