ginnoir
44183fb0fb
Adds standardize-vault-pages pipeline, exports intentional empty generations, and refreshes the 396-hack catalog with full banner coverage, status classification, and Fakemon generation tags.
760 lines
30 KiB
Python
760 lines
30 KiB
Python
#!/usr/bin/env python3
|
||
"""Enrich the Pokémon ROM-hack vault from the Discord scrape (.scrape/*/metadata.json).
|
||
|
||
Three subcommands, each --dry-run by default (writes previews to .scrape/_preview/)
|
||
and only touches the live vault with --apply:
|
||
|
||
backfill 211 Discord-import notes: fill missing frontmatter (base, version,
|
||
status, type, developer, release_date, banner, scrape_dir) parsed
|
||
from the scrape. Body is preserved verbatim.
|
||
merge 43 curated-overlap notes: keep the hand-written note, add a banner,
|
||
append Story/Features/community Links parsed from the scrape, and
|
||
add developer/release_date/banner/scrape_dir frontmatter. The curated
|
||
Summary and existing frontmatter are preserved.
|
||
create 83 unmatched scraped hacks: write brand-new wiki-schema notes.
|
||
|
||
Source of truth for matching = .scrape/_match_report.json (scrape_match.py).
|
||
Art is served at https://romhacks-files.ginnoir.com/_meta/<scrape_dir>/<file>.
|
||
"""
|
||
from __future__ import annotations
|
||
import json, os, re, sys, glob, unicodedata
|
||
from datetime import date
|
||
from urllib.parse import quote
|
||
|
||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
VAULT = r"C:\Users\MattC\Documents\Obsidian Vault"
|
||
HACKS = os.path.join(VAULT, "Pokémon ROM Hacks", "Hacks")
|
||
SCRAPE = os.path.join(ROOT, ".scrape")
|
||
PREVIEW = os.path.join(SCRAPE, "_preview")
|
||
ART_BASE = "https://romhacks-files.ginnoir.com/_meta"
|
||
TODAY = date.today().isoformat()
|
||
|
||
# ---------------------------------------------------------------- text cleaning
|
||
|
||
def clean(s: str) -> str:
|
||
if not s:
|
||
return ""
|
||
s = s.replace("", "").replace("", "")
|
||
s = s.replace("\xa0", " ").replace("’", "'").replace("‘", "'")
|
||
s = s.replace("“", '"').replace("”", '"').replace("–", "-")
|
||
s = re.sub(r"[ \t]+", " ", s)
|
||
s = re.sub(r"\n{3,}", "\n\n", s)
|
||
return s.strip()
|
||
|
||
|
||
# ---------------------------------------------------------------- parsing
|
||
|
||
MONTHS = {m: i for i, m in enumerate(
|
||
["january", "february", "march", "april", "may", "june", "july",
|
||
"august", "september", "october", "november", "december"], 1)}
|
||
|
||
PLATFORM_HINTS = [
|
||
(r"\bGBA\b|game boy advance", ("GBA", "console")),
|
||
(r"\bGBC\b|game boy color", ("GBC", "console")),
|
||
(r"\bNDS\b|nintendo ds\b|\bDS ROM", ("NDS", "console")),
|
||
(r"\b3DS\b", ("3DS", "console")),
|
||
(r"\bN64\b|Nintendo 64|Pokemon Stadium|Pokémon Stadium", ("N64", "console")),
|
||
(r"\bGameCube\b|Nintendo GameCube|Pokemon Colosseum|Pokémon Colosseum", ("GameCube", "console")),
|
||
(r"\bSwitch\b|Nintendo Switch", ("Switch", "console")),
|
||
(r"\bGB\b|game boy(?! advance| color)", ("GB", "console")),
|
||
(r"RPGXP|RPG ?Maker|Essentials|FanGame|fan game|GameMaker", ("PC", "fangame")),
|
||
]
|
||
|
||
BASE_CANON = {
|
||
"fire red": "FireRed", "firered": "FireRed", "fire-red": "FireRed",
|
||
"leaf green": "LeafGreen", "leafgreen": "LeafGreen",
|
||
"fire red/leaf green": "FireRed", "firered/leafgreen": "FireRed",
|
||
"emerald": "Emerald", "ruby": "Ruby", "sapphire": "Sapphire",
|
||
"ruby/sapphire": "Ruby",
|
||
"crystal": "Crystal", "gold": "Gold", "silver": "Silver",
|
||
"gold/silver": "Gold", "red": "Red", "blue": "Blue", "yellow": "Yellow",
|
||
"diamond": "Diamond", "pearl": "Pearl", "platinum": "Platinum",
|
||
"heart gold": "HeartGold", "heartgold": "HeartGold",
|
||
"soul silver": "SoulSilver", "soulsilver": "SoulSilver",
|
||
"black": "Black", "white": "White", "black 2": "Black 2",
|
||
"white 2": "White 2", "black2": "Black 2", "white2": "White 2",
|
||
}
|
||
|
||
SECTION_ALIASES = {
|
||
"description": "Description", "about": "Description", "overview": "Description",
|
||
"plot": "Story", "story": "Story", "synopsis": "Story",
|
||
"features": "Features", "feature": "Features", "key features": "Features",
|
||
"download": "Download", "downloads": "Download", "rom link": "Download",
|
||
"changelog": "Changelog", "credits": "Credits",
|
||
}
|
||
|
||
|
||
def norm_key(name: str) -> str:
|
||
name = unicodedata.normalize("NFKD", name)
|
||
name = "".join(c for c in name if not unicodedata.combining(c)).lower()
|
||
name = re.sub(r"\bpok[e]?mon\b", " ", name)
|
||
return re.sub(r"[^a-z0-9]+", "", name)
|
||
|
||
|
||
def parse_release(text: str):
|
||
m = re.search(r"(?:last updated on|released on|updated on|released)\s+"
|
||
r"([A-Za-z]+)\s+(\d{1,2}),?\s*(\d{4})", text, re.I)
|
||
if m:
|
||
mo = MONTHS.get(m.group(1).lower())
|
||
if mo:
|
||
return f"{m.group(3)}-{mo:02d}-{int(m.group(2)):02d}"
|
||
m = re.search(r"\b(20\d{2})\b", text)
|
||
return m.group(1) if m else None
|
||
|
||
|
||
def parse_intro(text: str) -> dict:
|
||
out: dict = {}
|
||
head = text[:400]
|
||
m = re.search(r"\bby\s+(.+?)(?:\s+based on|\s+made using|\s+using|\s+in English|"
|
||
r"\.\s|\,\s|\s+and it| \(|\n)", head)
|
||
if m:
|
||
dev = m.group(1).strip(" .,-")
|
||
if 1 < len(dev) < 60 and not dev.lower().startswith(("the ", "a ")):
|
||
out["developer"] = dev
|
||
m = re.search(r"based on\s+(?:Pok[eé]mon\s+)?([A-Za-z0-9][A-Za-z0-9 /&]+?)"
|
||
r"(?:\.|,|\n| in | with |!)", head, re.I)
|
||
if m:
|
||
raw = m.group(1).strip().lower()
|
||
raw = re.sub(r"\bversions?\b", "", raw).strip()
|
||
out["base"] = BASE_CANON.get(raw, m.group(1).strip())
|
||
for pat, (plat, kind) in PLATFORM_HINTS:
|
||
if re.search(pat, head, re.I):
|
||
out["platform"], out["kind"] = plat, kind
|
||
break
|
||
return out
|
||
|
||
|
||
# inline / line key-value metadata blocks, e.g. "**Creator:** X **Type:** GBA
|
||
# **Hack of:** FireRed **Version:** Final **Last Updated:** May 26, 2020".
|
||
KV_KEYS = (r"Creator|Developer|Dev|Author|Made by|Type of ROM|Type|Platform|"
|
||
r"System|Hack of|Base ROM|Base game|Base|Based on|Version|Status|"
|
||
r"Last Updated|Updated|Release Date|Released|Date|Language|Genre")
|
||
KV_RE = re.compile(rf"(?i)\b(?P<key>{KV_KEYS})\s*[:\-]\s*(?P<val>.+?)"
|
||
rf"(?=\s{{2,}}|\s*\b(?:{KV_KEYS})\b\s*[:\-]|$)", re.M)
|
||
KV_LINE = re.compile(rf"(?i)^\s*(?:{KV_KEYS})\s*[:\-]")
|
||
|
||
|
||
def parse_kv(text: str) -> dict:
|
||
"""Extract dev/platform/base/version/status/release from key-value blocks
|
||
(bold or plain). De-bold first so '**Creator:** X' -> 'Creator: X'."""
|
||
flat = text.replace("*", "")
|
||
got: dict[str, str] = {}
|
||
for m in KV_RE.finditer(flat):
|
||
k = m.group("key").lower().strip()
|
||
v = m.group("val").strip(" .,-")
|
||
if not v or len(v) > 80:
|
||
continue
|
||
got.setdefault(k, v)
|
||
out = {}
|
||
def first(*keys):
|
||
for k in keys:
|
||
if k in got:
|
||
return got[k]
|
||
return None
|
||
dev = first("creator", "developer", "dev", "author", "made by")
|
||
if dev:
|
||
out["developer"] = dev
|
||
typ = first("type of rom", "type", "platform", "system")
|
||
if typ:
|
||
for pat, (plat, kind) in PLATFORM_HINTS:
|
||
if re.search(pat, typ, re.I):
|
||
out["platform"], out["kind"] = plat, kind
|
||
break
|
||
base = first("hack of", "base rom", "base game", "base", "based on")
|
||
if base:
|
||
bl = re.sub(r"\bversions?\b", "", base.lower()).strip()
|
||
out["base"] = BASE_CANON.get(bl, base.strip())
|
||
if first("version"):
|
||
out["version"] = first("version")
|
||
if first("status"):
|
||
out["status"] = first("status")
|
||
if first("language"):
|
||
out["language"] = first("language")
|
||
rel = first("last updated", "updated", "release date", "released", "date")
|
||
if rel:
|
||
pr = parse_release(rel) or parse_release("released " + rel)
|
||
if pr:
|
||
out["release_date"] = pr
|
||
return out
|
||
|
||
|
||
def split_sections(text: str, name: str = ""):
|
||
lines = text.split("\n")
|
||
sections: dict[str, list[str]] = {}
|
||
lead: list[str] = []
|
||
cur = None
|
||
name_key = norm_key(name)
|
||
for raw in lines:
|
||
line = re.sub(r"^>\s?", "", raw.strip())
|
||
hkey = re.sub(r"[*#:_~`]", "", line).strip().lower()
|
||
if hkey in SECTION_ALIASES and len(line) < 40:
|
||
cur = SECTION_ALIASES[hkey]
|
||
sections.setdefault(cur, [])
|
||
continue
|
||
flat = line.replace("*", "")
|
||
low = flat.lower()
|
||
is_meta = (re.match(r"^pok[e]?mon", low) and (" is a" in low or " is an" in low)) \
|
||
or bool(KV_LINE.match(flat)) \
|
||
or low.startswith(("rom link", "- rom link", "it was last updated",
|
||
"made using", "by ")) \
|
||
or (name_key and norm_key(line) == name_key) # bare title repeat
|
||
if cur:
|
||
sections[cur].append(line)
|
||
elif not is_meta and line:
|
||
lead.append(line)
|
||
feats = []
|
||
for ln in sections.get("Features", []):
|
||
s = ln.strip().lstrip("-*•").strip()
|
||
if s and not s.lower().startswith("download"):
|
||
feats.append(s)
|
||
lead_text = " ".join(lead).strip()
|
||
return lead_text, {k: clean("\n".join(v)) for k, v in sections.items()}, feats
|
||
|
||
|
||
def collect_links(meta: dict, text: str):
|
||
urls = list(meta.get("links") or [])
|
||
for m in re.finditer(r"https?://[^\s)\]>'\"]+", text):
|
||
u = m.group(0).rstrip(".,)")
|
||
if u not in urls:
|
||
urls.append(u)
|
||
seen, out = set(), []
|
||
for u in urls:
|
||
if u not in seen:
|
||
seen.add(u); out.append(u)
|
||
return out
|
||
|
||
|
||
TYPE_RULES = [
|
||
("Difficulty", r"kaizo|difficulty hack|harder|increased difficulty|"
|
||
r"competitive|nuzlocke|challenging|boosted ai|smart ai"),
|
||
("Expansion", r"mega evolution|national dex|gen 8|gen 9|generation 8|"
|
||
r"generation 9|physical/special split|fairy type|decomp|"
|
||
r"expansion|all 8\d\d"),
|
||
("New Experience", r"new region|new story|original story|fakemon|"
|
||
r"new pok[e]?mon|custom region|brand new"),
|
||
("Roguelite", r"roguelite|roguelike|run-based|permadeath|randomized run"),
|
||
("Demake", r"demake|de-make|in gen ?[12] style|on gbc|1-bit"),
|
||
("Cosmetic", r"moemon|sprite pack|tileset|cosmetic|visual overhaul|reskin"),
|
||
("QoL", r"quality of life|quality-of-life|\bqol\b|reusable tms|exp share"),
|
||
]
|
||
|
||
|
||
def infer_types(text: str) -> list[str]:
|
||
low = text.lower()
|
||
hits = [name for name, pat in TYPE_RULES if re.search(pat, low)]
|
||
return hits[:2]
|
||
|
||
|
||
# base game -> (platform, canonical base) for name/desc heuristic fallback
|
||
GAME_PLATFORM = {
|
||
"emerald": ("GBA", "Emerald"), "fire red": ("GBA", "FireRed"),
|
||
"firered": ("GBA", "FireRed"), "leaf green": ("GBA", "LeafGreen"),
|
||
"leafgreen": ("GBA", "LeafGreen"), "ruby": ("GBA", "Ruby"),
|
||
"sapphire": ("GBA", "Sapphire"),
|
||
"crystal": ("GBC", "Crystal"), "gold": ("GBC", "Gold"),
|
||
"silver": ("GBC", "Silver"),
|
||
"diamond": ("NDS", "Diamond"), "pearl": ("NDS", "Pearl"),
|
||
"platinum": ("NDS", "Platinum"), "heartgold": ("NDS", "HeartGold"),
|
||
"soulsilver": ("NDS", "SoulSilver"), "heart gold": ("NDS", "HeartGold"),
|
||
"soul silver": ("NDS", "SoulSilver"), "black": ("NDS", "Black"),
|
||
"white": ("NDS", "White"), "black 2": ("NDS", "Black 2"),
|
||
"white 2": ("NDS", "White 2"),
|
||
"stadium": ("N64", "Stadium"), "pokemon stadium": ("N64", "Stadium"),
|
||
"pokémon stadium": ("N64", "Stadium"),
|
||
}
|
||
|
||
|
||
def infer_platform_base(name, desc, platform, base):
|
||
"""Fallback: if platform unknown, infer GBA/GBC/GB/NDS from a base game word
|
||
in the name or first part of the description. Never overrides a known value."""
|
||
if platform and base:
|
||
return platform, base
|
||
hay = (name + " " + desc[:300]).lower()
|
||
# if we already know base, set platform from it
|
||
if base and base != "—" and not platform:
|
||
bp = GAME_PLATFORM.get(base.lower())
|
||
if bp:
|
||
return bp[0], base
|
||
if not platform:
|
||
for word, (plat, canon) in GAME_PLATFORM.items():
|
||
if re.search(rf"\b{re.escape(word)}\b", hay):
|
||
return plat, (base if base and base != "—" else canon)
|
||
return platform, base
|
||
|
||
|
||
LEAD_PREFIX = re.compile(
|
||
r"^\s*(?:version|status|update[d]?)\b\s*[:\-]?\s*"
|
||
r"(?:completed|complete|beta|ongoing|final|released?)?\s*"
|
||
r"(?:v?\d[\d.\w]*)?\s*[:\-]?\s*", re.I)
|
||
|
||
|
||
def clean_lead(lead: str):
|
||
"""Strip a leading 'Version <x>'/'Completed' label off a lead line and
|
||
return (cleaned_lead, version, status) harvested from it."""
|
||
ver = status = None
|
||
mv = re.match(r"^\s*(?:version|ver)\b\s*[:\-]?\s*(?:completed|complete|"
|
||
r"beta|ongoing|final)?\s*(v?\d[\d.\w]*)", lead, re.I)
|
||
if mv:
|
||
ver = mv.group(1)
|
||
ms = re.match(r"^\s*(?:status\b\s*[:\-]?\s*)?(completed|complete|beta|ongoing)",
|
||
lead, re.I)
|
||
if ms:
|
||
status = ms.group(1)
|
||
new = LEAD_PREFIX.sub("", lead)
|
||
# a second pass for 'Version completed 1.3.1' style double-prefix
|
||
new = LEAD_PREFIX.sub("", new).strip(" :-")
|
||
if len(new) < 25:
|
||
new = ""
|
||
return new, ver, status
|
||
|
||
|
||
def build_record(meta: dict, libidx: dict) -> dict:
|
||
name = clean(meta.get("name", ""))
|
||
desc = clean(meta.get("description", ""))
|
||
scrape_dir = meta["_dir"]
|
||
li = libidx.get(scrape_dir, {})
|
||
intro = parse_intro(desc)
|
||
kv = parse_kv(desc)
|
||
lead, sections, feats = split_sections(desc, name)
|
||
rec = {
|
||
"name": name,
|
||
"scrape_dir": scrape_dir,
|
||
"platform": (li.get("platform") or intro.get("platform")
|
||
or kv.get("platform") or "").upper() or None,
|
||
"kind": li.get("kind") or intro.get("kind") or kv.get("kind"),
|
||
"base": intro.get("base") or kv.get("base"),
|
||
"developer": intro.get("developer") or kv.get("developer"),
|
||
"release_date": kv.get("release_date") or parse_release(desc),
|
||
"version": kv.get("version"),
|
||
"status": kv.get("status"),
|
||
"lead": lead,
|
||
"story": sections.get("Story", ""),
|
||
"description_body": sections.get("Description", ""),
|
||
"features": feats,
|
||
"links": [u for u in collect_links(meta, desc) if not u.startswith(ART_BASE)],
|
||
"art": (meta.get("art") or [None])[0],
|
||
"types": infer_types(desc),
|
||
}
|
||
# tidy the lead: strip leading 'Version x'/'Completed' labels, fall back to
|
||
# the Description section, harvest version/status if still unknown.
|
||
cl, lv, ls = clean_lead(rec["lead"])
|
||
rec["lead"] = cl or sections.get("Description", "")[:600]
|
||
if not rec["version"] and lv:
|
||
rec["version"] = lv
|
||
if not rec["status"] and ls:
|
||
rec["status"] = ls
|
||
if rec["platform"] == "PC" and not rec["base"]:
|
||
rec["base"] = "—"
|
||
if rec["platform"] != "PC":
|
||
p, b = infer_platform_base(name, desc, rec["platform"], rec["base"])
|
||
rec["platform"], rec["base"] = p, b
|
||
return rec
|
||
|
||
|
||
# ---------------------------------------------------------------- frontmatter io
|
||
|
||
def read_note(path):
|
||
text = open(path, encoding="utf-8").read()
|
||
m = re.match(r"^---\n(.*?)\n---\n?(.*)$", text, re.S)
|
||
if not m:
|
||
return "", text, text
|
||
return m.group(1), m.group(2), text
|
||
|
||
|
||
def fm_get(fm_text, key):
|
||
m = re.search(rf"(?im)^{re.escape(key)}:\s*(.*)$", fm_text)
|
||
return m.group(1).strip().strip('"') if m else None
|
||
|
||
|
||
def fm_set(fm_text, key, value, quote_it=True):
|
||
v = f'"{value}"' if quote_it and not str(value).startswith(("[", '"')) else value
|
||
line = f"{key}: {v}"
|
||
if re.search(rf"(?im)^{re.escape(key)}:\s*.*$", fm_text):
|
||
return re.sub(rf"(?im)^{re.escape(key)}:\s*.*$", line, fm_text, count=1)
|
||
return fm_text.rstrip() + "\n" + line
|
||
|
||
|
||
def art_url(rec):
|
||
if rec.get("art"):
|
||
return f"{ART_BASE}/{rec['scrape_dir']}/{quote(rec['art'])}"
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------- loaders
|
||
|
||
def load_scrape():
|
||
out = {}
|
||
for jf in glob.glob(os.path.join(SCRAPE, "*", "metadata.json")):
|
||
m = json.load(open(jf, encoding="utf-8"))
|
||
m["_dir"] = os.path.basename(os.path.dirname(jf))
|
||
out[m["_dir"]] = m
|
||
return out
|
||
|
||
|
||
# ---------------------------------------------------------------- frontmatter (de)serialize
|
||
|
||
FM_ORDER = ["title", "platform", "base", "version", "status", "type",
|
||
"generation", "generations", "fakemon", "developer", "release_date", "banner",
|
||
"library_path", "source", "homepage", "scrape_dir",
|
||
"added", "play_status", "rating", "tags"]
|
||
LIST_KEYS = {"type", "tags"}
|
||
|
||
|
||
def parse_fm(fm_text):
|
||
"""Parse simple block-style frontmatter into an ordered dict.
|
||
Scalars -> str; block/flow lists -> list[str]."""
|
||
fm, lines, i = {}, fm_text.split("\n"), 0
|
||
while i < len(lines):
|
||
km = re.match(r"^([\w]+):\s*(.*)$", lines[i])
|
||
if km:
|
||
k, v = km.group(1), km.group(2)
|
||
if v == "" and i + 1 < len(lines) and re.match(r"^\s*-\s", lines[i + 1]):
|
||
items = []
|
||
while i + 1 < len(lines) and re.match(r"^\s*-\s", lines[i + 1]):
|
||
items.append(lines[i + 1].strip()[2:].strip().strip('"'))
|
||
i += 1
|
||
fm[k] = items
|
||
elif v.strip().startswith("[") and v.strip().endswith("]"):
|
||
inner = v.strip()[1:-1].strip()
|
||
fm[k] = [x.strip().strip('"') for x in inner.split(",")] if inner else []
|
||
else:
|
||
fm[k] = v.strip().strip('"')
|
||
i += 1
|
||
return fm
|
||
|
||
|
||
def emit_fm(fm):
|
||
out = []
|
||
keys = [k for k in FM_ORDER if k in fm] + [k for k in fm if k not in FM_ORDER]
|
||
for k in keys:
|
||
v = fm[k]
|
||
if k in LIST_KEYS or isinstance(v, list):
|
||
v = v or []
|
||
if not v:
|
||
out.append(f"{k}: []")
|
||
else:
|
||
out.append(f"{k}:")
|
||
out.extend(f" - {x}" for x in v)
|
||
elif k == "rating":
|
||
out.append(f"{k}: {v}")
|
||
elif k == "added" and re.match(r"^\d{4}-\d{2}-\d{2}$", str(v)):
|
||
out.append(f"{k}: {v}")
|
||
elif k in ("version", "release_date", "generation"):
|
||
# always strings — avoid YAML number coercion (4.10 -> 4.1, 2024 -> int)
|
||
out.append(f'{k}: "{v}"')
|
||
else:
|
||
sv = str(v)
|
||
if sv == "" or re.search(r'[:#\[\]"\']', sv) or sv == "—":
|
||
out.append(f'{k}: "{sv}"')
|
||
else:
|
||
out.append(f"{k}: {sv}")
|
||
return "\n".join(out)
|
||
|
||
|
||
def canon_status(s):
|
||
if not s:
|
||
return None
|
||
low = s.lower()
|
||
if "beta" in low:
|
||
return "Beta"
|
||
if "complete" in low or "finished" in low or "full" in low:
|
||
return "Complete"
|
||
if any(w in low for w in ("ongoing", "wip", "development", "in progress",
|
||
"updates", "demo", "unfinished")):
|
||
return "Ongoing"
|
||
return None
|
||
|
||
|
||
PLATFORM_LABEL = {"GB": "Game Boy", "GBC": "Game Boy Color",
|
||
"GBA": "Game Boy Advance", "NDS": "Nintendo DS",
|
||
"3DS": "Nintendo 3DS", "N64": "Nintendo 64",
|
||
"GameCube": "Nintendo GameCube",
|
||
"Switch": "Nintendo Switch",
|
||
"PC": "PC / Joiplay", "—": "Unknown"}
|
||
|
||
|
||
def derive_tags(fm):
|
||
tags = ["hack"]
|
||
p = (fm.get("platform") or "").lower()
|
||
if p and p != "—":
|
||
tags.append(f"platform/{p}")
|
||
b = (fm.get("base") or "")
|
||
if b and b != "—":
|
||
tags.append(f"base/{b.lower().replace(' ', '')}")
|
||
st = (fm.get("status") or "")
|
||
if st and st != "—":
|
||
tags.append(f"status/{st.lower().replace(' ', '')}")
|
||
for t in (fm.get("type") or []):
|
||
tags.append(f"type/{t.lower().replace(' ', '').replace('+', 'plus')}")
|
||
if fm.get("scrape_dir"):
|
||
tags.append("source/discord")
|
||
return tags
|
||
|
||
|
||
# ---------------------------------------------------------------- body builders
|
||
|
||
def features_block(feats, limit=24):
|
||
return "\n".join(f"- {clean(f)}" for f in feats[:limit])
|
||
|
||
|
||
def links_block(links, limit=8):
|
||
out = []
|
||
for u in links[:limit]:
|
||
label = "Download" if re.search(r"mediafire|drive\.google|mega\.nz|"
|
||
r"github|\.zip|\.rar|/file", u) else "Link"
|
||
if "docs.google" in u:
|
||
label = "Guide / docs"
|
||
if "pokecommunity" in u or "pokeharbor" in u or "hackdex" in u:
|
||
label = "Info / thread"
|
||
out.append(f"- {label}: {u}")
|
||
return "\n".join(out)
|
||
|
||
|
||
def type_links(types):
|
||
return " ".join(f"[[Types#{t}|{t}]]" for t in types) if types else ""
|
||
|
||
|
||
# ---------------------------------------------------------------- writers
|
||
|
||
def out_write(path, content, apply):
|
||
if apply:
|
||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||
open(path, "w", encoding="utf-8").write(content)
|
||
else:
|
||
pp = os.path.join(PREVIEW, os.path.basename(path))
|
||
os.makedirs(PREVIEW, exist_ok=True)
|
||
open(pp, "w", encoding="utf-8").write(content)
|
||
|
||
|
||
def cmd_backfill(report, scrape, libidx, apply):
|
||
"""211 Discord notes: fill missing frontmatter only; body preserved."""
|
||
changed = 0
|
||
log = []
|
||
for e in report["results"]["discord_match"]:
|
||
stem = e["note_stem"]
|
||
path = os.path.join(HACKS, stem + ".md")
|
||
if not os.path.exists(path):
|
||
continue
|
||
rec = build_record(scrape[e["scrape_dir"]], libidx)
|
||
fm_text, body, _ = read_note(path)
|
||
fm = parse_fm(fm_text)
|
||
before = dict(fm)
|
||
# fill only missing / dash
|
||
def missing(k):
|
||
v = fm.get(k)
|
||
return v in (None, "", "—") or (k == "type" and not v)
|
||
if missing("base") and rec["base"] and rec["base"] != "—":
|
||
fm["base"] = rec["base"]
|
||
if missing("version") and rec["version"]:
|
||
fm["version"] = rec["version"]
|
||
if missing("status"):
|
||
cs = canon_status(rec["status"]) or canon_status(rec["lead"])
|
||
if cs:
|
||
fm["status"] = cs
|
||
if missing("type") and rec["types"]:
|
||
fm["type"] = rec["types"]
|
||
if missing("developer") and rec["developer"]:
|
||
fm["developer"] = rec["developer"]
|
||
if missing("release_date") and rec["release_date"]:
|
||
fm["release_date"] = rec["release_date"]
|
||
if missing("banner"):
|
||
au = art_url(rec)
|
||
if au:
|
||
fm["banner"] = au
|
||
fm["scrape_dir"] = e["scrape_dir"]
|
||
fm["tags"] = derive_tags(fm)
|
||
if fm != before:
|
||
changed += 1
|
||
new = f"---\n{emit_fm(fm)}\n---\n{body}"
|
||
out_write(path, new, apply)
|
||
log.append(f" {stem}: +{sorted(set(fm)-set(k for k in before if before[k] not in (None,'','—')))}")
|
||
print(f"backfill: {changed} notes updated"
|
||
f"{' (APPLIED)' if apply else ' (dry-run -> .scrape/_preview)'}")
|
||
for l in log[:30]:
|
||
print(l)
|
||
|
||
|
||
def cmd_merge(report, scrape, libidx, apply):
|
||
"""43 curated overlaps: keep note, add banner + Story/Features/community links."""
|
||
changed = 0
|
||
for e in report["results"]["curated_overlap"]:
|
||
stem = e["note_stem"]
|
||
path = os.path.join(HACKS, stem + ".md")
|
||
if not os.path.exists(path):
|
||
continue
|
||
rec = build_record(scrape[e["scrape_dir"]], libidx)
|
||
fm_text, body, _ = read_note(path)
|
||
fm = parse_fm(fm_text)
|
||
if rec["developer"] and not fm.get("developer"):
|
||
fm["developer"] = rec["developer"]
|
||
if rec["release_date"] and not fm.get("release_date"):
|
||
fm["release_date"] = rec["release_date"]
|
||
au = art_url(rec)
|
||
if au and not fm.get("banner"):
|
||
fm["banner"] = au
|
||
fm["scrape_dir"] = e["scrape_dir"]
|
||
|
||
# body: insert banner image after callout blockquote (if no image yet)
|
||
if au and "![" not in body:
|
||
m = re.search(r"((?:^>.*\n)+)", body, re.M)
|
||
if m:
|
||
ins = m.end()
|
||
body = body[:ins] + f"\n\n" + body[ins:]
|
||
# build append sections before the [[Index footer
|
||
add = []
|
||
if rec["story"] and "## Story" not in body:
|
||
add.append(f"## Story\n\n{rec['story']}")
|
||
if rec["features"] and "## Features" not in body:
|
||
add.append(f"## Features\n\n{features_block(rec['features'])}")
|
||
new_links = [u for u in rec["links"] if u not in body]
|
||
if new_links and "## Community links" not in body:
|
||
add.append(f"## Community links\n\n{links_block(new_links)}")
|
||
if add:
|
||
block = "\n" + "\n\n".join(add) + "\n\n"
|
||
# insert after Summary: before the first of Type/Links/In the library,
|
||
# else before the [[Index footer.
|
||
anchor = None
|
||
for h in (r"\n## Type", r"\n## Links", r"\n## In the library",
|
||
r"\n## Files", r"\n\[\[Index"):
|
||
m2 = re.search(h, body)
|
||
if m2:
|
||
anchor = m2.start()
|
||
break
|
||
if anchor is not None:
|
||
body = body[:anchor] + "\n" + block.rstrip() + "\n" + body[anchor:]
|
||
else:
|
||
body = body.rstrip() + "\n\n" + block
|
||
new = f"---\n{emit_fm(fm)}\n---\n{body}"
|
||
out_write(path, new, apply)
|
||
changed += 1
|
||
print(f"merge: {changed} curated notes enriched"
|
||
f"{' (APPLIED)' if apply else ' (dry-run -> .scrape/_preview)'}")
|
||
|
||
|
||
def safe_stem(name):
|
||
s = re.sub(r"^\s*pok[eé]mon\s+", "", name, flags=re.I).strip()
|
||
s = s.replace("/", " ").replace(":", " -").replace("|", " ")
|
||
s = re.sub(r'[<>"\\?*]', "", s)
|
||
s = re.sub(r"\s+", " ", s).strip()
|
||
return s or name
|
||
|
||
|
||
def build_body(rec, fm, *, owned, library_note=None):
|
||
"""Clean, uniform wiki body shared by create + rebuild. owned=True writes an
|
||
'In the library' (RomM/served) block; owned=False writes a catalog-entry note."""
|
||
title = fm["title"]
|
||
plat, base = fm.get("platform", "—"), fm.get("base", "—")
|
||
status = fm.get("status", "—")
|
||
ver = fm.get("version", "—")
|
||
au = fm.get("banner")
|
||
kindline = {"console": "ROM hack", "fangame": "fan-game",
|
||
"patch": "patch"}.get(rec["kind"], "ROM hack / fan-game")
|
||
callout = (f"> [!info] {', '.join(fm.get('type') or []) or kindline} — **{status}**\n"
|
||
f"> {PLATFORM_LABEL.get(plat, plat)} · base **{base}** · "
|
||
f"version **{ver}**"
|
||
+ (f" · by **{fm['developer']}**" if fm.get("developer") else ""))
|
||
parts = [f"# {title}", "", callout, ""]
|
||
if au:
|
||
parts += [f"", ""]
|
||
parts += ["## Summary", "",
|
||
rec["lead"] or rec["description_body"]
|
||
or "Catalogued from the Discord ROM-hack archive. See links below.", ""]
|
||
if rec["story"]:
|
||
parts += ["## Story", "", rec["story"], ""]
|
||
if rec["features"]:
|
||
parts += ["## Features", "", features_block(rec["features"]), ""]
|
||
if fm.get("type"):
|
||
parts += ["## Type", type_links(fm["type"]), ""]
|
||
if rec["links"]:
|
||
parts += ["## Links", "", links_block(rec["links"]), ""]
|
||
if owned:
|
||
slug = library_note or rec["scrape_dir"]
|
||
parts += ["## In the library", "",
|
||
f"- Served files: [browse]({ART_BASE.replace('/_meta','')}/{slug}/)",
|
||
f"- Library path: `/storage1/labdata/romhacks/library/{slug}/`", ""]
|
||
else:
|
||
parts += ["## In the library", "",
|
||
"> [!note] Catalog entry imported from the Discord archive on "
|
||
f"{TODAY}. Not yet placed in the RomM library.", ""]
|
||
parts += ["[[Index|← back to directory]]", ""]
|
||
return "\n".join(parts)
|
||
|
||
|
||
def cmd_create(report, scrape, libidx, apply):
|
||
"""83 unmatched: brand-new wiki-schema notes."""
|
||
created = skipped = 0
|
||
existing = {os.path.splitext(f)[0].lower() for f in os.listdir(HACKS)}
|
||
for e in report["results"]["unmatched"]:
|
||
rec = build_record(scrape[e["scrape_dir"]], libidx)
|
||
stem = safe_stem(rec["name"])
|
||
if stem.lower() in existing:
|
||
skipped += 1
|
||
continue
|
||
plat = rec["platform"] or "—"
|
||
base = rec["base"] or "—"
|
||
status = canon_status(rec["status"]) or canon_status(rec["lead"]) or "—"
|
||
au = art_url(rec)
|
||
title = rec["name"] if rec["name"].lower().startswith(("pokémon", "pokemon")) \
|
||
else f"Pokémon {stem}"
|
||
title = title.replace("Pokemon", "Pokémon")
|
||
fm = {"title": title, "platform": plat, "base": base,
|
||
"version": rec["version"] or "—", "status": status,
|
||
"type": rec["types"], "generation": "—"}
|
||
if rec["developer"]:
|
||
fm["developer"] = rec["developer"]
|
||
if rec["release_date"]:
|
||
fm["release_date"] = rec["release_date"]
|
||
if au:
|
||
fm["banner"] = au
|
||
fm["library_path"] = "—"
|
||
if rec["links"]:
|
||
fm["source"] = rec["links"][0]
|
||
fm["scrape_dir"] = e["scrape_dir"]
|
||
fm["added"] = TODAY
|
||
fm["play_status"] = "Unplayed"
|
||
fm["tags"] = derive_tags(fm)
|
||
content = f"---\n{emit_fm(fm)}\n---\n" + build_body(rec, fm, owned=False)
|
||
out_write(os.path.join(HACKS, stem + ".md"), content, apply)
|
||
created += 1
|
||
print(f"create: {created} new notes, {skipped} skipped (name collision)"
|
||
f"{' (APPLIED)' if apply else ' (dry-run -> .scrape/_preview)'}")
|
||
|
||
|
||
def cmd_rebuild(report, scrape, libidx, apply):
|
||
"""211 Discord notes: regenerate the BODY into the clean uniform schema,
|
||
preserving (already-backfilled) frontmatter. Removes the messy auto-import
|
||
callout / blockquoted Description / duplicated Download artifacts."""
|
||
done = 0
|
||
owned_slugs = set(libidx)
|
||
for e in report["results"]["discord_match"]:
|
||
stem = e["note_stem"]
|
||
path = os.path.join(HACKS, stem + ".md")
|
||
if not os.path.exists(path):
|
||
continue
|
||
rec = build_record(scrape[e["scrape_dir"]], libidx)
|
||
fm_text, _, _ = read_note(path)
|
||
fm = parse_fm(fm_text)
|
||
owned = e["scrape_dir"] in owned_slugs
|
||
body = build_body(rec, fm, owned=owned, library_note=e["scrape_dir"])
|
||
content = f"---\n{emit_fm(fm)}\n---\n{body}"
|
||
out_write(path, content, apply)
|
||
done += 1
|
||
print(f"rebuild: {done} Discord note bodies normalized"
|
||
f"{' (APPLIED)' if apply else ' (dry-run -> .scrape/_preview)'}")
|
||
|
||
|
||
def main():
|
||
cmds = {"backfill": cmd_backfill, "merge": cmd_merge,
|
||
"create": cmd_create, "rebuild": cmd_rebuild}
|
||
if len(sys.argv) < 2 or sys.argv[1] not in cmds:
|
||
sys.exit(f"usage: enrich_vault.py {{{'|'.join(cmds)}}} [--apply]")
|
||
apply = "--apply" in sys.argv
|
||
report = json.load(open(os.path.join(SCRAPE, "_match_report.json"), encoding="utf-8"))
|
||
scrape = load_scrape()
|
||
libidx = json.load(open(os.path.join(SCRAPE, "_library_index.json"), encoding="utf-8"))
|
||
cmds[sys.argv[1]](report, scrape, libidx, apply)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|