ginnoir
44183fb0fb
Adds standardize-vault-pages pipeline, exports intentional empty generations, and refreshes the 396-hack catalog with full banner coverage, status classification, and Fakemon generation tags.
479 lines
16 KiB
Python
479 lines
16 KiB
Python
#!/usr/bin/env python3
|
||
"""Standardize ROM-hack vault pages for site export.
|
||
|
||
This pass is intentionally conservative: it uses the current vault note text,
|
||
existing scrape metadata, and deterministic inference rules. Web-researched facts
|
||
belong in .scrape/web_facts*.json and should be applied with enrich_web.py.
|
||
|
||
Run:
|
||
python scripts/standardize-vault-pages.py
|
||
python scripts/standardize-vault-pages.py --apply
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from collections import Counter
|
||
from urllib.parse import urlparse
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
import enrich_vault as E
|
||
|
||
GEN_ROMAN = {
|
||
1: "Gen I",
|
||
2: "Gen II",
|
||
3: "Gen III",
|
||
4: "Gen IV",
|
||
5: "Gen V",
|
||
6: "Gen VI",
|
||
7: "Gen VII",
|
||
8: "Gen VIII",
|
||
9: "Gen IX",
|
||
}
|
||
ROMAN_TO_INT = {
|
||
"i": 1,
|
||
"ii": 2,
|
||
"iii": 3,
|
||
"iv": 4,
|
||
"v": 5,
|
||
"vi": 6,
|
||
"vii": 7,
|
||
"viii": 8,
|
||
"ix": 9,
|
||
}
|
||
REGIONS = ["Alolan", "Galarian", "Hisuian", "Paldean"]
|
||
FAKEMON_GEN_TAG = "Fakemon"
|
||
PURE_CUSTOM_DEX_STEMS = {
|
||
"Cope",
|
||
"Fakemon Fire Red",
|
||
"Pisces",
|
||
"Solar Light Lunar Dark",
|
||
"Touhoumon Another World",
|
||
"Void",
|
||
}
|
||
GENERATION_SORT_ORDER = list(GEN_ROMAN.values()) + REGIONS + [FAKEMON_GEN_TAG]
|
||
COUNT_TO_GEN = [
|
||
(1025, 9),
|
||
(1008, 9),
|
||
(905, 8),
|
||
(898, 8),
|
||
(809, 7),
|
||
(807, 7),
|
||
(721, 6),
|
||
(649, 5),
|
||
(493, 4),
|
||
(386, 3),
|
||
(251, 2),
|
||
(151, 1),
|
||
]
|
||
BASE_MAX_GEN = {
|
||
"Red": 1,
|
||
"Red and Blue": 1,
|
||
"Blue": 1,
|
||
"Yellow": 1,
|
||
"Stadium": 1,
|
||
"Gold": 2,
|
||
"Silver": 2,
|
||
"Crystal": 2,
|
||
"Ruby": 3,
|
||
"Sapphire": 3,
|
||
"Emerald": 3,
|
||
"FireRed": 3,
|
||
"LeafGreen": 3,
|
||
"Diamond": 4,
|
||
"Pearl": 4,
|
||
"Platinum": 4,
|
||
"HeartGold": 4,
|
||
"SoulSilver": 4,
|
||
"Black": 5,
|
||
"White": 5,
|
||
"Black / White": 5,
|
||
"Pokemon Black & White": 5,
|
||
"Black 2": 5,
|
||
"White 2": 5,
|
||
"X": 6,
|
||
"Y": 6,
|
||
"Omega Ruby": 6,
|
||
"Alpha Sapphire": 6,
|
||
"Sun": 7,
|
||
"Moon": 7,
|
||
"Ultra Sun": 7,
|
||
"Ultra Moon": 7,
|
||
"Ultra Sun / Ultra Moon": 7,
|
||
"Sword": 8,
|
||
"Shield": 8,
|
||
"Sword / Shield": 8,
|
||
"Scarlet": 9,
|
||
"Violet": 9,
|
||
"Scarlet / Violet": 9,
|
||
"XD: Gale of Darkness": 3,
|
||
}
|
||
BAD_BASE_VALUES = {
|
||
"how far you are into the journey",
|
||
"precedents set",
|
||
"the north atlantic island nation of Iceland",
|
||
}
|
||
|
||
|
||
def gen_label_to_int(raw: str) -> int | None:
|
||
value = raw.strip().lower().replace("generation", "").replace("gen", "").strip()
|
||
value = value.strip(" .:-")
|
||
if value.isdigit():
|
||
n = int(value)
|
||
return n if 1 <= n <= 9 else None
|
||
return ROMAN_TO_INT.get(value)
|
||
|
||
|
||
def gens_through(n: int) -> list[str]:
|
||
return [GEN_ROMAN[i] for i in range(1, n + 1)]
|
||
|
||
|
||
def unique_ordered(values: list[str]) -> list[str]:
|
||
seen = set()
|
||
out = []
|
||
for value in values:
|
||
if value and value not in seen:
|
||
seen.add(value)
|
||
out.append(value)
|
||
return out
|
||
|
||
|
||
def sort_generations(generations: list[str]) -> list[str]:
|
||
rank = {label: index for index, label in enumerate(GENERATION_SORT_ORDER)}
|
||
return unique_ordered(sorted(generations, key=lambda label: rank.get(label, len(GENERATION_SORT_ORDER))))
|
||
|
||
|
||
def apply_fakemon_generation_tag(fm: dict, stem: str) -> list[str]:
|
||
generations = [g for g in (fm.get("generations") or []) if g != FAKEMON_GEN_TAG]
|
||
if fm.get("fakemon") != "Yes":
|
||
return sort_generations(generations)
|
||
if stem in PURE_CUSTOM_DEX_STEMS:
|
||
return [FAKEMON_GEN_TAG]
|
||
return sort_generations(generations + [FAKEMON_GEN_TAG])
|
||
|
||
|
||
def note_sections(body: str) -> dict[str, str]:
|
||
sections: dict[str, str] = {}
|
||
for match in re.finditer(r"(?ms)^## ([^\n]+)\s*\n+(.+?)(?=^## |\n\[\[Index|\Z)", body):
|
||
sections[match.group(1).strip()] = match.group(2).strip()
|
||
return sections
|
||
|
||
|
||
def set_section(body: str, heading: str, content: str) -> str:
|
||
block = f"## {heading}\n\n{content.rstrip()}\n\n"
|
||
pattern = re.compile(rf"(?ms)^## {re.escape(heading)}\s*\n.*?(?=^## |\n\[\[Index|\Z)")
|
||
if pattern.search(body):
|
||
return pattern.sub(block, body, count=1)
|
||
summary = re.search(r"(?ms)^## Summary\s*\n.*?(?=^## |\n\[\[Index|\Z)", body)
|
||
if summary:
|
||
return body[: summary.end()] + block + body[summary.end() :]
|
||
footer = re.search(r"(?m)^\[\[Index", body)
|
||
if footer:
|
||
return body[: footer.start()] + block + body[footer.start() :]
|
||
return body.rstrip() + "\n\n" + block
|
||
|
||
|
||
def clean_summary(text: str) -> str:
|
||
text = re.sub(r"\s+", " ", text).strip()
|
||
text = re.sub(r"\s*-\s*(?:Rom link|Download|wiki|Link)\s*:?\s*https?://\S+", "", text, flags=re.I)
|
||
return text.strip()
|
||
|
||
|
||
def extract_urls(text: str) -> list[str]:
|
||
urls = []
|
||
for match in re.finditer(r"https?://[^\s)\]>'\"]+", text):
|
||
urls.append(match.group(0).rstrip(".,)"))
|
||
return unique_ordered(urls)
|
||
|
||
|
||
def link_label(url: str) -> str:
|
||
host = urlparse(url).netloc.lower()
|
||
if "pokecommunity" in host:
|
||
return "PokeCommunity"
|
||
if "hackdex" in host:
|
||
return "HackDex"
|
||
if "docs.google" in host or "pastebin" in host:
|
||
return "Documentation"
|
||
if "github" in host:
|
||
return "GitHub"
|
||
if any(x in host for x in ("mediafire", "mega.nz", "drive.google")):
|
||
return "Download"
|
||
if "fandom" in host or "wiki" in host:
|
||
return "Wiki"
|
||
return "Link"
|
||
|
||
|
||
def is_placeholder_link(url: str) -> bool:
|
||
parsed = urlparse(url)
|
||
host = parsed.netloc.lower()
|
||
path = parsed.path.strip("/")
|
||
if "duckduckgo.com" in host:
|
||
return True
|
||
if host == "drive.google.com" and not path:
|
||
return True
|
||
return False
|
||
|
||
|
||
def normalize_links_section(existing: str, extra_urls: list[str]) -> str:
|
||
pairs: list[tuple[str, str]] = []
|
||
for line in existing.splitlines():
|
||
url_match = re.search(r"(https?://\S+)", line)
|
||
if not url_match:
|
||
continue
|
||
url = url_match.group(1).rstrip(".,)")
|
||
label_match = re.match(r"^\s*-\s*([^:]+):", line)
|
||
pairs.append(((label_match.group(1).strip() if label_match else link_label(url)), url))
|
||
for url in extra_urls:
|
||
pairs.append((link_label(url), url))
|
||
|
||
has_real_source = any(not is_placeholder_link(url) for _, url in pairs)
|
||
seen = set()
|
||
lines = []
|
||
for label, url in pairs:
|
||
if has_real_source and is_placeholder_link(url):
|
||
continue
|
||
if url in seen:
|
||
continue
|
||
seen.add(url)
|
||
lines.append(f"- {label}: {url}")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def normalize_features(text: str) -> tuple[str, list[str]]:
|
||
lines = []
|
||
extracted_urls = []
|
||
for raw in text.splitlines():
|
||
line = raw.strip()
|
||
if not line:
|
||
continue
|
||
line = re.sub(r"^[-*•➡️\s]+", "", line).strip()
|
||
markdown_link = re.fullmatch(r"\[([^\]]+)\]\((https?://[^)]+)\)", line)
|
||
if markdown_link:
|
||
extracted_urls.append(markdown_link.group(2))
|
||
if re.search(r"document|documentation|more information|wiki|download|link", markdown_link.group(1), re.I):
|
||
continue
|
||
urls = extract_urls(line)
|
||
if urls:
|
||
extracted_urls.extend(urls)
|
||
if re.fullmatch(r"(?:Document|Documentation|More information|Wiki|Download|Link)s?\**:?", line.split("http", 1)[0].strip(), re.I):
|
||
continue
|
||
line = re.sub(r"\s*https?://\S+", "", line).strip(" -")
|
||
line = line.strip("* ")
|
||
if not line or re.fullmatch(r"more informations?", line, re.I):
|
||
continue
|
||
if not line.endswith((".", "!", "?")) and len(line) > 80:
|
||
line += "."
|
||
lines.append(f"- {line}")
|
||
return "\n".join(unique_ordered(lines)), extracted_urls
|
||
|
||
|
||
def ignore_generation_context(text: str, start: int, end: int) -> bool:
|
||
window = text[max(0, start - 35) : min(len(text), end + 45)]
|
||
if re.search(r"\b(?:dex|national dex|pok[eé]mon|mons|roster|catch|available)\b", window, re.I):
|
||
return False
|
||
return bool(re.search(r"\b(?:battle|engine|mechanics?|moves?|abilities|standard|style)\b", window, re.I))
|
||
|
||
|
||
def find_generation_max(text: str) -> int | None:
|
||
explicit: list[int] = []
|
||
if re.search(r"\b(?:all generations|all gens|from all gens|from all generations)\b", text, re.I):
|
||
explicit.append(9)
|
||
for match in re.finditer(
|
||
r"\bgen(?:eration)?s?\s*(\d|i{1,3}|iv|v|vi{0,3}|ix)\s*(?:-|–|—|to|through|thru|up to|and)\s*(?:gen(?:eration)?s?\s*)?(\d|i{1,3}|iv|v|vi{0,3}|ix)\b",
|
||
text,
|
||
re.I,
|
||
):
|
||
if ignore_generation_context(text, match.start(), match.end()):
|
||
continue
|
||
a = gen_label_to_int(match.group(1))
|
||
b = gen_label_to_int(match.group(2))
|
||
if a and b:
|
||
explicit.append(max(a, b))
|
||
for match in re.finditer(r"\b(?:gen(?:eration)?s?|through gen|up to gen)\s*(\d|i{1,3}|iv|v|vi{0,3}|ix)\b", text, re.I):
|
||
if ignore_generation_context(text, match.start(), match.end()):
|
||
continue
|
||
n = gen_label_to_int(match.group(1))
|
||
if n:
|
||
explicit.append(n)
|
||
for count, gen in COUNT_TO_GEN:
|
||
if re.search(rf"\b{count}\+?\s+(?:pok[eé]mon|mons|national dex|dex)\b", text, re.I):
|
||
explicit.append(gen)
|
||
if re.search(rf"\b(?:pok[eé]mon|mons|national dex|dex)\s*(?:up to|through|of)?\s*{count}\+?\b", text, re.I):
|
||
explicit.append(gen)
|
||
return max(explicit) if explicit else None
|
||
|
||
|
||
def infer_generations(fm: dict, body: str) -> list[str]:
|
||
if "generations" in fm:
|
||
return list(fm["generations"])
|
||
title = fm.get("title", "")
|
||
text = f"{title}\n{body}"
|
||
max_gen = find_generation_max(text)
|
||
replaces_official = re.search(
|
||
r"(?:replaces|removes|rids) .{0,80}(?:official\s+)?pok[eé]mon"
|
||
r"|replaces .{0,80}with .{0,80}(?:touhou characters|boneka)",
|
||
text,
|
||
re.I,
|
||
)
|
||
custom_species_roster = re.search(
|
||
r"(?:entirely new|all[- ]new|full new|brand-new|custom)\s+(?:dex|pok[eé]dex|roster|mons|pok[eé]mon)"
|
||
r"|(?:dex|pok[eé]dex|roster)\s+full of\s+(?:new|custom)?\s*mons"
|
||
r"|(?:over|more than)\s+\d+\s+original\s+fakemon"
|
||
r"|\ball\s+\d+\s+boneka\b"
|
||
r"|\b(?:boneka|touhou characters?)\s+roster\b",
|
||
text,
|
||
re.I,
|
||
)
|
||
pure_custom_roster = fm.get("fakemon") == "Yes" and (replaces_official or custom_species_roster)
|
||
regions = [region for region in REGIONS if re.search(rf"\b{region}\b", text, re.I)]
|
||
if pure_custom_roster:
|
||
return regions
|
||
if max_gen is None and not pure_custom_roster:
|
||
base = fm.get("base")
|
||
max_gen = BASE_MAX_GEN.get(base)
|
||
generations = gens_through(max_gen) if max_gen else []
|
||
return unique_ordered(generations + regions)
|
||
|
||
|
||
def infer_fakemon(fm: dict, body: str) -> str:
|
||
if fm.get("fakemon") in {"Yes", "No"}:
|
||
return fm["fakemon"]
|
||
text = f"{fm.get('title', '')}\n{body}"
|
||
text = re.sub(r"Fakemon\s+\*\*(?:Yes|No)\*\*", "", text, flags=re.I)
|
||
text = re.sub(r"\b(?:without|no|not)\s+(?:a\s+)?fakemon(?:\s+roster)?\b", "", text, flags=re.I)
|
||
strong = [
|
||
r"\bfakemon\b",
|
||
r"fan[- ]made pok[eé]mon",
|
||
r"original (?:fakemon|monsters)",
|
||
r"custom (?:pok[eé]mon|monsters|mons|pokedex|pok[eé]dex)(?!\s+(?:sprites?|cries?))",
|
||
r"all[- ]new (?:pok[eé]mon|monsters|pokedex|pok[eé]dex)",
|
||
r"\d{2,4}\s+(?:new|original|custom)?\s*(?:fakemon|monsters) designed",
|
||
r"replaces .{0,80}official pok[eé]mon",
|
||
]
|
||
if any(re.search(pattern, text, re.I) for pattern in strong):
|
||
return "Yes"
|
||
return "No"
|
||
|
||
|
||
def update_callout(body: str, fm: dict) -> str:
|
||
generations = ", ".join(fm.get("generations") or [])
|
||
extra = []
|
||
if generations:
|
||
extra.append(f"roster **{generations}**")
|
||
if fm.get("fakemon"):
|
||
extra.append(f"Fakemon **{fm['fakemon']}**")
|
||
if not extra:
|
||
return body
|
||
lines = body.splitlines()
|
||
for idx, line in enumerate(lines):
|
||
if line.startswith("> ") and " · base **" in line:
|
||
line = re.sub(r"\s*·\s*roster \*\*[^*]+\*\*", "", line)
|
||
line = re.sub(r"\s*·\s*Fakemon \*\*(?:Yes|No)\*\*", "", line)
|
||
lines[idx] = line + " · " + " · ".join(extra)
|
||
return "\n".join(lines)
|
||
return body
|
||
|
||
|
||
def standardize_note(path: str) -> tuple[bool, dict]:
|
||
fm_text, body, _ = E.read_note(path)
|
||
fm = E.parse_fm(fm_text)
|
||
before_fm = dict(fm)
|
||
before_body = body
|
||
sections = note_sections(body)
|
||
|
||
if fm.get("base") in BAD_BASE_VALUES:
|
||
fm["base"] = "—"
|
||
fm["fakemon"] = infer_fakemon(fm, body)
|
||
fm["generations"] = apply_fakemon_generation_tag(
|
||
{**fm, "generations": infer_generations(fm, body)},
|
||
os.path.splitext(os.path.basename(path))[0],
|
||
)
|
||
fm["tags"] = E.derive_tags(fm)
|
||
|
||
if sections.get("Summary"):
|
||
cleaned = clean_summary(sections["Summary"])
|
||
if cleaned and cleaned != sections["Summary"]:
|
||
body = set_section(body, "Summary", cleaned)
|
||
|
||
extracted_urls: list[str] = []
|
||
if sections.get("Features"):
|
||
features, extracted_urls = normalize_features(sections["Features"])
|
||
if features and features != sections["Features"]:
|
||
body = set_section(body, "Features", features)
|
||
|
||
type_links = E.type_links(fm.get("type") or [])
|
||
if type_links:
|
||
body = set_section(body, "Type", type_links)
|
||
|
||
current_sections = note_sections(body)
|
||
if current_sections.get("Links") or extracted_urls:
|
||
links = normalize_links_section(current_sections.get("Links", ""), extracted_urls)
|
||
if links:
|
||
body = set_section(body, "Links", links)
|
||
|
||
body = update_callout(body, fm)
|
||
content = f"---\n{E.emit_fm(fm)}\n---\n{body.rstrip()}\n"
|
||
changed = before_fm != fm or before_body.rstrip() != body.rstrip()
|
||
audit = {
|
||
"stem": os.path.splitext(os.path.basename(path))[0],
|
||
"fakemon": fm.get("fakemon"),
|
||
"generations": fm.get("generations") or [],
|
||
"changed": changed,
|
||
"had_features": bool(sections.get("Features")),
|
||
"feature_urls_moved": len(extracted_urls),
|
||
}
|
||
return changed, {"content": content, "audit": audit}
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("--apply", action="store_true")
|
||
parser.add_argument("--limit", type=int)
|
||
parser.add_argument("--report", default=os.path.join(E.SCRAPE, "standardize-report.json"))
|
||
args = parser.parse_args()
|
||
|
||
paths = [os.path.join(E.HACKS, fn) for fn in sorted(os.listdir(E.HACKS)) if fn.endswith(".md")]
|
||
if args.limit:
|
||
paths = paths[: args.limit]
|
||
|
||
changed = 0
|
||
audits = []
|
||
for path in paths:
|
||
did_change, result = standardize_note(path)
|
||
audits.append(result["audit"])
|
||
if did_change:
|
||
changed += 1
|
||
if args.apply:
|
||
with open(path, "w", encoding="utf-8", newline="\n") as f:
|
||
f.write(result["content"])
|
||
else:
|
||
os.makedirs(E.PREVIEW, exist_ok=True)
|
||
with open(os.path.join(E.PREVIEW, os.path.basename(path)), "w", encoding="utf-8", newline="\n") as f:
|
||
f.write(result["content"])
|
||
|
||
summary = {
|
||
"notes": len(paths),
|
||
"changed": changed,
|
||
"fakemon": dict(Counter(a["fakemon"] for a in audits)),
|
||
"with_generations": sum(1 for a in audits if a["generations"]),
|
||
"without_generations": sum(1 for a in audits if not a["generations"]),
|
||
"feature_urls_moved": sum(a["feature_urls_moved"] for a in audits),
|
||
"audits": audits,
|
||
}
|
||
os.makedirs(os.path.dirname(args.report), exist_ok=True)
|
||
with open(args.report, "w", encoding="utf-8") as f:
|
||
json.dump(summary, f, indent=2, ensure_ascii=False)
|
||
print(f"standardized {len(paths)} notes; changed={changed}; apply={args.apply}")
|
||
print(f"fakemon={summary['fakemon']}")
|
||
print(f"with_generations={summary['with_generations']} without_generations={summary['without_generations']}")
|
||
print(f"feature_urls_moved={summary['feature_urls_moved']}")
|
||
print(f"report={args.report}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|