Complete catalog standardization: generations, banners, and Fakemon roster tag.
Adds standardize-vault-pages pipeline, exports intentional empty generations, and refreshes the 396-hack catalog with full banner coverage, status classification, and Fakemon generation tags.
This commit is contained in:
ginnoir
@@ -100,10 +100,13 @@ def wikilink(stem: str) -> str:
|
||||
# ---------------------------------------------------------------- MOC config
|
||||
|
||||
# Logical hardware order for human-facing lists.
|
||||
PLATFORM_ORDER = ["GB", "GBC", "GBA", "NDS", "3DS", "PC", "Patch", DASH]
|
||||
PLATFORM_ORDER = ["GB", "GBC", "GBA", "NDS", "3DS", "N64", "GameCube", "Switch", "PC", "Patch", DASH]
|
||||
PLATFORM_LABEL = {
|
||||
"GB": "Game Boy", "GBC": "Game Boy Color", "GBA": "Game Boy Advance",
|
||||
"NDS": "Nintendo DS", "3DS": "Nintendo 3DS", "PC": "PC / Joiplay fan-games",
|
||||
"NDS": "Nintendo DS", "3DS": "Nintendo 3DS", "N64": "Nintendo 64",
|
||||
"GameCube": "Nintendo GameCube",
|
||||
"Switch": "Nintendo Switch",
|
||||
"PC": "PC / Joiplay fan-games",
|
||||
"Patch": "Patch-only", DASH: "Unknown platform",
|
||||
}
|
||||
|
||||
@@ -117,6 +120,9 @@ PLATFORM_MOCS = {
|
||||
"GB": ("Game Boy (GB)", plat_pred("GB")),
|
||||
"NDS": ("Nintendo DS (NDS)", plat_pred("NDS")),
|
||||
"3DS": ("Nintendo 3DS (3DS)", plat_pred("3DS")),
|
||||
"N64": ("Nintendo 64 (N64)", plat_pred("N64")),
|
||||
"GameCube": ("Nintendo GameCube", plat_pred("GameCube")),
|
||||
"Switch": ("Nintendo Switch", plat_pred("Switch")),
|
||||
"PC": ("PC / Joiplay Fan-games", plat_pred("PC")),
|
||||
"Patch": ("Patch-only", plat_pred("Patch")),
|
||||
"Unknown": ("Unknown platform", plat_pred(DASH)),
|
||||
@@ -128,6 +134,18 @@ NEW_PLATFORM_MOCS = {
|
||||
"Nintendo 3DS Pokémon fan-games and hacks. A small but growing set — "
|
||||
"mostly Citra/Luma-targeted projects harvested from the Discord catalog.",
|
||||
'WHERE platform = "3DS"'),
|
||||
"N64": ("N64",
|
||||
"Nintendo 64 Pokémon mods and overhaul patches. These are tracked "
|
||||
"separately from handheld ROM hacks because they target Nintendo 64-era base games.",
|
||||
'WHERE platform = "N64"'),
|
||||
"GameCube": ("GameCube",
|
||||
"Nintendo GameCube Pokémon mods and overhaul patches. These are tracked "
|
||||
"separately from handheld ROM hacks because they target GameCube-era base games.",
|
||||
'WHERE platform = "GameCube"'),
|
||||
"Switch": ("Switch",
|
||||
"Nintendo Switch Pokémon mods and overhaul patches. These are tracked "
|
||||
"separately from handheld ROM hacks because they target Switch-era base games.",
|
||||
'WHERE platform = "Switch"'),
|
||||
"PC": ("PC",
|
||||
"RPG-Maker / Joiplay fan-games (run on PC or via Joiplay on Android). "
|
||||
"These are standalone games, **not** console ROM hacks — they live under "
|
||||
|
||||
@@ -20,8 +20,9 @@ OUT_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__)
|
||||
OUT = os.path.join(OUT_DIR, "catalog.json")
|
||||
|
||||
SCALARS = ["title", "platform", "base", "version", "status", "generation",
|
||||
"developer", "release_date", "banner", "homepage", "source",
|
||||
"fakemon", "developer", "release_date", "banner", "homepage", "source",
|
||||
"library_path", "play_status", "scrape_dir", "engine"]
|
||||
LIST_SCALARS = ["generations"]
|
||||
|
||||
|
||||
def section(body, heading):
|
||||
@@ -57,6 +58,9 @@ def main():
|
||||
for k in SCALARS:
|
||||
if fm.get(k) not in (None, ""):
|
||||
rec[k] = fm[k]
|
||||
for k in LIST_SCALARS:
|
||||
if k in fm:
|
||||
rec[k] = fm[k]
|
||||
rec["type"] = fm.get("type") or []
|
||||
rec["tags"] = fm.get("tags") or []
|
||||
if fm.get("rating") not in (None, ""):
|
||||
|
||||
+10
-2
@@ -53,6 +53,9 @@ PLATFORM_HINTS = [
|
||||
(r"\bGBC\b|game boy color", ("GBC", "console")),
|
||||
(r"\bNDS\b|nintendo ds\b|\bDS ROM", ("NDS", "console")),
|
||||
(r"\b3DS\b", ("3DS", "console")),
|
||||
(r"\bN64\b|Nintendo 64|Pokemon Stadium|Pokémon Stadium", ("N64", "console")),
|
||||
(r"\bGameCube\b|Nintendo GameCube|Pokemon Colosseum|Pokémon Colosseum", ("GameCube", "console")),
|
||||
(r"\bSwitch\b|Nintendo Switch", ("Switch", "console")),
|
||||
(r"\bGB\b|game boy(?! advance| color)", ("GB", "console")),
|
||||
(r"RPGXP|RPG ?Maker|Essentials|FanGame|fan game|GameMaker", ("PC", "fangame")),
|
||||
]
|
||||
@@ -256,6 +259,8 @@ GAME_PLATFORM = {
|
||||
"soul silver": ("NDS", "SoulSilver"), "black": ("NDS", "Black"),
|
||||
"white": ("NDS", "White"), "black 2": ("NDS", "Black 2"),
|
||||
"white 2": ("NDS", "White 2"),
|
||||
"stadium": ("N64", "Stadium"), "pokemon stadium": ("N64", "Stadium"),
|
||||
"pokémon stadium": ("N64", "Stadium"),
|
||||
}
|
||||
|
||||
|
||||
@@ -389,7 +394,7 @@ def load_scrape():
|
||||
# ---------------------------------------------------------------- frontmatter (de)serialize
|
||||
|
||||
FM_ORDER = ["title", "platform", "base", "version", "status", "type",
|
||||
"generation", "developer", "release_date", "banner",
|
||||
"generation", "generations", "fakemon", "developer", "release_date", "banner",
|
||||
"library_path", "source", "homepage", "scrape_dir",
|
||||
"added", "play_status", "rating", "tags"]
|
||||
LIST_KEYS = {"type", "tags"}
|
||||
@@ -462,7 +467,10 @@ def canon_status(s):
|
||||
|
||||
PLATFORM_LABEL = {"GB": "Game Boy", "GBC": "Game Boy Color",
|
||||
"GBA": "Game Boy Advance", "NDS": "Nintendo DS",
|
||||
"3DS": "Nintendo 3DS", "PC": "PC / Joiplay", "—": "Unknown"}
|
||||
"3DS": "Nintendo 3DS", "N64": "Nintendo 64",
|
||||
"GameCube": "Nintendo GameCube",
|
||||
"Switch": "Nintendo Switch",
|
||||
"PC": "PC / Joiplay", "—": "Unknown"}
|
||||
|
||||
|
||||
def derive_tags(fm):
|
||||
|
||||
+50
-2
@@ -20,7 +20,50 @@ import enrich_vault as E
|
||||
|
||||
FACTS_GLOB = os.path.join(E.SCRAPE, "web_facts*.json")
|
||||
SCALAR_FIELDS = ["tagline", "developer", "version", "status", "release_date",
|
||||
"base", "platform", "generation", "homepage"]
|
||||
"base", "platform", "generation", "homepage", "source", "fakemon",
|
||||
"banner"]
|
||||
LIST_FIELDS = ["generations"]
|
||||
|
||||
|
||||
def normalize_link_fact(link):
|
||||
if isinstance(link, str):
|
||||
return "Link", link
|
||||
if isinstance(link, dict) and link.get("url"):
|
||||
return link.get("label") or "Link", link["url"]
|
||||
return None
|
||||
|
||||
|
||||
def merge_links_section(body, links):
|
||||
normalized = [x for x in (normalize_link_fact(link) for link in links) if x]
|
||||
if not normalized:
|
||||
return body
|
||||
existing = ""
|
||||
match = re.search(r"(?ms)^## Links\s*\n(.*?)(?=^## |\n\[\[Index|\Z)", body)
|
||||
if match:
|
||||
existing = match.group(1).strip()
|
||||
pairs = []
|
||||
for line in existing.splitlines():
|
||||
url_match = re.search(r"(https?://\S+)", line)
|
||||
if not url_match:
|
||||
continue
|
||||
label_match = re.match(r"^\s*-\s*([^:]+):", line)
|
||||
pairs.append(((label_match.group(1).strip() if label_match else "Link"), url_match.group(1).rstrip(".,)")))
|
||||
pairs.extend(normalized)
|
||||
seen = set()
|
||||
lines = []
|
||||
for label, url in pairs:
|
||||
if url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
lines.append(f"- {label}: {url}")
|
||||
return set_section(body, "Links", "\n".join(lines))
|
||||
|
||||
|
||||
def facts_sort_key(path: str) -> int:
|
||||
match = re.search(r"web_facts(\d*)\.json$", os.path.basename(path))
|
||||
if not match or not match.group(1):
|
||||
return 0
|
||||
return int(match.group(1))
|
||||
|
||||
|
||||
def set_section(body, heading, content):
|
||||
@@ -57,6 +100,9 @@ def apply_one(stem, facts):
|
||||
for k in SCALAR_FIELDS:
|
||||
if facts.get(k):
|
||||
fm[k] = facts[k]
|
||||
for k in LIST_FIELDS:
|
||||
if k in facts:
|
||||
fm[k] = facts[k]
|
||||
if facts.get("type"):
|
||||
fm["type"] = facts["type"]
|
||||
fm["tags"] = E.derive_tags(fm)
|
||||
@@ -77,6 +123,8 @@ def apply_one(stem, facts):
|
||||
body = set_section(body, "Features", feats)
|
||||
if facts.get("notability"):
|
||||
body = set_section(body, "Why it stands out", facts["notability"].strip())
|
||||
if facts.get("links"):
|
||||
body = merge_links_section(body, facts["links"])
|
||||
|
||||
return f"---\n{E.emit_fm(fm)}\n---\n{body}", "ok"
|
||||
|
||||
@@ -84,7 +132,7 @@ def apply_one(stem, facts):
|
||||
def main():
|
||||
apply = "--apply" in sys.argv
|
||||
facts = {}
|
||||
for fp in sorted(glob.glob(FACTS_GLOB)):
|
||||
for fp in sorted(glob.glob(FACTS_GLOB), key=facts_sort_key):
|
||||
facts.update(json.load(open(fp, encoding="utf-8")))
|
||||
ok = miss = 0
|
||||
for stem, f in facts.items():
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Standardize ROM-hack vault pages for site export.
|
||||
|
||||
This pass is intentionally conservative: it uses the current vault note text,
|
||||
existing scrape metadata, and deterministic inference rules. Web-researched facts
|
||||
belong in .scrape/web_facts*.json and should be applied with enrich_web.py.
|
||||
|
||||
Run:
|
||||
python scripts/standardize-vault-pages.py
|
||||
python scripts/standardize-vault-pages.py --apply
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from urllib.parse import urlparse
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import enrich_vault as E
|
||||
|
||||
GEN_ROMAN = {
|
||||
1: "Gen I",
|
||||
2: "Gen II",
|
||||
3: "Gen III",
|
||||
4: "Gen IV",
|
||||
5: "Gen V",
|
||||
6: "Gen VI",
|
||||
7: "Gen VII",
|
||||
8: "Gen VIII",
|
||||
9: "Gen IX",
|
||||
}
|
||||
ROMAN_TO_INT = {
|
||||
"i": 1,
|
||||
"ii": 2,
|
||||
"iii": 3,
|
||||
"iv": 4,
|
||||
"v": 5,
|
||||
"vi": 6,
|
||||
"vii": 7,
|
||||
"viii": 8,
|
||||
"ix": 9,
|
||||
}
|
||||
REGIONS = ["Alolan", "Galarian", "Hisuian", "Paldean"]
|
||||
FAKEMON_GEN_TAG = "Fakemon"
|
||||
PURE_CUSTOM_DEX_STEMS = {
|
||||
"Cope",
|
||||
"Fakemon Fire Red",
|
||||
"Pisces",
|
||||
"Solar Light Lunar Dark",
|
||||
"Touhoumon Another World",
|
||||
"Void",
|
||||
}
|
||||
GENERATION_SORT_ORDER = list(GEN_ROMAN.values()) + REGIONS + [FAKEMON_GEN_TAG]
|
||||
COUNT_TO_GEN = [
|
||||
(1025, 9),
|
||||
(1008, 9),
|
||||
(905, 8),
|
||||
(898, 8),
|
||||
(809, 7),
|
||||
(807, 7),
|
||||
(721, 6),
|
||||
(649, 5),
|
||||
(493, 4),
|
||||
(386, 3),
|
||||
(251, 2),
|
||||
(151, 1),
|
||||
]
|
||||
BASE_MAX_GEN = {
|
||||
"Red": 1,
|
||||
"Red and Blue": 1,
|
||||
"Blue": 1,
|
||||
"Yellow": 1,
|
||||
"Stadium": 1,
|
||||
"Gold": 2,
|
||||
"Silver": 2,
|
||||
"Crystal": 2,
|
||||
"Ruby": 3,
|
||||
"Sapphire": 3,
|
||||
"Emerald": 3,
|
||||
"FireRed": 3,
|
||||
"LeafGreen": 3,
|
||||
"Diamond": 4,
|
||||
"Pearl": 4,
|
||||
"Platinum": 4,
|
||||
"HeartGold": 4,
|
||||
"SoulSilver": 4,
|
||||
"Black": 5,
|
||||
"White": 5,
|
||||
"Black / White": 5,
|
||||
"Pokemon Black & White": 5,
|
||||
"Black 2": 5,
|
||||
"White 2": 5,
|
||||
"X": 6,
|
||||
"Y": 6,
|
||||
"Omega Ruby": 6,
|
||||
"Alpha Sapphire": 6,
|
||||
"Sun": 7,
|
||||
"Moon": 7,
|
||||
"Ultra Sun": 7,
|
||||
"Ultra Moon": 7,
|
||||
"Ultra Sun / Ultra Moon": 7,
|
||||
"Sword": 8,
|
||||
"Shield": 8,
|
||||
"Sword / Shield": 8,
|
||||
"Scarlet": 9,
|
||||
"Violet": 9,
|
||||
"Scarlet / Violet": 9,
|
||||
"XD: Gale of Darkness": 3,
|
||||
}
|
||||
BAD_BASE_VALUES = {
|
||||
"how far you are into the journey",
|
||||
"precedents set",
|
||||
"the north atlantic island nation of Iceland",
|
||||
}
|
||||
|
||||
|
||||
def gen_label_to_int(raw: str) -> int | None:
|
||||
value = raw.strip().lower().replace("generation", "").replace("gen", "").strip()
|
||||
value = value.strip(" .:-")
|
||||
if value.isdigit():
|
||||
n = int(value)
|
||||
return n if 1 <= n <= 9 else None
|
||||
return ROMAN_TO_INT.get(value)
|
||||
|
||||
|
||||
def gens_through(n: int) -> list[str]:
|
||||
return [GEN_ROMAN[i] for i in range(1, n + 1)]
|
||||
|
||||
|
||||
def unique_ordered(values: list[str]) -> list[str]:
|
||||
seen = set()
|
||||
out = []
|
||||
for value in values:
|
||||
if value and value not in seen:
|
||||
seen.add(value)
|
||||
out.append(value)
|
||||
return out
|
||||
|
||||
|
||||
def sort_generations(generations: list[str]) -> list[str]:
|
||||
rank = {label: index for index, label in enumerate(GENERATION_SORT_ORDER)}
|
||||
return unique_ordered(sorted(generations, key=lambda label: rank.get(label, len(GENERATION_SORT_ORDER))))
|
||||
|
||||
|
||||
def apply_fakemon_generation_tag(fm: dict, stem: str) -> list[str]:
|
||||
generations = [g for g in (fm.get("generations") or []) if g != FAKEMON_GEN_TAG]
|
||||
if fm.get("fakemon") != "Yes":
|
||||
return sort_generations(generations)
|
||||
if stem in PURE_CUSTOM_DEX_STEMS:
|
||||
return [FAKEMON_GEN_TAG]
|
||||
return sort_generations(generations + [FAKEMON_GEN_TAG])
|
||||
|
||||
|
||||
def note_sections(body: str) -> dict[str, str]:
|
||||
sections: dict[str, str] = {}
|
||||
for match in re.finditer(r"(?ms)^## ([^\n]+)\s*\n+(.+?)(?=^## |\n\[\[Index|\Z)", body):
|
||||
sections[match.group(1).strip()] = match.group(2).strip()
|
||||
return sections
|
||||
|
||||
|
||||
def set_section(body: str, heading: str, content: str) -> str:
|
||||
block = f"## {heading}\n\n{content.rstrip()}\n\n"
|
||||
pattern = re.compile(rf"(?ms)^## {re.escape(heading)}\s*\n.*?(?=^## |\n\[\[Index|\Z)")
|
||||
if pattern.search(body):
|
||||
return pattern.sub(block, body, count=1)
|
||||
summary = re.search(r"(?ms)^## Summary\s*\n.*?(?=^## |\n\[\[Index|\Z)", body)
|
||||
if summary:
|
||||
return body[: summary.end()] + block + body[summary.end() :]
|
||||
footer = re.search(r"(?m)^\[\[Index", body)
|
||||
if footer:
|
||||
return body[: footer.start()] + block + body[footer.start() :]
|
||||
return body.rstrip() + "\n\n" + block
|
||||
|
||||
|
||||
def clean_summary(text: str) -> str:
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
text = re.sub(r"\s*-\s*(?:Rom link|Download|wiki|Link)\s*:?\s*https?://\S+", "", text, flags=re.I)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def extract_urls(text: str) -> list[str]:
|
||||
urls = []
|
||||
for match in re.finditer(r"https?://[^\s)\]>'\"]+", text):
|
||||
urls.append(match.group(0).rstrip(".,)"))
|
||||
return unique_ordered(urls)
|
||||
|
||||
|
||||
def link_label(url: str) -> str:
|
||||
host = urlparse(url).netloc.lower()
|
||||
if "pokecommunity" in host:
|
||||
return "PokeCommunity"
|
||||
if "hackdex" in host:
|
||||
return "HackDex"
|
||||
if "docs.google" in host or "pastebin" in host:
|
||||
return "Documentation"
|
||||
if "github" in host:
|
||||
return "GitHub"
|
||||
if any(x in host for x in ("mediafire", "mega.nz", "drive.google")):
|
||||
return "Download"
|
||||
if "fandom" in host or "wiki" in host:
|
||||
return "Wiki"
|
||||
return "Link"
|
||||
|
||||
|
||||
def is_placeholder_link(url: str) -> bool:
|
||||
parsed = urlparse(url)
|
||||
host = parsed.netloc.lower()
|
||||
path = parsed.path.strip("/")
|
||||
if "duckduckgo.com" in host:
|
||||
return True
|
||||
if host == "drive.google.com" and not path:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def normalize_links_section(existing: str, extra_urls: list[str]) -> str:
|
||||
pairs: list[tuple[str, str]] = []
|
||||
for line in existing.splitlines():
|
||||
url_match = re.search(r"(https?://\S+)", line)
|
||||
if not url_match:
|
||||
continue
|
||||
url = url_match.group(1).rstrip(".,)")
|
||||
label_match = re.match(r"^\s*-\s*([^:]+):", line)
|
||||
pairs.append(((label_match.group(1).strip() if label_match else link_label(url)), url))
|
||||
for url in extra_urls:
|
||||
pairs.append((link_label(url), url))
|
||||
|
||||
has_real_source = any(not is_placeholder_link(url) for _, url in pairs)
|
||||
seen = set()
|
||||
lines = []
|
||||
for label, url in pairs:
|
||||
if has_real_source and is_placeholder_link(url):
|
||||
continue
|
||||
if url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
lines.append(f"- {label}: {url}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def normalize_features(text: str) -> tuple[str, list[str]]:
|
||||
lines = []
|
||||
extracted_urls = []
|
||||
for raw in text.splitlines():
|
||||
line = raw.strip()
|
||||
if not line:
|
||||
continue
|
||||
line = re.sub(r"^[-*•➡️\s]+", "", line).strip()
|
||||
markdown_link = re.fullmatch(r"\[([^\]]+)\]\((https?://[^)]+)\)", line)
|
||||
if markdown_link:
|
||||
extracted_urls.append(markdown_link.group(2))
|
||||
if re.search(r"document|documentation|more information|wiki|download|link", markdown_link.group(1), re.I):
|
||||
continue
|
||||
urls = extract_urls(line)
|
||||
if urls:
|
||||
extracted_urls.extend(urls)
|
||||
if re.fullmatch(r"(?:Document|Documentation|More information|Wiki|Download|Link)s?\**:?", line.split("http", 1)[0].strip(), re.I):
|
||||
continue
|
||||
line = re.sub(r"\s*https?://\S+", "", line).strip(" -")
|
||||
line = line.strip("* ")
|
||||
if not line or re.fullmatch(r"more informations?", line, re.I):
|
||||
continue
|
||||
if not line.endswith((".", "!", "?")) and len(line) > 80:
|
||||
line += "."
|
||||
lines.append(f"- {line}")
|
||||
return "\n".join(unique_ordered(lines)), extracted_urls
|
||||
|
||||
|
||||
def ignore_generation_context(text: str, start: int, end: int) -> bool:
|
||||
window = text[max(0, start - 35) : min(len(text), end + 45)]
|
||||
if re.search(r"\b(?:dex|national dex|pok[eé]mon|mons|roster|catch|available)\b", window, re.I):
|
||||
return False
|
||||
return bool(re.search(r"\b(?:battle|engine|mechanics?|moves?|abilities|standard|style)\b", window, re.I))
|
||||
|
||||
|
||||
def find_generation_max(text: str) -> int | None:
|
||||
explicit: list[int] = []
|
||||
if re.search(r"\b(?:all generations|all gens|from all gens|from all generations)\b", text, re.I):
|
||||
explicit.append(9)
|
||||
for match in re.finditer(
|
||||
r"\bgen(?:eration)?s?\s*(\d|i{1,3}|iv|v|vi{0,3}|ix)\s*(?:-|–|—|to|through|thru|up to|and)\s*(?:gen(?:eration)?s?\s*)?(\d|i{1,3}|iv|v|vi{0,3}|ix)\b",
|
||||
text,
|
||||
re.I,
|
||||
):
|
||||
if ignore_generation_context(text, match.start(), match.end()):
|
||||
continue
|
||||
a = gen_label_to_int(match.group(1))
|
||||
b = gen_label_to_int(match.group(2))
|
||||
if a and b:
|
||||
explicit.append(max(a, b))
|
||||
for match in re.finditer(r"\b(?:gen(?:eration)?s?|through gen|up to gen)\s*(\d|i{1,3}|iv|v|vi{0,3}|ix)\b", text, re.I):
|
||||
if ignore_generation_context(text, match.start(), match.end()):
|
||||
continue
|
||||
n = gen_label_to_int(match.group(1))
|
||||
if n:
|
||||
explicit.append(n)
|
||||
for count, gen in COUNT_TO_GEN:
|
||||
if re.search(rf"\b{count}\+?\s+(?:pok[eé]mon|mons|national dex|dex)\b", text, re.I):
|
||||
explicit.append(gen)
|
||||
if re.search(rf"\b(?:pok[eé]mon|mons|national dex|dex)\s*(?:up to|through|of)?\s*{count}\+?\b", text, re.I):
|
||||
explicit.append(gen)
|
||||
return max(explicit) if explicit else None
|
||||
|
||||
|
||||
def infer_generations(fm: dict, body: str) -> list[str]:
|
||||
if "generations" in fm:
|
||||
return list(fm["generations"])
|
||||
title = fm.get("title", "")
|
||||
text = f"{title}\n{body}"
|
||||
max_gen = find_generation_max(text)
|
||||
replaces_official = re.search(
|
||||
r"(?:replaces|removes|rids) .{0,80}(?:official\s+)?pok[eé]mon"
|
||||
r"|replaces .{0,80}with .{0,80}(?:touhou characters|boneka)",
|
||||
text,
|
||||
re.I,
|
||||
)
|
||||
custom_species_roster = re.search(
|
||||
r"(?:entirely new|all[- ]new|full new|brand-new|custom)\s+(?:dex|pok[eé]dex|roster|mons|pok[eé]mon)"
|
||||
r"|(?:dex|pok[eé]dex|roster)\s+full of\s+(?:new|custom)?\s*mons"
|
||||
r"|(?:over|more than)\s+\d+\s+original\s+fakemon"
|
||||
r"|\ball\s+\d+\s+boneka\b"
|
||||
r"|\b(?:boneka|touhou characters?)\s+roster\b",
|
||||
text,
|
||||
re.I,
|
||||
)
|
||||
pure_custom_roster = fm.get("fakemon") == "Yes" and (replaces_official or custom_species_roster)
|
||||
regions = [region for region in REGIONS if re.search(rf"\b{region}\b", text, re.I)]
|
||||
if pure_custom_roster:
|
||||
return regions
|
||||
if max_gen is None and not pure_custom_roster:
|
||||
base = fm.get("base")
|
||||
max_gen = BASE_MAX_GEN.get(base)
|
||||
generations = gens_through(max_gen) if max_gen else []
|
||||
return unique_ordered(generations + regions)
|
||||
|
||||
|
||||
def infer_fakemon(fm: dict, body: str) -> str:
|
||||
if fm.get("fakemon") in {"Yes", "No"}:
|
||||
return fm["fakemon"]
|
||||
text = f"{fm.get('title', '')}\n{body}"
|
||||
text = re.sub(r"Fakemon\s+\*\*(?:Yes|No)\*\*", "", text, flags=re.I)
|
||||
text = re.sub(r"\b(?:without|no|not)\s+(?:a\s+)?fakemon(?:\s+roster)?\b", "", text, flags=re.I)
|
||||
strong = [
|
||||
r"\bfakemon\b",
|
||||
r"fan[- ]made pok[eé]mon",
|
||||
r"original (?:fakemon|monsters)",
|
||||
r"custom (?:pok[eé]mon|monsters|mons|pokedex|pok[eé]dex)(?!\s+(?:sprites?|cries?))",
|
||||
r"all[- ]new (?:pok[eé]mon|monsters|pokedex|pok[eé]dex)",
|
||||
r"\d{2,4}\s+(?:new|original|custom)?\s*(?:fakemon|monsters) designed",
|
||||
r"replaces .{0,80}official pok[eé]mon",
|
||||
]
|
||||
if any(re.search(pattern, text, re.I) for pattern in strong):
|
||||
return "Yes"
|
||||
return "No"
|
||||
|
||||
|
||||
def update_callout(body: str, fm: dict) -> str:
|
||||
generations = ", ".join(fm.get("generations") or [])
|
||||
extra = []
|
||||
if generations:
|
||||
extra.append(f"roster **{generations}**")
|
||||
if fm.get("fakemon"):
|
||||
extra.append(f"Fakemon **{fm['fakemon']}**")
|
||||
if not extra:
|
||||
return body
|
||||
lines = body.splitlines()
|
||||
for idx, line in enumerate(lines):
|
||||
if line.startswith("> ") and " · base **" in line:
|
||||
line = re.sub(r"\s*·\s*roster \*\*[^*]+\*\*", "", line)
|
||||
line = re.sub(r"\s*·\s*Fakemon \*\*(?:Yes|No)\*\*", "", line)
|
||||
lines[idx] = line + " · " + " · ".join(extra)
|
||||
return "\n".join(lines)
|
||||
return body
|
||||
|
||||
|
||||
def standardize_note(path: str) -> tuple[bool, dict]:
|
||||
fm_text, body, _ = E.read_note(path)
|
||||
fm = E.parse_fm(fm_text)
|
||||
before_fm = dict(fm)
|
||||
before_body = body
|
||||
sections = note_sections(body)
|
||||
|
||||
if fm.get("base") in BAD_BASE_VALUES:
|
||||
fm["base"] = "—"
|
||||
fm["fakemon"] = infer_fakemon(fm, body)
|
||||
fm["generations"] = apply_fakemon_generation_tag(
|
||||
{**fm, "generations": infer_generations(fm, body)},
|
||||
os.path.splitext(os.path.basename(path))[0],
|
||||
)
|
||||
fm["tags"] = E.derive_tags(fm)
|
||||
|
||||
if sections.get("Summary"):
|
||||
cleaned = clean_summary(sections["Summary"])
|
||||
if cleaned and cleaned != sections["Summary"]:
|
||||
body = set_section(body, "Summary", cleaned)
|
||||
|
||||
extracted_urls: list[str] = []
|
||||
if sections.get("Features"):
|
||||
features, extracted_urls = normalize_features(sections["Features"])
|
||||
if features and features != sections["Features"]:
|
||||
body = set_section(body, "Features", features)
|
||||
|
||||
type_links = E.type_links(fm.get("type") or [])
|
||||
if type_links:
|
||||
body = set_section(body, "Type", type_links)
|
||||
|
||||
current_sections = note_sections(body)
|
||||
if current_sections.get("Links") or extracted_urls:
|
||||
links = normalize_links_section(current_sections.get("Links", ""), extracted_urls)
|
||||
if links:
|
||||
body = set_section(body, "Links", links)
|
||||
|
||||
body = update_callout(body, fm)
|
||||
content = f"---\n{E.emit_fm(fm)}\n---\n{body.rstrip()}\n"
|
||||
changed = before_fm != fm or before_body.rstrip() != body.rstrip()
|
||||
audit = {
|
||||
"stem": os.path.splitext(os.path.basename(path))[0],
|
||||
"fakemon": fm.get("fakemon"),
|
||||
"generations": fm.get("generations") or [],
|
||||
"changed": changed,
|
||||
"had_features": bool(sections.get("Features")),
|
||||
"feature_urls_moved": len(extracted_urls),
|
||||
}
|
||||
return changed, {"content": content, "audit": audit}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
parser.add_argument("--limit", type=int)
|
||||
parser.add_argument("--report", default=os.path.join(E.SCRAPE, "standardize-report.json"))
|
||||
args = parser.parse_args()
|
||||
|
||||
paths = [os.path.join(E.HACKS, fn) for fn in sorted(os.listdir(E.HACKS)) if fn.endswith(".md")]
|
||||
if args.limit:
|
||||
paths = paths[: args.limit]
|
||||
|
||||
changed = 0
|
||||
audits = []
|
||||
for path in paths:
|
||||
did_change, result = standardize_note(path)
|
||||
audits.append(result["audit"])
|
||||
if did_change:
|
||||
changed += 1
|
||||
if args.apply:
|
||||
with open(path, "w", encoding="utf-8", newline="\n") as f:
|
||||
f.write(result["content"])
|
||||
else:
|
||||
os.makedirs(E.PREVIEW, exist_ok=True)
|
||||
with open(os.path.join(E.PREVIEW, os.path.basename(path)), "w", encoding="utf-8", newline="\n") as f:
|
||||
f.write(result["content"])
|
||||
|
||||
summary = {
|
||||
"notes": len(paths),
|
||||
"changed": changed,
|
||||
"fakemon": dict(Counter(a["fakemon"] for a in audits)),
|
||||
"with_generations": sum(1 for a in audits if a["generations"]),
|
||||
"without_generations": sum(1 for a in audits if not a["generations"]),
|
||||
"feature_urls_moved": sum(a["feature_urls_moved"] for a in audits),
|
||||
"audits": audits,
|
||||
}
|
||||
os.makedirs(os.path.dirname(args.report), exist_ok=True)
|
||||
with open(args.report, "w", encoding="utf-8") as f:
|
||||
json.dump(summary, f, indent=2, ensure_ascii=False)
|
||||
print(f"standardized {len(paths)} notes; changed={changed}; apply={args.apply}")
|
||||
print(f"fakemon={summary['fakemon']}")
|
||||
print(f"with_generations={summary['with_generations']} without_generations={summary['without_generations']}")
|
||||
print(f"feature_urls_moved={summary['feature_urls_moved']}")
|
||||
print(f"report={args.report}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user