Complete catalog standardization: generations, banners, and Fakemon roster tag.
Adds standardize-vault-pages pipeline, exports intentional empty generations, and refreshes the 396-hack catalog with full banner coverage, status classification, and Fakemon generation tags.
This commit is contained in:
@@ -100,10 +100,13 @@ def wikilink(stem: str) -> str:
|
|||||||
# ---------------------------------------------------------------- MOC config
|
# ---------------------------------------------------------------- MOC config
|
||||||
|
|
||||||
# Logical hardware order for human-facing lists.
|
# Logical hardware order for human-facing lists.
|
||||||
PLATFORM_ORDER = ["GB", "GBC", "GBA", "NDS", "3DS", "PC", "Patch", DASH]
|
PLATFORM_ORDER = ["GB", "GBC", "GBA", "NDS", "3DS", "N64", "GameCube", "Switch", "PC", "Patch", DASH]
|
||||||
PLATFORM_LABEL = {
|
PLATFORM_LABEL = {
|
||||||
"GB": "Game Boy", "GBC": "Game Boy Color", "GBA": "Game Boy Advance",
|
"GB": "Game Boy", "GBC": "Game Boy Color", "GBA": "Game Boy Advance",
|
||||||
"NDS": "Nintendo DS", "3DS": "Nintendo 3DS", "PC": "PC / Joiplay fan-games",
|
"NDS": "Nintendo DS", "3DS": "Nintendo 3DS", "N64": "Nintendo 64",
|
||||||
|
"GameCube": "Nintendo GameCube",
|
||||||
|
"Switch": "Nintendo Switch",
|
||||||
|
"PC": "PC / Joiplay fan-games",
|
||||||
"Patch": "Patch-only", DASH: "Unknown platform",
|
"Patch": "Patch-only", DASH: "Unknown platform",
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -117,6 +120,9 @@ PLATFORM_MOCS = {
|
|||||||
"GB": ("Game Boy (GB)", plat_pred("GB")),
|
"GB": ("Game Boy (GB)", plat_pred("GB")),
|
||||||
"NDS": ("Nintendo DS (NDS)", plat_pred("NDS")),
|
"NDS": ("Nintendo DS (NDS)", plat_pred("NDS")),
|
||||||
"3DS": ("Nintendo 3DS (3DS)", plat_pred("3DS")),
|
"3DS": ("Nintendo 3DS (3DS)", plat_pred("3DS")),
|
||||||
|
"N64": ("Nintendo 64 (N64)", plat_pred("N64")),
|
||||||
|
"GameCube": ("Nintendo GameCube", plat_pred("GameCube")),
|
||||||
|
"Switch": ("Nintendo Switch", plat_pred("Switch")),
|
||||||
"PC": ("PC / Joiplay Fan-games", plat_pred("PC")),
|
"PC": ("PC / Joiplay Fan-games", plat_pred("PC")),
|
||||||
"Patch": ("Patch-only", plat_pred("Patch")),
|
"Patch": ("Patch-only", plat_pred("Patch")),
|
||||||
"Unknown": ("Unknown platform", plat_pred(DASH)),
|
"Unknown": ("Unknown platform", plat_pred(DASH)),
|
||||||
@@ -128,6 +134,18 @@ NEW_PLATFORM_MOCS = {
|
|||||||
"Nintendo 3DS Pokémon fan-games and hacks. A small but growing set — "
|
"Nintendo 3DS Pokémon fan-games and hacks. A small but growing set — "
|
||||||
"mostly Citra/Luma-targeted projects harvested from the Discord catalog.",
|
"mostly Citra/Luma-targeted projects harvested from the Discord catalog.",
|
||||||
'WHERE platform = "3DS"'),
|
'WHERE platform = "3DS"'),
|
||||||
|
"N64": ("N64",
|
||||||
|
"Nintendo 64 Pokémon mods and overhaul patches. These are tracked "
|
||||||
|
"separately from handheld ROM hacks because they target Nintendo 64-era base games.",
|
||||||
|
'WHERE platform = "N64"'),
|
||||||
|
"GameCube": ("GameCube",
|
||||||
|
"Nintendo GameCube Pokémon mods and overhaul patches. These are tracked "
|
||||||
|
"separately from handheld ROM hacks because they target GameCube-era base games.",
|
||||||
|
'WHERE platform = "GameCube"'),
|
||||||
|
"Switch": ("Switch",
|
||||||
|
"Nintendo Switch Pokémon mods and overhaul patches. These are tracked "
|
||||||
|
"separately from handheld ROM hacks because they target Switch-era base games.",
|
||||||
|
'WHERE platform = "Switch"'),
|
||||||
"PC": ("PC",
|
"PC": ("PC",
|
||||||
"RPG-Maker / Joiplay fan-games (run on PC or via Joiplay on Android). "
|
"RPG-Maker / Joiplay fan-games (run on PC or via Joiplay on Android). "
|
||||||
"These are standalone games, **not** console ROM hacks — they live under "
|
"These are standalone games, **not** console ROM hacks — they live under "
|
||||||
|
|||||||
@@ -20,8 +20,9 @@ OUT_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__)
|
|||||||
OUT = os.path.join(OUT_DIR, "catalog.json")
|
OUT = os.path.join(OUT_DIR, "catalog.json")
|
||||||
|
|
||||||
SCALARS = ["title", "platform", "base", "version", "status", "generation",
|
SCALARS = ["title", "platform", "base", "version", "status", "generation",
|
||||||
"developer", "release_date", "banner", "homepage", "source",
|
"fakemon", "developer", "release_date", "banner", "homepage", "source",
|
||||||
"library_path", "play_status", "scrape_dir", "engine"]
|
"library_path", "play_status", "scrape_dir", "engine"]
|
||||||
|
LIST_SCALARS = ["generations"]
|
||||||
|
|
||||||
|
|
||||||
def section(body, heading):
|
def section(body, heading):
|
||||||
@@ -57,6 +58,9 @@ def main():
|
|||||||
for k in SCALARS:
|
for k in SCALARS:
|
||||||
if fm.get(k) not in (None, ""):
|
if fm.get(k) not in (None, ""):
|
||||||
rec[k] = fm[k]
|
rec[k] = fm[k]
|
||||||
|
for k in LIST_SCALARS:
|
||||||
|
if k in fm:
|
||||||
|
rec[k] = fm[k]
|
||||||
rec["type"] = fm.get("type") or []
|
rec["type"] = fm.get("type") or []
|
||||||
rec["tags"] = fm.get("tags") or []
|
rec["tags"] = fm.get("tags") or []
|
||||||
if fm.get("rating") not in (None, ""):
|
if fm.get("rating") not in (None, ""):
|
||||||
|
|||||||
+10
-2
@@ -53,6 +53,9 @@ PLATFORM_HINTS = [
|
|||||||
(r"\bGBC\b|game boy color", ("GBC", "console")),
|
(r"\bGBC\b|game boy color", ("GBC", "console")),
|
||||||
(r"\bNDS\b|nintendo ds\b|\bDS ROM", ("NDS", "console")),
|
(r"\bNDS\b|nintendo ds\b|\bDS ROM", ("NDS", "console")),
|
||||||
(r"\b3DS\b", ("3DS", "console")),
|
(r"\b3DS\b", ("3DS", "console")),
|
||||||
|
(r"\bN64\b|Nintendo 64|Pokemon Stadium|Pokémon Stadium", ("N64", "console")),
|
||||||
|
(r"\bGameCube\b|Nintendo GameCube|Pokemon Colosseum|Pokémon Colosseum", ("GameCube", "console")),
|
||||||
|
(r"\bSwitch\b|Nintendo Switch", ("Switch", "console")),
|
||||||
(r"\bGB\b|game boy(?! advance| color)", ("GB", "console")),
|
(r"\bGB\b|game boy(?! advance| color)", ("GB", "console")),
|
||||||
(r"RPGXP|RPG ?Maker|Essentials|FanGame|fan game|GameMaker", ("PC", "fangame")),
|
(r"RPGXP|RPG ?Maker|Essentials|FanGame|fan game|GameMaker", ("PC", "fangame")),
|
||||||
]
|
]
|
||||||
@@ -256,6 +259,8 @@ GAME_PLATFORM = {
|
|||||||
"soul silver": ("NDS", "SoulSilver"), "black": ("NDS", "Black"),
|
"soul silver": ("NDS", "SoulSilver"), "black": ("NDS", "Black"),
|
||||||
"white": ("NDS", "White"), "black 2": ("NDS", "Black 2"),
|
"white": ("NDS", "White"), "black 2": ("NDS", "Black 2"),
|
||||||
"white 2": ("NDS", "White 2"),
|
"white 2": ("NDS", "White 2"),
|
||||||
|
"stadium": ("N64", "Stadium"), "pokemon stadium": ("N64", "Stadium"),
|
||||||
|
"pokémon stadium": ("N64", "Stadium"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -389,7 +394,7 @@ def load_scrape():
|
|||||||
# ---------------------------------------------------------------- frontmatter (de)serialize
|
# ---------------------------------------------------------------- frontmatter (de)serialize
|
||||||
|
|
||||||
FM_ORDER = ["title", "platform", "base", "version", "status", "type",
|
FM_ORDER = ["title", "platform", "base", "version", "status", "type",
|
||||||
"generation", "developer", "release_date", "banner",
|
"generation", "generations", "fakemon", "developer", "release_date", "banner",
|
||||||
"library_path", "source", "homepage", "scrape_dir",
|
"library_path", "source", "homepage", "scrape_dir",
|
||||||
"added", "play_status", "rating", "tags"]
|
"added", "play_status", "rating", "tags"]
|
||||||
LIST_KEYS = {"type", "tags"}
|
LIST_KEYS = {"type", "tags"}
|
||||||
@@ -462,7 +467,10 @@ def canon_status(s):
|
|||||||
|
|
||||||
PLATFORM_LABEL = {"GB": "Game Boy", "GBC": "Game Boy Color",
|
PLATFORM_LABEL = {"GB": "Game Boy", "GBC": "Game Boy Color",
|
||||||
"GBA": "Game Boy Advance", "NDS": "Nintendo DS",
|
"GBA": "Game Boy Advance", "NDS": "Nintendo DS",
|
||||||
"3DS": "Nintendo 3DS", "PC": "PC / Joiplay", "—": "Unknown"}
|
"3DS": "Nintendo 3DS", "N64": "Nintendo 64",
|
||||||
|
"GameCube": "Nintendo GameCube",
|
||||||
|
"Switch": "Nintendo Switch",
|
||||||
|
"PC": "PC / Joiplay", "—": "Unknown"}
|
||||||
|
|
||||||
|
|
||||||
def derive_tags(fm):
|
def derive_tags(fm):
|
||||||
|
|||||||
+50
-2
@@ -20,7 +20,50 @@ import enrich_vault as E
|
|||||||
|
|
||||||
FACTS_GLOB = os.path.join(E.SCRAPE, "web_facts*.json")
|
FACTS_GLOB = os.path.join(E.SCRAPE, "web_facts*.json")
|
||||||
SCALAR_FIELDS = ["tagline", "developer", "version", "status", "release_date",
|
SCALAR_FIELDS = ["tagline", "developer", "version", "status", "release_date",
|
||||||
"base", "platform", "generation", "homepage"]
|
"base", "platform", "generation", "homepage", "source", "fakemon",
|
||||||
|
"banner"]
|
||||||
|
LIST_FIELDS = ["generations"]
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_link_fact(link):
|
||||||
|
if isinstance(link, str):
|
||||||
|
return "Link", link
|
||||||
|
if isinstance(link, dict) and link.get("url"):
|
||||||
|
return link.get("label") or "Link", link["url"]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def merge_links_section(body, links):
|
||||||
|
normalized = [x for x in (normalize_link_fact(link) for link in links) if x]
|
||||||
|
if not normalized:
|
||||||
|
return body
|
||||||
|
existing = ""
|
||||||
|
match = re.search(r"(?ms)^## Links\s*\n(.*?)(?=^## |\n\[\[Index|\Z)", body)
|
||||||
|
if match:
|
||||||
|
existing = match.group(1).strip()
|
||||||
|
pairs = []
|
||||||
|
for line in existing.splitlines():
|
||||||
|
url_match = re.search(r"(https?://\S+)", line)
|
||||||
|
if not url_match:
|
||||||
|
continue
|
||||||
|
label_match = re.match(r"^\s*-\s*([^:]+):", line)
|
||||||
|
pairs.append(((label_match.group(1).strip() if label_match else "Link"), url_match.group(1).rstrip(".,)")))
|
||||||
|
pairs.extend(normalized)
|
||||||
|
seen = set()
|
||||||
|
lines = []
|
||||||
|
for label, url in pairs:
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
lines.append(f"- {label}: {url}")
|
||||||
|
return set_section(body, "Links", "\n".join(lines))
|
||||||
|
|
||||||
|
|
||||||
|
def facts_sort_key(path: str) -> int:
|
||||||
|
match = re.search(r"web_facts(\d*)\.json$", os.path.basename(path))
|
||||||
|
if not match or not match.group(1):
|
||||||
|
return 0
|
||||||
|
return int(match.group(1))
|
||||||
|
|
||||||
|
|
||||||
def set_section(body, heading, content):
|
def set_section(body, heading, content):
|
||||||
@@ -57,6 +100,9 @@ def apply_one(stem, facts):
|
|||||||
for k in SCALAR_FIELDS:
|
for k in SCALAR_FIELDS:
|
||||||
if facts.get(k):
|
if facts.get(k):
|
||||||
fm[k] = facts[k]
|
fm[k] = facts[k]
|
||||||
|
for k in LIST_FIELDS:
|
||||||
|
if k in facts:
|
||||||
|
fm[k] = facts[k]
|
||||||
if facts.get("type"):
|
if facts.get("type"):
|
||||||
fm["type"] = facts["type"]
|
fm["type"] = facts["type"]
|
||||||
fm["tags"] = E.derive_tags(fm)
|
fm["tags"] = E.derive_tags(fm)
|
||||||
@@ -77,6 +123,8 @@ def apply_one(stem, facts):
|
|||||||
body = set_section(body, "Features", feats)
|
body = set_section(body, "Features", feats)
|
||||||
if facts.get("notability"):
|
if facts.get("notability"):
|
||||||
body = set_section(body, "Why it stands out", facts["notability"].strip())
|
body = set_section(body, "Why it stands out", facts["notability"].strip())
|
||||||
|
if facts.get("links"):
|
||||||
|
body = merge_links_section(body, facts["links"])
|
||||||
|
|
||||||
return f"---\n{E.emit_fm(fm)}\n---\n{body}", "ok"
|
return f"---\n{E.emit_fm(fm)}\n---\n{body}", "ok"
|
||||||
|
|
||||||
@@ -84,7 +132,7 @@ def apply_one(stem, facts):
|
|||||||
def main():
|
def main():
|
||||||
apply = "--apply" in sys.argv
|
apply = "--apply" in sys.argv
|
||||||
facts = {}
|
facts = {}
|
||||||
for fp in sorted(glob.glob(FACTS_GLOB)):
|
for fp in sorted(glob.glob(FACTS_GLOB), key=facts_sort_key):
|
||||||
facts.update(json.load(open(fp, encoding="utf-8")))
|
facts.update(json.load(open(fp, encoding="utf-8")))
|
||||||
ok = miss = 0
|
ok = miss = 0
|
||||||
for stem, f in facts.items():
|
for stem, f in facts.items():
|
||||||
|
|||||||
@@ -0,0 +1,478 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Standardize ROM-hack vault pages for site export.
|
||||||
|
|
||||||
|
This pass is intentionally conservative: it uses the current vault note text,
|
||||||
|
existing scrape metadata, and deterministic inference rules. Web-researched facts
|
||||||
|
belong in .scrape/web_facts*.json and should be applied with enrich_web.py.
|
||||||
|
|
||||||
|
Run:
|
||||||
|
python scripts/standardize-vault-pages.py
|
||||||
|
python scripts/standardize-vault-pages.py --apply
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from collections import Counter
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
import enrich_vault as E
|
||||||
|
|
||||||
|
GEN_ROMAN = {
|
||||||
|
1: "Gen I",
|
||||||
|
2: "Gen II",
|
||||||
|
3: "Gen III",
|
||||||
|
4: "Gen IV",
|
||||||
|
5: "Gen V",
|
||||||
|
6: "Gen VI",
|
||||||
|
7: "Gen VII",
|
||||||
|
8: "Gen VIII",
|
||||||
|
9: "Gen IX",
|
||||||
|
}
|
||||||
|
ROMAN_TO_INT = {
|
||||||
|
"i": 1,
|
||||||
|
"ii": 2,
|
||||||
|
"iii": 3,
|
||||||
|
"iv": 4,
|
||||||
|
"v": 5,
|
||||||
|
"vi": 6,
|
||||||
|
"vii": 7,
|
||||||
|
"viii": 8,
|
||||||
|
"ix": 9,
|
||||||
|
}
|
||||||
|
REGIONS = ["Alolan", "Galarian", "Hisuian", "Paldean"]
|
||||||
|
FAKEMON_GEN_TAG = "Fakemon"
|
||||||
|
PURE_CUSTOM_DEX_STEMS = {
|
||||||
|
"Cope",
|
||||||
|
"Fakemon Fire Red",
|
||||||
|
"Pisces",
|
||||||
|
"Solar Light Lunar Dark",
|
||||||
|
"Touhoumon Another World",
|
||||||
|
"Void",
|
||||||
|
}
|
||||||
|
GENERATION_SORT_ORDER = list(GEN_ROMAN.values()) + REGIONS + [FAKEMON_GEN_TAG]
|
||||||
|
COUNT_TO_GEN = [
|
||||||
|
(1025, 9),
|
||||||
|
(1008, 9),
|
||||||
|
(905, 8),
|
||||||
|
(898, 8),
|
||||||
|
(809, 7),
|
||||||
|
(807, 7),
|
||||||
|
(721, 6),
|
||||||
|
(649, 5),
|
||||||
|
(493, 4),
|
||||||
|
(386, 3),
|
||||||
|
(251, 2),
|
||||||
|
(151, 1),
|
||||||
|
]
|
||||||
|
BASE_MAX_GEN = {
|
||||||
|
"Red": 1,
|
||||||
|
"Red and Blue": 1,
|
||||||
|
"Blue": 1,
|
||||||
|
"Yellow": 1,
|
||||||
|
"Stadium": 1,
|
||||||
|
"Gold": 2,
|
||||||
|
"Silver": 2,
|
||||||
|
"Crystal": 2,
|
||||||
|
"Ruby": 3,
|
||||||
|
"Sapphire": 3,
|
||||||
|
"Emerald": 3,
|
||||||
|
"FireRed": 3,
|
||||||
|
"LeafGreen": 3,
|
||||||
|
"Diamond": 4,
|
||||||
|
"Pearl": 4,
|
||||||
|
"Platinum": 4,
|
||||||
|
"HeartGold": 4,
|
||||||
|
"SoulSilver": 4,
|
||||||
|
"Black": 5,
|
||||||
|
"White": 5,
|
||||||
|
"Black / White": 5,
|
||||||
|
"Pokemon Black & White": 5,
|
||||||
|
"Black 2": 5,
|
||||||
|
"White 2": 5,
|
||||||
|
"X": 6,
|
||||||
|
"Y": 6,
|
||||||
|
"Omega Ruby": 6,
|
||||||
|
"Alpha Sapphire": 6,
|
||||||
|
"Sun": 7,
|
||||||
|
"Moon": 7,
|
||||||
|
"Ultra Sun": 7,
|
||||||
|
"Ultra Moon": 7,
|
||||||
|
"Ultra Sun / Ultra Moon": 7,
|
||||||
|
"Sword": 8,
|
||||||
|
"Shield": 8,
|
||||||
|
"Sword / Shield": 8,
|
||||||
|
"Scarlet": 9,
|
||||||
|
"Violet": 9,
|
||||||
|
"Scarlet / Violet": 9,
|
||||||
|
"XD: Gale of Darkness": 3,
|
||||||
|
}
|
||||||
|
BAD_BASE_VALUES = {
|
||||||
|
"how far you are into the journey",
|
||||||
|
"precedents set",
|
||||||
|
"the north atlantic island nation of Iceland",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def gen_label_to_int(raw: str) -> int | None:
|
||||||
|
value = raw.strip().lower().replace("generation", "").replace("gen", "").strip()
|
||||||
|
value = value.strip(" .:-")
|
||||||
|
if value.isdigit():
|
||||||
|
n = int(value)
|
||||||
|
return n if 1 <= n <= 9 else None
|
||||||
|
return ROMAN_TO_INT.get(value)
|
||||||
|
|
||||||
|
|
||||||
|
def gens_through(n: int) -> list[str]:
|
||||||
|
return [GEN_ROMAN[i] for i in range(1, n + 1)]
|
||||||
|
|
||||||
|
|
||||||
|
def unique_ordered(values: list[str]) -> list[str]:
|
||||||
|
seen = set()
|
||||||
|
out = []
|
||||||
|
for value in values:
|
||||||
|
if value and value not in seen:
|
||||||
|
seen.add(value)
|
||||||
|
out.append(value)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def sort_generations(generations: list[str]) -> list[str]:
|
||||||
|
rank = {label: index for index, label in enumerate(GENERATION_SORT_ORDER)}
|
||||||
|
return unique_ordered(sorted(generations, key=lambda label: rank.get(label, len(GENERATION_SORT_ORDER))))
|
||||||
|
|
||||||
|
|
||||||
|
def apply_fakemon_generation_tag(fm: dict, stem: str) -> list[str]:
|
||||||
|
generations = [g for g in (fm.get("generations") or []) if g != FAKEMON_GEN_TAG]
|
||||||
|
if fm.get("fakemon") != "Yes":
|
||||||
|
return sort_generations(generations)
|
||||||
|
if stem in PURE_CUSTOM_DEX_STEMS:
|
||||||
|
return [FAKEMON_GEN_TAG]
|
||||||
|
return sort_generations(generations + [FAKEMON_GEN_TAG])
|
||||||
|
|
||||||
|
|
||||||
|
def note_sections(body: str) -> dict[str, str]:
|
||||||
|
sections: dict[str, str] = {}
|
||||||
|
for match in re.finditer(r"(?ms)^## ([^\n]+)\s*\n+(.+?)(?=^## |\n\[\[Index|\Z)", body):
|
||||||
|
sections[match.group(1).strip()] = match.group(2).strip()
|
||||||
|
return sections
|
||||||
|
|
||||||
|
|
||||||
|
def set_section(body: str, heading: str, content: str) -> str:
|
||||||
|
block = f"## {heading}\n\n{content.rstrip()}\n\n"
|
||||||
|
pattern = re.compile(rf"(?ms)^## {re.escape(heading)}\s*\n.*?(?=^## |\n\[\[Index|\Z)")
|
||||||
|
if pattern.search(body):
|
||||||
|
return pattern.sub(block, body, count=1)
|
||||||
|
summary = re.search(r"(?ms)^## Summary\s*\n.*?(?=^## |\n\[\[Index|\Z)", body)
|
||||||
|
if summary:
|
||||||
|
return body[: summary.end()] + block + body[summary.end() :]
|
||||||
|
footer = re.search(r"(?m)^\[\[Index", body)
|
||||||
|
if footer:
|
||||||
|
return body[: footer.start()] + block + body[footer.start() :]
|
||||||
|
return body.rstrip() + "\n\n" + block
|
||||||
|
|
||||||
|
|
||||||
|
def clean_summary(text: str) -> str:
|
||||||
|
text = re.sub(r"\s+", " ", text).strip()
|
||||||
|
text = re.sub(r"\s*-\s*(?:Rom link|Download|wiki|Link)\s*:?\s*https?://\S+", "", text, flags=re.I)
|
||||||
|
return text.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def extract_urls(text: str) -> list[str]:
|
||||||
|
urls = []
|
||||||
|
for match in re.finditer(r"https?://[^\s)\]>'\"]+", text):
|
||||||
|
urls.append(match.group(0).rstrip(".,)"))
|
||||||
|
return unique_ordered(urls)
|
||||||
|
|
||||||
|
|
||||||
|
def link_label(url: str) -> str:
|
||||||
|
host = urlparse(url).netloc.lower()
|
||||||
|
if "pokecommunity" in host:
|
||||||
|
return "PokeCommunity"
|
||||||
|
if "hackdex" in host:
|
||||||
|
return "HackDex"
|
||||||
|
if "docs.google" in host or "pastebin" in host:
|
||||||
|
return "Documentation"
|
||||||
|
if "github" in host:
|
||||||
|
return "GitHub"
|
||||||
|
if any(x in host for x in ("mediafire", "mega.nz", "drive.google")):
|
||||||
|
return "Download"
|
||||||
|
if "fandom" in host or "wiki" in host:
|
||||||
|
return "Wiki"
|
||||||
|
return "Link"
|
||||||
|
|
||||||
|
|
||||||
|
def is_placeholder_link(url: str) -> bool:
|
||||||
|
parsed = urlparse(url)
|
||||||
|
host = parsed.netloc.lower()
|
||||||
|
path = parsed.path.strip("/")
|
||||||
|
if "duckduckgo.com" in host:
|
||||||
|
return True
|
||||||
|
if host == "drive.google.com" and not path:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_links_section(existing: str, extra_urls: list[str]) -> str:
|
||||||
|
pairs: list[tuple[str, str]] = []
|
||||||
|
for line in existing.splitlines():
|
||||||
|
url_match = re.search(r"(https?://\S+)", line)
|
||||||
|
if not url_match:
|
||||||
|
continue
|
||||||
|
url = url_match.group(1).rstrip(".,)")
|
||||||
|
label_match = re.match(r"^\s*-\s*([^:]+):", line)
|
||||||
|
pairs.append(((label_match.group(1).strip() if label_match else link_label(url)), url))
|
||||||
|
for url in extra_urls:
|
||||||
|
pairs.append((link_label(url), url))
|
||||||
|
|
||||||
|
has_real_source = any(not is_placeholder_link(url) for _, url in pairs)
|
||||||
|
seen = set()
|
||||||
|
lines = []
|
||||||
|
for label, url in pairs:
|
||||||
|
if has_real_source and is_placeholder_link(url):
|
||||||
|
continue
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
lines.append(f"- {label}: {url}")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_features(text: str) -> tuple[str, list[str]]:
|
||||||
|
lines = []
|
||||||
|
extracted_urls = []
|
||||||
|
for raw in text.splitlines():
|
||||||
|
line = raw.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
line = re.sub(r"^[-*•➡️\s]+", "", line).strip()
|
||||||
|
markdown_link = re.fullmatch(r"\[([^\]]+)\]\((https?://[^)]+)\)", line)
|
||||||
|
if markdown_link:
|
||||||
|
extracted_urls.append(markdown_link.group(2))
|
||||||
|
if re.search(r"document|documentation|more information|wiki|download|link", markdown_link.group(1), re.I):
|
||||||
|
continue
|
||||||
|
urls = extract_urls(line)
|
||||||
|
if urls:
|
||||||
|
extracted_urls.extend(urls)
|
||||||
|
if re.fullmatch(r"(?:Document|Documentation|More information|Wiki|Download|Link)s?\**:?", line.split("http", 1)[0].strip(), re.I):
|
||||||
|
continue
|
||||||
|
line = re.sub(r"\s*https?://\S+", "", line).strip(" -")
|
||||||
|
line = line.strip("* ")
|
||||||
|
if not line or re.fullmatch(r"more informations?", line, re.I):
|
||||||
|
continue
|
||||||
|
if not line.endswith((".", "!", "?")) and len(line) > 80:
|
||||||
|
line += "."
|
||||||
|
lines.append(f"- {line}")
|
||||||
|
return "\n".join(unique_ordered(lines)), extracted_urls
|
||||||
|
|
||||||
|
|
||||||
|
def ignore_generation_context(text: str, start: int, end: int) -> bool:
|
||||||
|
window = text[max(0, start - 35) : min(len(text), end + 45)]
|
||||||
|
if re.search(r"\b(?:dex|national dex|pok[eé]mon|mons|roster|catch|available)\b", window, re.I):
|
||||||
|
return False
|
||||||
|
return bool(re.search(r"\b(?:battle|engine|mechanics?|moves?|abilities|standard|style)\b", window, re.I))
|
||||||
|
|
||||||
|
|
||||||
|
def find_generation_max(text: str) -> int | None:
|
||||||
|
explicit: list[int] = []
|
||||||
|
if re.search(r"\b(?:all generations|all gens|from all gens|from all generations)\b", text, re.I):
|
||||||
|
explicit.append(9)
|
||||||
|
for match in re.finditer(
|
||||||
|
r"\bgen(?:eration)?s?\s*(\d|i{1,3}|iv|v|vi{0,3}|ix)\s*(?:-|–|—|to|through|thru|up to|and)\s*(?:gen(?:eration)?s?\s*)?(\d|i{1,3}|iv|v|vi{0,3}|ix)\b",
|
||||||
|
text,
|
||||||
|
re.I,
|
||||||
|
):
|
||||||
|
if ignore_generation_context(text, match.start(), match.end()):
|
||||||
|
continue
|
||||||
|
a = gen_label_to_int(match.group(1))
|
||||||
|
b = gen_label_to_int(match.group(2))
|
||||||
|
if a and b:
|
||||||
|
explicit.append(max(a, b))
|
||||||
|
for match in re.finditer(r"\b(?:gen(?:eration)?s?|through gen|up to gen)\s*(\d|i{1,3}|iv|v|vi{0,3}|ix)\b", text, re.I):
|
||||||
|
if ignore_generation_context(text, match.start(), match.end()):
|
||||||
|
continue
|
||||||
|
n = gen_label_to_int(match.group(1))
|
||||||
|
if n:
|
||||||
|
explicit.append(n)
|
||||||
|
for count, gen in COUNT_TO_GEN:
|
||||||
|
if re.search(rf"\b{count}\+?\s+(?:pok[eé]mon|mons|national dex|dex)\b", text, re.I):
|
||||||
|
explicit.append(gen)
|
||||||
|
if re.search(rf"\b(?:pok[eé]mon|mons|national dex|dex)\s*(?:up to|through|of)?\s*{count}\+?\b", text, re.I):
|
||||||
|
explicit.append(gen)
|
||||||
|
return max(explicit) if explicit else None
|
||||||
|
|
||||||
|
|
||||||
|
def infer_generations(fm: dict, body: str) -> list[str]:
|
||||||
|
if "generations" in fm:
|
||||||
|
return list(fm["generations"])
|
||||||
|
title = fm.get("title", "")
|
||||||
|
text = f"{title}\n{body}"
|
||||||
|
max_gen = find_generation_max(text)
|
||||||
|
replaces_official = re.search(
|
||||||
|
r"(?:replaces|removes|rids) .{0,80}(?:official\s+)?pok[eé]mon"
|
||||||
|
r"|replaces .{0,80}with .{0,80}(?:touhou characters|boneka)",
|
||||||
|
text,
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
custom_species_roster = re.search(
|
||||||
|
r"(?:entirely new|all[- ]new|full new|brand-new|custom)\s+(?:dex|pok[eé]dex|roster|mons|pok[eé]mon)"
|
||||||
|
r"|(?:dex|pok[eé]dex|roster)\s+full of\s+(?:new|custom)?\s*mons"
|
||||||
|
r"|(?:over|more than)\s+\d+\s+original\s+fakemon"
|
||||||
|
r"|\ball\s+\d+\s+boneka\b"
|
||||||
|
r"|\b(?:boneka|touhou characters?)\s+roster\b",
|
||||||
|
text,
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
pure_custom_roster = fm.get("fakemon") == "Yes" and (replaces_official or custom_species_roster)
|
||||||
|
regions = [region for region in REGIONS if re.search(rf"\b{region}\b", text, re.I)]
|
||||||
|
if pure_custom_roster:
|
||||||
|
return regions
|
||||||
|
if max_gen is None and not pure_custom_roster:
|
||||||
|
base = fm.get("base")
|
||||||
|
max_gen = BASE_MAX_GEN.get(base)
|
||||||
|
generations = gens_through(max_gen) if max_gen else []
|
||||||
|
return unique_ordered(generations + regions)
|
||||||
|
|
||||||
|
|
||||||
|
def infer_fakemon(fm: dict, body: str) -> str:
|
||||||
|
if fm.get("fakemon") in {"Yes", "No"}:
|
||||||
|
return fm["fakemon"]
|
||||||
|
text = f"{fm.get('title', '')}\n{body}"
|
||||||
|
text = re.sub(r"Fakemon\s+\*\*(?:Yes|No)\*\*", "", text, flags=re.I)
|
||||||
|
text = re.sub(r"\b(?:without|no|not)\s+(?:a\s+)?fakemon(?:\s+roster)?\b", "", text, flags=re.I)
|
||||||
|
strong = [
|
||||||
|
r"\bfakemon\b",
|
||||||
|
r"fan[- ]made pok[eé]mon",
|
||||||
|
r"original (?:fakemon|monsters)",
|
||||||
|
r"custom (?:pok[eé]mon|monsters|mons|pokedex|pok[eé]dex)(?!\s+(?:sprites?|cries?))",
|
||||||
|
r"all[- ]new (?:pok[eé]mon|monsters|pokedex|pok[eé]dex)",
|
||||||
|
r"\d{2,4}\s+(?:new|original|custom)?\s*(?:fakemon|monsters) designed",
|
||||||
|
r"replaces .{0,80}official pok[eé]mon",
|
||||||
|
]
|
||||||
|
if any(re.search(pattern, text, re.I) for pattern in strong):
|
||||||
|
return "Yes"
|
||||||
|
return "No"
|
||||||
|
|
||||||
|
|
||||||
|
def update_callout(body: str, fm: dict) -> str:
|
||||||
|
generations = ", ".join(fm.get("generations") or [])
|
||||||
|
extra = []
|
||||||
|
if generations:
|
||||||
|
extra.append(f"roster **{generations}**")
|
||||||
|
if fm.get("fakemon"):
|
||||||
|
extra.append(f"Fakemon **{fm['fakemon']}**")
|
||||||
|
if not extra:
|
||||||
|
return body
|
||||||
|
lines = body.splitlines()
|
||||||
|
for idx, line in enumerate(lines):
|
||||||
|
if line.startswith("> ") and " · base **" in line:
|
||||||
|
line = re.sub(r"\s*·\s*roster \*\*[^*]+\*\*", "", line)
|
||||||
|
line = re.sub(r"\s*·\s*Fakemon \*\*(?:Yes|No)\*\*", "", line)
|
||||||
|
lines[idx] = line + " · " + " · ".join(extra)
|
||||||
|
return "\n".join(lines)
|
||||||
|
return body
|
||||||
|
|
||||||
|
|
||||||
|
def standardize_note(path: str) -> tuple[bool, dict]:
|
||||||
|
fm_text, body, _ = E.read_note(path)
|
||||||
|
fm = E.parse_fm(fm_text)
|
||||||
|
before_fm = dict(fm)
|
||||||
|
before_body = body
|
||||||
|
sections = note_sections(body)
|
||||||
|
|
||||||
|
if fm.get("base") in BAD_BASE_VALUES:
|
||||||
|
fm["base"] = "—"
|
||||||
|
fm["fakemon"] = infer_fakemon(fm, body)
|
||||||
|
fm["generations"] = apply_fakemon_generation_tag(
|
||||||
|
{**fm, "generations": infer_generations(fm, body)},
|
||||||
|
os.path.splitext(os.path.basename(path))[0],
|
||||||
|
)
|
||||||
|
fm["tags"] = E.derive_tags(fm)
|
||||||
|
|
||||||
|
if sections.get("Summary"):
|
||||||
|
cleaned = clean_summary(sections["Summary"])
|
||||||
|
if cleaned and cleaned != sections["Summary"]:
|
||||||
|
body = set_section(body, "Summary", cleaned)
|
||||||
|
|
||||||
|
extracted_urls: list[str] = []
|
||||||
|
if sections.get("Features"):
|
||||||
|
features, extracted_urls = normalize_features(sections["Features"])
|
||||||
|
if features and features != sections["Features"]:
|
||||||
|
body = set_section(body, "Features", features)
|
||||||
|
|
||||||
|
type_links = E.type_links(fm.get("type") or [])
|
||||||
|
if type_links:
|
||||||
|
body = set_section(body, "Type", type_links)
|
||||||
|
|
||||||
|
current_sections = note_sections(body)
|
||||||
|
if current_sections.get("Links") or extracted_urls:
|
||||||
|
links = normalize_links_section(current_sections.get("Links", ""), extracted_urls)
|
||||||
|
if links:
|
||||||
|
body = set_section(body, "Links", links)
|
||||||
|
|
||||||
|
body = update_callout(body, fm)
|
||||||
|
content = f"---\n{E.emit_fm(fm)}\n---\n{body.rstrip()}\n"
|
||||||
|
changed = before_fm != fm or before_body.rstrip() != body.rstrip()
|
||||||
|
audit = {
|
||||||
|
"stem": os.path.splitext(os.path.basename(path))[0],
|
||||||
|
"fakemon": fm.get("fakemon"),
|
||||||
|
"generations": fm.get("generations") or [],
|
||||||
|
"changed": changed,
|
||||||
|
"had_features": bool(sections.get("Features")),
|
||||||
|
"feature_urls_moved": len(extracted_urls),
|
||||||
|
}
|
||||||
|
return changed, {"content": content, "audit": audit}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--apply", action="store_true")
|
||||||
|
parser.add_argument("--limit", type=int)
|
||||||
|
parser.add_argument("--report", default=os.path.join(E.SCRAPE, "standardize-report.json"))
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
paths = [os.path.join(E.HACKS, fn) for fn in sorted(os.listdir(E.HACKS)) if fn.endswith(".md")]
|
||||||
|
if args.limit:
|
||||||
|
paths = paths[: args.limit]
|
||||||
|
|
||||||
|
changed = 0
|
||||||
|
audits = []
|
||||||
|
for path in paths:
|
||||||
|
did_change, result = standardize_note(path)
|
||||||
|
audits.append(result["audit"])
|
||||||
|
if did_change:
|
||||||
|
changed += 1
|
||||||
|
if args.apply:
|
||||||
|
with open(path, "w", encoding="utf-8", newline="\n") as f:
|
||||||
|
f.write(result["content"])
|
||||||
|
else:
|
||||||
|
os.makedirs(E.PREVIEW, exist_ok=True)
|
||||||
|
with open(os.path.join(E.PREVIEW, os.path.basename(path)), "w", encoding="utf-8", newline="\n") as f:
|
||||||
|
f.write(result["content"])
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
"notes": len(paths),
|
||||||
|
"changed": changed,
|
||||||
|
"fakemon": dict(Counter(a["fakemon"] for a in audits)),
|
||||||
|
"with_generations": sum(1 for a in audits if a["generations"]),
|
||||||
|
"without_generations": sum(1 for a in audits if not a["generations"]),
|
||||||
|
"feature_urls_moved": sum(a["feature_urls_moved"] for a in audits),
|
||||||
|
"audits": audits,
|
||||||
|
}
|
||||||
|
os.makedirs(os.path.dirname(args.report), exist_ok=True)
|
||||||
|
with open(args.report, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(summary, f, indent=2, ensure_ascii=False)
|
||||||
|
print(f"standardized {len(paths)} notes; changed={changed}; apply={args.apply}")
|
||||||
|
print(f"fakemon={summary['fakemon']}")
|
||||||
|
print(f"with_generations={summary['with_generations']} without_generations={summary['without_generations']}")
|
||||||
|
print(f"feature_urls_moved={summary['feature_urls_moved']}")
|
||||||
|
print(f"report={args.report}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
+7982
-2635
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user