Add a native-script Wikidata corpus: real names in their own scripts

AGENTS.md

@@ -261,7 +261,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_
 
 **Adding a flag-gated post-parse transform** (reorder/adjust) — add a `Constants` boolean (default `False`), implement a `handle_*()` method, and call it in `post_process()` after `handle_firstnames()` and before `handle_capitalization()`, gated on the flag. Default-off keeps existing parses byte-for-byte unchanged. Two shipped examples: `patronymic_name_order` gates both `handle_east_slavic_patronymic_name_order()` (#85) and `handle_turkic_patronymic_name_order()` (#185) — one flag driving two independent handlers, added in the same `post_process()` slot; `middle_name_as_last` gates `handle_middle_name_as_last()` (#133), which folds `middle_list` into `last_list`.
 
-**Validating a new parsing rule** — before implementing, simulate it in a throwaway script against `TEST_NAMES` (plus a few target-language examples) to catch regressions/false-positives early. E.g. this surfaced the `last_base` `do`/`st`/`mc` empties and the patronymic `"David Michael Abramovich"` false-reorder. After the change, `tests/v2/test_snapshot.py` lists every corpus name it moved, grouped by changed field -- read them, and re-record with `uv run python tools/differential/snapshot.py --record` once every move is intended, so the PR diff of `snapshot.jsonl` shows the reviewer the same list. `uv run python tools/differential/cldr_agreement.py` reports how the tree's agreement with CLDR's stated name fields moved; `tests/v2/test_cldr_agreement.py` fails on any move, and `tools/differential/cldr_agreement.json` doubles as a measured list of names the parser and CLDR read differently. The same command and test score the GND corpus's real names (`gnd_fields.jsonl`, record `gnd_agreement.json`) by the same rules, and `--record` rewrites every record, printing per source what it accepts. A locale-pack change reads the second view the same command prints, per pack (`cldr_agreement_locales.json`, #672); its `JA` rows run only with the `ja` extra installed.
+**Validating a new parsing rule** — before implementing, simulate it in a throwaway script against `TEST_NAMES` (plus a few target-language examples) to catch regressions/false-positives early. E.g. this surfaced the `last_base` `do`/`st`/`mc` empties and the patronymic `"David Michael Abramovich"` false-reorder. After the change, `tests/v2/test_snapshot.py` lists every corpus name it moved, grouped by changed field -- read them, and re-record with `uv run python tools/differential/snapshot.py --record` once every move is intended, so the PR diff of `snapshot.jsonl` shows the reviewer the same list. `uv run python tools/differential/cldr_agreement.py` reports how the tree's agreement with CLDR's stated name fields moved; `tests/v2/test_cldr_agreement.py` fails on any move, and `tools/differential/cldr_agreement.json` doubles as a measured list of names the parser and CLDR read differently. The same command and test score the GND corpus's real names (`gnd_fields.jsonl`, record `gnd_agreement.json`) by the same rules, and `--record` rewrites every record, printing per source what it accepts. A locale-pack change reads the second view the same command prints, per pack (`cldr_agreement_locales.json`, #672, and `wikidata_native_agreement_locales.json` for real Han and kana names); its `JA` rows run only with the `ja` extra installed. A change to Han, Hangul or kana order or to Korean segmentation reads `wikidata_native_agreement.json` too: the only real names in those scripts the corpora hold. It holds no honorific, so a change to the honorific peel moves nothing there.
 
 ## Gotchas
 

tests/v2/test_cldr_agreement.py

@@ -12,12 +12,16 @@
 tools/ is not a package, so the module is loaded by path, as
 _differential_fixtures.py loads compare.py.
 """
+import contextlib
+import functools
 import importlib.util
+import io
 import json
 import sys
 from collections import Counter
 from pathlib import Path
 from types import ModuleType
+from typing import Any
 
 import pytest
 
@@ -101,10 +105,18 @@ def test_each_pending_field_is_scored_inside_the_role_it_names(field: str) -> No
                 and "Vila" in roles[r].split()]
 
 
-def test_the_summary_says_where_each_pending_field_is_scored(
-        capsys: pytest.CaptureFixture[str]) -> None:
-    agreement.main([])
-    out = capsys.readouterr().out
+@functools.cache
+def _summary_run() -> tuple[int, str]:
+    """One `main([])` run and what it printed, shared by the tests that
+    read the summary: each run scores every source (about 0.7 s)."""
+    out = io.StringIO()
+    with contextlib.redirect_stdout(out):
+        code = agreement.main([])
+    return code, out.getvalue()
+
+
+def test_the_summary_says_where_each_pending_field_is_scored() -> None:
+    _, out = _summary_run()
     assert "patronymic" in out and "(scored inside given_names)" in out
     assert "(scored inside family)" in out
 
@@ -240,7 +252,8 @@ def test_changes_reports_a_rendering_that_is_gone() -> None:
 #: an emptied one re-recorded would match its emptied record.
 _CORPORA = {"CLDR": ("corpus_cldr.jsonl", 1000), "GND": ("corpus_gnd.jsonl", 500),
             "Wikidata": ("corpus_wikidata.jsonl", 500),
-            "Conventions": ("corpus_conventions.jsonl", 25)}
+            "Conventions": ("corpus_conventions.jsonl", 25),
+            "Wikidata native": ("corpus_wikidata_native.jsonl", 700)}
 #: Sources whose fields file states only some of its corpus's names: the
 #: conventions table leaves a row without fields where no field can state
 #: it, and says why in the row's `unstated`.
@@ -317,16 +330,23 @@ def test_nameparser_still_agrees_with_each_source_where_it_did(source: str) -> N
         "and say why in the commit."])
 
 
-def test_the_summary_counts_each_role_and_matches_the_record(
-        capsys: pytest.CaptureFixture[str]) -> None:
-    assert agreement.main([]) == 0
-    out = capsys.readouterr().out
+def test_the_summary_counts_each_role_and_matches_the_record() -> None:
+    code, out = _summary_run()
+    assert code == 0, out
     for role in agreement.ROLES:
         assert any(line.split()[0] == role and "agree" in line
                    for line in out.splitlines() if line.strip()), role
     for source in agreement.sources():
         assert f"{source} view matches the record" in out
-    assert "locale view matches the record" in out
+    for source in agreement.locale_sources():
+        assert f"{source} locale view matches the record" in out
+    # each locale view prints its own default count, not another source's
+    native = out.split("Wikidata native locale view", 1)[1]
+    rows = agreement.load_rows(agreement.NATIVE_FIELDS)
+    wrong = agreement.disagreements(rows)
+    zh = agreement.locale_rows(rows, agreement.locale_sources()["Wikidata native"], "ZH")
+    before = sum(agreement.rendering_key(r) not in wrong for r in zh)
+    assert f"ZH     zh-cn, zh-tw: {before}/{len(zh)} default" in native
 
 
 @pytest.mark.parametrize("text", [
@@ -361,20 +381,35 @@ def test_every_locale_pack_is_measured_or_listed_as_having_no_cldr_file() -> Non
     assert mapped | agreement.NO_CLDR == {code.upper() for code in locales.available()}
 
 
-def test_every_mapped_cldr_file_has_renderings() -> None:
-    # A misspelled file in the table would measure nothing and pass.
-    files = {row["file"] for row in agreement.load_rows()}
-    assert set(agreement.LOCALE_FILES) <= files
+def test_every_mapped_cldr_file_and_native_stratum_has_renderings() -> None:
+    # A misspelled file or stratum in the tables would measure nothing and pass.
+    assert set(agreement.LOCALE_FILES) <= {row["file"] for row in agreement.load_rows()}
+    native = agreement.load_rows(agreement.NATIVE_FIELDS)
+    assert set(agreement.NATIVE_LABELS) <= {row["label"] for row in native}
+
+
+_CLDR, _NATIVE = (agreement.locale_sources()[name] for name in ("CLDR", "Wikidata native"))
+_ZH_ROW = {"file": "Q1", "label": "zh-cn", "order": "surnameFirst",
+           "text": "王小明", "shape": None, "fields": {"surname": "王", "given": "小明"}}
 
 
 def test_the_locale_view_parses_with_the_pack() -> None:
     # A view that fell back to the default parser would record the
     # default's disagreements and still pass its own ratchet. 王小明 is
     # one unspaced Han token: only ZH segments it.
-    row = {"file": "zh", "label": "nativeG", "order": "surnameFirst",
-           "text": "王小明", "shape": 4, "fields": {"surname": "王", "given": "小明"}}
-    assert agreement.disagreements([row], locale=True) == {}
-    assert agreement.disagreements([row]) != {}
+    assert agreement.disagreements([_ZH_ROW], "ZH") == {}
+    assert agreement.disagreements([_ZH_ROW]) != {}
+
+
+def test_each_source_finds_a_renderings_pack_in_its_own_key() -> None:
+    # CLDR by its file, the native-script source by its stratum: a QID
+    # names no language, and a CLDR label names no stratum.
+    assert _CLDR.pack_of("zh/nativeG/givenFirst") == "ZH"
+    assert _NATIVE.pack_of("Q1/zh-cn/surnameFirst") == "ZH"
+    assert _NATIVE.pack_of("Q1/ja-jp/surnameFirst") == "JA"
+    assert _NATIVE.pack_of("Q1/ko-kr/surnameFirst") is None
+    assert _CLDR.pack_of("Q1/zh-cn/surnameFirst") is None
+    assert agreement.locale_rows([_ZH_ROW], _NATIVE, "ZH") == [_ZH_ROW]
 
 
 def test_a_pack_parser_keeps_the_family_first_order_of_its_shape() -> None:
@@ -384,45 +419,112 @@ def test_a_pack_parser_keeps_the_family_first_order_of_its_shape() -> None:
     assert (name.family, name.given) == ("Ivanov", "Ivan")
 
 
-def test_a_recorded_entry_whose_file_left_the_table_is_reported_gone() -> None:
+def test_a_recorded_entry_no_pack_reads_is_reported_gone() -> None:
     # No pack's check reads such an entry, so without this it would sit
     # in the record unguarded -- and survive every re-record.
-    [message] = agreement.unmapped({"zh/nativeG/givenFirst": {}, "xx/nativeG/sorting": {}})
+    [message] = agreement.unmapped({"zh/nativeG/givenFirst": {}, "xx/nativeG/sorting": {}},
+                                   _CLDR)
     assert message.startswith("gone: xx/nativeG/sorting")
-    assert agreement.unmapped(agreement.read_record(agreement.LOCALE_RECORD)) == []
+    [message] = agreement.unmapped({"Q1/zh-cn/surnameFirst": {},
+                                    "Q2/xx-yy/surnameFirst": {}}, _NATIVE)
+    assert message.startswith("gone: Q2/xx-yy/surnameFirst")
+    for name, source in agreement.locale_sources().items():
+        assert agreement.unmapped(agreement.read_record(source.record), source) == [], name
+
+
+def test_a_recorded_rendering_that_is_gone_is_reported_by_its_packs_check() -> None:
+    recorded: dict[str, dict[str, Any]] = {"Q2/zh-cn/surnameFirst": {"text": "王"}}
+    [message] = agreement.locale_changes([_ZH_ROW], _NATIVE, "ZH", recorded)
+    assert message.startswith("gone: Q2/zh-cn/surnameFirst")
+
+
+def _redirect(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
+    """Point every record, each source's and each locale one, into
+    tmp_path through sources() and locale_sources() themselves, so a
+    source added later cannot be written into the tree by a test."""
+    redirected = {name: (fields, tmp_path / f"{name}.json")
+                  for name, (fields, _) in agreement.sources().items()}
+    redirected_locale = {name: source._replace(record=tmp_path / f"{name}.locales.json")
+                         for name, source in agreement.locale_sources().items()}
+    monkeypatch.setattr(agreement, "sources", lambda: redirected)
+    monkeypatch.setattr(agreement, "locale_sources", lambda: redirected_locale)
+    return redirected_locale
 
 
-def test_a_re_record_drops_entries_whose_file_left_the_table(
+def test_a_re_record_drops_entries_no_pack_reads(
         tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     # The carry-over that keeps an unmeasured pack's entries must not
     # keep an unmapped file's too.
-    # Every record is redirected through sources() itself, so a source
-    # added later cannot be written into the tree by this test.
-    record = tmp_path / "locales.json"
+    real, real_locale = agreement.sources(), agreement.locale_sources()
+    record = _redirect(tmp_path, monkeypatch)["CLDR"].record
     agreement.write_record({"xx/nativeG/sorting": {"text": "X"}}, record)
-    real = agreement.sources()
-    redirected = {name: (fields, tmp_path / f"{name}.json")
-                  for name, (fields, _) in real.items()}
-    monkeypatch.setattr(agreement, "LOCALE_RECORD", record)
-    monkeypatch.setattr(agreement, "sources", lambda: redirected)
     assert agreement.main(["--record"]) == 0
     assert "xx/nativeG/sorting" not in agreement.read_record(record)
-    # and the re-record wrote every source's record, not CLDR's alone
-    for name, (fields, written) in redirected.items():
+    # and the re-record wrote every source's record, not CLDR's alone, and
+    # every locale record: each its committed record's measured packs (an
+    # unmeasured pack's entries are carried over only from the old record,
+    # which here holds none of them)
+    for name, (_, written) in agreement.sources().items():
         assert agreement.read_record(written) == agreement.read_record(real[name][1]), name
+    for name, source in agreement.locale_sources().items():
+        committed = real_locale[name]
+        measured = {k: v for k, v in agreement.read_record(committed.record).items()
+                    if agreement.pack_available(committed.pack_of(k) or "")}
+        assert agreement.read_record(source.record) == measured, name
+
+
+def test_every_locale_record_is_read() -> None:
+    # Dropping a source from locale_sources() would leave its committed
+    # record read by nothing, and its rows' ratchet would simply vanish
+    # from the parametrization.
+    records = {source.record.name for source in agreement.locale_sources().values()}
+    assert records == {path.name for path in _TOOLS.glob("*_agreement_locales.json")}
+    assert set(agreement.locale_sources()) <= set(agreement.sources())
+
+
+@pytest.mark.parametrize("source", sorted(agreement.locale_sources()))
+def test_a_re_record_keeps_an_unmeasured_packs_entries(
+        source: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
+        capsys: pytest.CaptureFixture[str]) -> None:
+    # Without the ja extra, --record must carry the JA entries over, or a
+    # run without it would erase them and only CI's ja-extra job would see.
+    committed = agreement.locale_sources()[source]
+    redirected = committed._replace(record=tmp_path / committed.record.name)
+    redirected.record.write_text(committed.record.read_text(encoding="utf-8"),
+                                 encoding="utf-8")
+    monkeypatch.setattr(agreement, "_EXTRAS", {"JA": "a_module_that_is_not_installed"})
+    rows = agreement.load_rows(agreement.sources()[source][0])
+    agreement._record_locale(source, rows, redirected)
+    old, new = agreement.read_record(committed.record), agreement.read_record(redirected.record)
+    kept = {k: v for k, v in old.items() if committed.pack_of(k) == "JA"}
+    assert kept, f"{source} records no JA entry: the carry-over is not exercised"
+    assert {k: v for k, v in new.items() if committed.pack_of(k) == "JA"} == kept
+    assert "JA not measured" in capsys.readouterr().out
+
+
+def test_the_summary_still_checks_a_pack_with_no_renderings_left(
+        capsys: pytest.CaptureFixture[str]) -> None:
+    # Its recorded entries are gone, and must be reported as such.
+    rows = [r for r in agreement.load_rows()
+            if _CLDR.pack_of(agreement.rendering_key(r)) != "ZH"]
+    problems = agreement._locale_summary("CLDR", rows, _CLDR, {})
+    assert problems and all(p.startswith("gone:") for p in problems)
 
 
-@pytest.mark.parametrize("pack", [
-    pytest.param(pack, marks=pytest.mark.skipif(
+@pytest.mark.parametrize(("source", "pack"), [
+    pytest.param(source, pack, marks=pytest.mark.skipif(
         not agreement.pack_available(pack),
         reason=f"needs {agreement._EXTRAS.get(pack)}; CI's ja-extra job runs it"))
-    for pack in sorted(set(agreement.LOCALE_FILES.values()))])
-def test_each_pack_still_agrees_with_cldr_where_it_did(pack: str) -> None:
+    for source in sorted(agreement.locale_sources())
+    for pack in agreement._packs()])
+def test_each_pack_still_agrees_with_each_source_where_it_did(source: str, pack: str) -> None:
+    locale = agreement.locale_sources()[source]
     problems = agreement.locale_changes(
-        agreement.load_rows(), pack, agreement.read_record(agreement.LOCALE_RECORD))
+        agreement.load_rows(agreement.sources()[source][0]), locale, pack,
+        agreement.read_record(locale.record))
     assert not problems, "\n".join([
         *problems[:40],
         f"{len(problems)} change(s) against "
-        f"tools/differential/cldr_agreement_locales.json for {pack}.",
+        f"tools/differential/{locale.record.name} for {pack}.",
         f"If every one is intended, re-record with `{agreement.RECORD_COMMAND}` "
         "and say why in the commit."])

tests/v2/test_corpus_build.py

@@ -17,7 +17,8 @@
 
 corpus_build = load_tool("corpus_build")
 
-_BUILDERS = ("build_cldr_corpus", "build_gnd_corpus", "build_wikidata_corpus")
+_BUILDERS = ("build_cldr_corpus", "build_gnd_corpus", "build_wikidata_corpus",
+             "build_wikidata_native_corpus")
 
 
 @pytest.mark.parametrize("stem", _BUILDERS)