From 3436d80084f53f681e78b21f8599678cb5003876 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 16 Oct 2026 23:09:32 -0700 Subject: [PATCH 1/3] Carry the short-pool guard to the Wikidata corpus builder (#696) Move the pool fetch and both size checks into pooled() in build_wikidata_corpus.py; both Wikidata builders call it. Mark the parent's Iceland (270 of 600) and Malaysia (17 of 240) strata whole, from a live re-pool on 2026-10-16; the other six filled their pools. Offline rebuilds of both corpora are byte-identical. Co-Authored-By: Claude Sonnet 5.5 --- tests/v2/test_wikidata_corpus.py | 31 +++++++++-- tools/differential/README.md | 2 +- tools/differential/build_wikidata_corpus.py | 55 +++++++++++++++---- .../build_wikidata_native_corpus.py | 19 +------ 4 files changed, 74 insertions(+), 33 deletions(-) diff --git a/tests/v2/test_wikidata_corpus.py b/tests/v2/test_wikidata_corpus.py index 14481125..74aee670 100644 --- a/tests/v2/test_wikidata_corpus.py +++ b/tests/v2/test_wikidata_corpus.py @@ -305,15 +305,21 @@ def _serve(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: bytes) -> list return serve(monkeypatch, tmp_path, body) -_RESULTS = json.dumps({"results": {"bindings": [ - {"item": {"value": "http://www.wikidata.org/entity/Q7747"}}]}}).encode() +def _results(n: int) -> bytes: + return json.dumps({"results": {"bindings": [ + {"item": {"value": f"http://www.wikidata.org/entity/Q{7747 + i}"}} + for i in range(n)]}}).encode() + + +# a pool the size of _RU's (quota 1, six times that) +_RESULTS = _results(build.POOL_FACTOR) def test_a_cached_response_is_read_without_fetching( monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: fetched = _serve(monkeypatch, tmp_path, _RESULTS) - assert build.pool(_RU, refresh=False) == ["Q7747"] - assert build.pool(_RU, refresh=False) == ["Q7747"] + assert build.pool(_RU, refresh=False)[0] == "Q7747" + assert len(build.pool(_RU, refresh=False)) == build.POOL_FACTOR assert len(fetched) == 1 build.pool(_RU, refresh=True) assert len(fetched) == 2 @@ -348,6 +354,23 @@ def test_a_query_matching_nothing_stops_the_build( build.pool(_RU, refresh=False) +def test_a_pool_short_of_its_size_stops_the_build_unless_it_is_the_whole( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _serve(monkeypatch, tmp_path, _results(3)) + with pytest.raises(ValueError, match="patronymic-ru: the query matched 3 people, " + "fewer than its pool of 6"): + build.pool(_RU, refresh=False) + assert len(build.pool(_RU._replace(whole=True), refresh=False)) == 3 + + +def test_a_stratum_marked_whole_whose_pool_fills_stops_the_build( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _serve(monkeypatch, tmp_path, _results(build.POOL_FACTOR)) + with pytest.raises(ValueError, match="patronymic-ru: marked whole=True, but the " + "query filled"): + build.pool(_RU._replace(whole=True), refresh=False) + + def test_an_item_the_response_leaves_out_stops_the_build( monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: _serve(monkeypatch, tmp_path, b'{"entities": {"Q1": {"id": "Q1"}}}') diff --git a/tools/differential/README.md b/tools/differential/README.md index 56441cba..188e4fc2 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -425,7 +425,7 @@ A second view measures the locale packs (#672). `LOCALE_FILES` in `cldr_agreemen **The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, and a response that is not what was asked for each stop the build before anything is written; Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree, in a `wikidata/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. -**The native-script Wikidata source.** `build_wikidata_native_corpus.py` exists because every other real-name corpus is Latin script, apart from Wikidata's Russian patronymics, so the script-conditional readings (family-first Han, Hangul and kana; Korean surname segmentation) had been tested only against CLDR's invented names. No text holds an honorific, so the corpus does not exercise the CJK honorific peel; it shows only that the peel leaves plain names alone. It reuses `build_wikidata_corpus.py`'s fetches and token alignment and adds the following. Strata are a country and a language, not a property, and the query keeps people with a label in that language. A text is used only if every letter and combining mark is in the stratum's script (`mul` labels are Latin: `Anushka Sharma` would otherwise pass as a Hindi name). Unspaced text is aligned by character under the same exactly-one-part rule, and nothing is inferred: a Chinese item stating only its surname is left out rather than reading the rest of the text as the given name. Feminine surname forms are kept per language (Greek `Λελούδα` for `Λελούδας`, Bulgarian and Kazakh `-ова`, Ukrainian `-ська`), so the filter does not drop women. A Han, kana or Hangul text carries no shape, since the parser reads those scripts family-first by itself and that is what is being tested; pure katakana keeps the order it is written in, as the parser does. Each person gives one rendering, P1559 first, and its disagreements are in `wikidata_native_agreement.json`. On 2026-10-16 every spaced stratum agreed in every role except seven Arabic surnames of two words (`عبد العزيز`, `عز الدين`, `شكر الله`, `ذو الغنى`: the parser reads the first word as a middle name), a Persian double surname, and a Bulgarian name whose two stated family names include what is probably the father's name (`Динко Цветков Дерменджиев`); Hangul agreed on 120 of 120; Han read by default disagrees on every unspaced Chinese name and 49 of 60 Japanese ones, while the locale view reads 74/80 under `ZH` (six surnames it does not segment: `詹`, `卫`, `柯`, `茅`, `华`, `文`) and 59/60 under `JA`. Wikidata's P734 is a convention, not a fact, where the culture has none: an Arabic nisba (`البغدادي`) is entered as a family name, and the corpus scores it as one. `--lc-check` compares each name with the native-script variants of its Library of Congress record (P244), splitting by LC's comma or, in CJK, a space, and counts an LCCN LC no longer has; it runs after the build has written its files, and adds nothing to them and filters nothing from them. A pool that comes back shorter than its size stops the build unless its stratum is marked as the whole population (Thailand, Kazakhstan, Bangladesh), so a query that starts matching less cannot drift the sample unseen; a stratum so marked whose pool comes back full stops it too, since the mark is then stale. On 2026-10-16, 99 of the 910 names had an LC variant that splits them, and 96 agreed; the other three are Arabic names whose LC record names a person written the other way round (`Ibrāhīm, Ṣāliḥ` for Wikidata's `إبراهيم صالح`), so the P244 link or the order is suspect, not the split. LC writes most Han variants unspaced, so it confirms almost no Chinese or Japanese split. The 907 corpus lines (904 names new to the snapshot; three Ukrainian names were already in `corpus_wikidata.jsonl`) cost about 2.9 s in the corpus-reading test modules and about 0.6 s in the agreement tests, measured locally without coverage on 2026-10-16. Raw responses are cached under `$NAMEPARSER_TOOLS_CACHE`, in `wikidata/` and `lc/`. +**The native-script Wikidata source.** `build_wikidata_native_corpus.py` exists because every other real-name corpus is Latin script, apart from Wikidata's Russian patronymics, so the script-conditional readings (family-first Han, Hangul and kana; Korean surname segmentation) had been tested only against CLDR's invented names. No text holds an honorific, so the corpus does not exercise the CJK honorific peel; it shows only that the peel leaves plain names alone. It reuses `build_wikidata_corpus.py`'s fetches and token alignment and adds the following. Strata are a country and a language, not a property, and the query keeps people with a label in that language. A text is used only if every letter and combining mark is in the stratum's script (`mul` labels are Latin: `Anushka Sharma` would otherwise pass as a Hindi name). Unspaced text is aligned by character under the same exactly-one-part rule, and nothing is inferred: a Chinese item stating only its surname is left out rather than reading the rest of the text as the given name. Feminine surname forms are kept per language (Greek `Λελούδα` for `Λελούδας`, Bulgarian and Kazakh `-ова`, Ukrainian `-ська`), so the filter does not drop women. A Han, kana or Hangul text carries no shape, since the parser reads those scripts family-first by itself and that is what is being tested; pure katakana keeps the order it is written in, as the parser does. Each person gives one rendering, P1559 first, and its disagreements are in `wikidata_native_agreement.json`. On 2026-10-16 every spaced stratum agreed in every role except seven Arabic surnames of two words (`عبد العزيز`, `عز الدين`, `شكر الله`, `ذو الغنى`: the parser reads the first word as a middle name), a Persian double surname, and a Bulgarian name whose two stated family names include what is probably the father's name (`Динко Цветков Дерменджиев`); Hangul agreed on 120 of 120; Han read by default disagrees on every unspaced Chinese name and 49 of 60 Japanese ones, while the locale view reads 74/80 under `ZH` (six surnames it does not segment: `詹`, `卫`, `柯`, `茅`, `华`, `文`) and 59/60 under `JA`. Wikidata's P734 is a convention, not a fact, where the culture has none: an Arabic nisba (`البغدادي`) is entered as a family name, and the corpus scores it as one. `--lc-check` compares each name with the native-script variants of its Library of Congress record (P244), splitting by LC's comma or, in CJK, a space, and counts an LCCN LC no longer has; it runs after the build has written its files, and adds nothing to them and filters nothing from them. A pool that comes back shorter than its size stops the build unless its stratum is marked as the whole population (Thailand, Kazakhstan, Bangladesh), so a query that starts matching less cannot drift the sample unseen; a stratum so marked whose pool comes back full stops it too, since the mark is then stale. The check is `pooled()` in `build_wikidata_corpus.py`, which both Wikidata builders call; the parent's Iceland (270 of a possible 600) and Malaysia (17 of 240) are marked whole, the other six filled their pools, re-pooled 2026-10-16 (#696). On 2026-10-16, 99 of the 910 names had an LC variant that splits them, and 96 agreed; the other three are Arabic names whose LC record names a person written the other way round (`Ibrāhīm, Ṣāliḥ` for Wikidata's `إبراهيم صالح`), so the P244 link or the order is suspect, not the split. LC writes most Han variants unspaced, so it confirms almost no Chinese or Japanese split. The 907 corpus lines (904 names new to the snapshot; three Ukrainian names were already in `corpus_wikidata.jsonl`) cost about 2.9 s in the corpus-reading test modules and about 0.6 s in the agreement tests, measured locally without coverage on 2026-10-16. Raw responses are cached under `$NAMEPARSER_TOOLS_CACHE`, in `wikidata/` and `lc/`. **The conventions source.** `conventions_fields.jsonl` states the parts of the hand-written conventions corpus, from the `fields` of `conventions.py`'s rows, and its disagreements are in `conventions_agreement.json`. The names are invented, so the record is a worklist of conventions rather than a measure of real data: a patronymic read as a family name (`Hassan bin Yusof`), a title read as a given name (`U Kyaw Win`). It uses the Wikidata source's `patronymic`, written with its connective (`bin Yusof`), and adds `honorific`, a word of address written after the name (`Bey`, `ssi`, `san`), scored inside `suffix` as nameparser reads `さん` (Derek, 2026-10-16). A row no field can state honestly is left out of the fields file and stays in the corpus, which the snapshot still watches: the kunya `Abu Karim`, the `-né` married form `Szabóné Kiss Éva`, and `arap`/`ole`/`wa`, whose last word is a parent's name or a family name depending on the family. Such a row says why in its `unstated`, and the builder refuses a row with neither `fields` nor `unstated`, so this fields file states a subset of its corpus without a row's fields being lost unseen. Being hand-written, it is also held to two checks the generated sources pass anyway: its fields partition each text word for word (`test_the_fields_partition_their_rendering`), and the scorer raises on a field name no rule reads, for every source. diff --git a/tools/differential/build_wikidata_corpus.py b/tools/differential/build_wikidata_corpus.py index daafccd5..7aa9e1ad 100644 --- a/tools/differential/build_wikidata_corpus.py +++ b/tools/differential/build_wikidata_corpus.py @@ -36,7 +36,9 @@ with wbgetentities in batches of 50, and a stratum takes the first items in that order that survive the alignment below, up to its quota. A stratum that falls below its floor stops the build before anything is -written, as does a query that matches nothing or a response that is not +written, as does a query that matches nothing, a pool short of its size +(unless the stratum is marked `whole`, its population smaller than the +pool, which a full pool then contradicts) or a response that is not what was asked for: a Wikidata change must fail the build, not shrink the corpus. A stratum's count says what was sampled, not how common the shape is. @@ -166,6 +168,14 @@ class Stratum(NamedTuple): #: The fewest names the build accepts; below it, it stops. Set below #: the quota only where the pool is known to run short. least: int | None = None + #: Whether the query's whole population is smaller than `size`; a + #: pool short of `size` stops the build unless this is set. + whole: bool = False + + @property + def size(self) -> int: + """How many people the query takes.""" + return self.quota * POOL_FACTOR @property def floor(self) -> int: @@ -179,10 +189,12 @@ def floor(self) -> int: Stratum("patronymic-ru", "Q159", "P5056", "ru", 110), Stratum("patronymic-su", "Q15180", "P5056", "ru", 60), Stratum("patronymic-ri", "Q34266", "P5056", "ru", 40), - Stratum("patronymic-is", "Q189", "P5056", "is", 100), + # Icelandic public figures with a P5056 statement: 270 of a possible + # 600 on 2026-10-16, so the pool is the whole population. + Stratum("patronymic-is", "Q189", "P5056", "is", 100, whole=True), # 17 Malaysian public figures with a P5056 statement on 2026-10-16, # 9 aligning: the pool is the limit, and the floor leaves 3 to lose. - Stratum("patronymic-my", "Q833", "P5056", "ms", 40, least=6), + Stratum("patronymic-my", "Q833", "P5056", "ms", 40, least=6, whole=True), ) @@ -216,21 +228,40 @@ def query(stratum: Stratum) -> str: ?item wdt:{stratum.prop} ?part; wdt:P31 wd:Q5; wdt:P27 wd:{stratum.country}; wikibase:sitelinks ?links. FILTER(?links >= {MIN_SITELINKS} || EXISTS {{ ?item wdt:P570 ?died }}) -}} GROUP BY ?item ORDER BY MD5(STR(?item)) LIMIT {stratum.quota * POOL_FACTOR}""" - - -def pool(stratum: Stratum, refresh: bool) -> list[str]: - """The stratum's QIDs in MD5 order; raises on a query matching - nothing, since that means the query or Wikidata changed.""" - data = urllib.parse.urlencode({"query": query(stratum), "format": "json"}).encode() +}} GROUP BY ?item ORDER BY MD5(STR(?item)) LIMIT {stratum.size}""" + + +def pooled(label: str, text: str, size: int, whole: bool, + refresh: bool) -> list[str]: + """The QIDs a WDQS query returns, in its order, and ValueError unless + their number is the one the stratum expects: some, and `size` of + them -- the query's LIMIT -- unless `whole` says the population is + smaller than that (then fewer, and a full pool means the mark is + stale). A query that starts matching fewer people (a property + deprecated, a country item merged) would otherwise rebuild the + corpus from a smaller, different sample; both Wikidata builders pool + through here.""" + data = urllib.parse.urlencode({"query": text, "format": "json"}).encode() result = _fetch(SPARQL, data, refresh, "results") qids = [b["item"]["value"].rsplit("/", 1)[1] for b in result["results"]["bindings"]] if not qids: - raise ValueError(f"{stratum.label}: the query matches nothing; has a " - f"property or item changed?\n{query(stratum)}") + raise ValueError(f"{label}: the query matches nothing; has a " + f"property or item changed?\n{text}") + if len(qids) < size and not whole: + raise ValueError(f"{label}: the query matched {len(qids)} people, " + f"fewer than its pool of {size}; if that is now " + "the whole population, mark the stratum whole=True") + if len(qids) >= size and whole: + raise ValueError(f"{label}: marked whole=True, but the query filled " + f"its pool of {size}; drop whole=True") return qids +def pool(stratum: Stratum, refresh: bool) -> list[str]: + """The stratum's QIDs in MD5 order.""" + return pooled(stratum.label, query(stratum), stratum.size, stratum.whole, refresh) + + def entities(ids: Iterable[str], props: str, refresh: bool, languages: Iterable[str] = ()) -> dict[str, Any]: """Each requested item, keyed by the id asked for (a redirected item diff --git a/tools/differential/build_wikidata_native_corpus.py b/tools/differential/build_wikidata_native_corpus.py index 36813252..9001e8e7 100644 --- a/tools/differential/build_wikidata_native_corpus.py +++ b/tools/differential/build_wikidata_native_corpus.py @@ -252,22 +252,9 @@ def query(stratum: Stratum) -> str: def pool(stratum: Stratum, refresh: bool) -> list[str]: - """The stratum's QIDs in MD5 order; raises on a query matching - nothing, since that means the query or Wikidata changed.""" - data = urllib.parse.urlencode({"query": query(stratum), "format": "json"}).encode() - result = wikidata._fetch(wikidata.SPARQL, data, refresh, "results") - qids = [b["item"]["value"].rsplit("/", 1)[1] for b in result["results"]["bindings"]] - if not qids: - raise ValueError(f"{stratum.label}: the query matches nothing; has a " - f"property or item changed?\n{query(stratum)}") - if len(qids) < stratum.pool and not stratum.whole: - raise ValueError(f"{stratum.label}: the query matched {len(qids)} people, " - f"fewer than its pool of {stratum.pool}; if that is now " - "the whole population, mark the stratum whole=True") - if len(qids) == stratum.pool and stratum.whole: - raise ValueError(f"{stratum.label}: marked whole=True, but the query filled " - f"its pool of {stratum.pool}; drop whole=True") - return qids + """The stratum's QIDs in MD5 order.""" + return wikidata.pooled(stratum.label, query(stratum), stratum.pool, + stratum.whole, refresh) def entities(ids: list[str], language: str, refresh: bool) -> dict[str, Any]: From 951ea3d78405f9e95a94581433cec8a4736d47cb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 16 Oct 2026 23:11:33 -0700 Subject: [PATCH 2/3] Address review of the short-pool guard: pin the whole marks and the LIMIT, fix stale prose (#696) Co-Authored-By: Claude Sonnet 5.5 --- tests/v2/test_wikidata_corpus.py | 11 +++++++++++ tools/differential/README.md | 4 ++-- tools/differential/build_wikidata_corpus.py | 6 +++--- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/v2/test_wikidata_corpus.py b/tests/v2/test_wikidata_corpus.py index 74aee670..7f6c39c7 100644 --- a/tests/v2/test_wikidata_corpus.py +++ b/tests/v2/test_wikidata_corpus.py @@ -298,6 +298,17 @@ def test_build_stops_before_writing_when_a_stratum_falls_short( build.build(refresh=False) +def test_a_query_takes_as_many_people_as_the_pool_check_expects() -> None: + assert f"LIMIT {build.POOL_FACTOR * _RU.quota}" in build.query(_RU) + assert _RU.size == build.POOL_FACTOR * _RU.quota + + +def test_the_strata_marked_whole_are_the_ones_measured_short() -> None: + # Pooled 2026-10-16: Iceland 270 of 600, Malaysia 17 of 240; the others + # filled. A live rebuild is the only other check of these marks. + assert {s.label for s in build.STRATA if s.whole} == {"patronymic-is", "patronymic-my"} + + # -- the network and the cache ------------------------------------------------ def _serve(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: bytes) -> list[str]: diff --git a/tools/differential/README.md b/tools/differential/README.md index 188e4fc2..c00e5507 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -423,9 +423,9 @@ A second view measures the locale packs (#672). `LOCALE_FILES` in `cldr_agreemen **Adding a source.** Map only what the source states. The comparison reads a missing field as a statement that the role is empty, so a field the source leaves unstated must be mapped so that its role is not compared at all: #688's first build wrote a prefixless GND surname as `surname-core`, and 19 renderings recorded the parser's `de` in `Vélez de Guevara` as a disagreement no fix could clear; its fix then missed the same gap inside a several-word core after a prefix (`Zuylen van Nijevelt, Julius van`), so test each way a field can be left unstated, not the one that was reported. A field must also occur in its rendering's text, word by word (`test_every_committed_field_occurs_in_its_rendering`), or it is expected empty too. A build must fail rather than shrink: give every stratum a floor, check a response's shape before caching it (an API error arrives as HTTP 200), and raise on a query that matches nothing. But tell a broken fetch from a fact about the data: #689's first fix raised on any item Wikidata reported missing, so one deleted name-part item anywhere in a pool would have stopped every rebuild, with `--refresh` advice that could not help; a missing part now only refuses the person stating it. A source sampled by script must check each text's script itself: Wikidata's `mul` labels are Latin, and the native-script build's first probe kept `Anushka Sharma` as a Hindi name. And give each refusal a test row that fails for the reason it names, not for another: that build's review found an all-Hebrew text standing for "a Hebrew point is not Arabic", which passed with the mark check deleted. A builder change has one oracle the suite cannot run, since it needs the local cache: rebuild offline from `$NAMEPARSER_TOOLS_CACHE` and diff the result against the committed fields file. The same review's mutation pass scored every surviving mutation that way, and four of them would have moved committed rows or the figures quoted from them with every test passing. The cache (`cache_dir`, `cached_json`, which checks a response before caching it and writes atomically) and the two output files (`corpus_lines`, `fields_lines`, `write_corpus`) are shared in `corpus_build.py`; a new builder imports them rather than copying another builder's. -**The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, and a response that is not what was asked for each stop the build before anything is written; Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree, in a `wikidata/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. +**The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, a pool short of its size (unless the stratum is marked `whole`, which a full pool then contradicts) and a response that is not what was asked for each stop the build before anything is written; Iceland and Malaysia are marked `whole` (see the native paragraph below for the measurement); Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree, in a `wikidata/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. -**The native-script Wikidata source.** `build_wikidata_native_corpus.py` exists because every other real-name corpus is Latin script, apart from Wikidata's Russian patronymics, so the script-conditional readings (family-first Han, Hangul and kana; Korean surname segmentation) had been tested only against CLDR's invented names. No text holds an honorific, so the corpus does not exercise the CJK honorific peel; it shows only that the peel leaves plain names alone. It reuses `build_wikidata_corpus.py`'s fetches and token alignment and adds the following. Strata are a country and a language, not a property, and the query keeps people with a label in that language. A text is used only if every letter and combining mark is in the stratum's script (`mul` labels are Latin: `Anushka Sharma` would otherwise pass as a Hindi name). Unspaced text is aligned by character under the same exactly-one-part rule, and nothing is inferred: a Chinese item stating only its surname is left out rather than reading the rest of the text as the given name. Feminine surname forms are kept per language (Greek `Λελούδα` for `Λελούδας`, Bulgarian and Kazakh `-ова`, Ukrainian `-ська`), so the filter does not drop women. A Han, kana or Hangul text carries no shape, since the parser reads those scripts family-first by itself and that is what is being tested; pure katakana keeps the order it is written in, as the parser does. Each person gives one rendering, P1559 first, and its disagreements are in `wikidata_native_agreement.json`. On 2026-10-16 every spaced stratum agreed in every role except seven Arabic surnames of two words (`عبد العزيز`, `عز الدين`, `شكر الله`, `ذو الغنى`: the parser reads the first word as a middle name), a Persian double surname, and a Bulgarian name whose two stated family names include what is probably the father's name (`Динко Цветков Дерменджиев`); Hangul agreed on 120 of 120; Han read by default disagrees on every unspaced Chinese name and 49 of 60 Japanese ones, while the locale view reads 74/80 under `ZH` (six surnames it does not segment: `詹`, `卫`, `柯`, `茅`, `华`, `文`) and 59/60 under `JA`. Wikidata's P734 is a convention, not a fact, where the culture has none: an Arabic nisba (`البغدادي`) is entered as a family name, and the corpus scores it as one. `--lc-check` compares each name with the native-script variants of its Library of Congress record (P244), splitting by LC's comma or, in CJK, a space, and counts an LCCN LC no longer has; it runs after the build has written its files, and adds nothing to them and filters nothing from them. A pool that comes back shorter than its size stops the build unless its stratum is marked as the whole population (Thailand, Kazakhstan, Bangladesh), so a query that starts matching less cannot drift the sample unseen; a stratum so marked whose pool comes back full stops it too, since the mark is then stale. The check is `pooled()` in `build_wikidata_corpus.py`, which both Wikidata builders call; the parent's Iceland (270 of a possible 600) and Malaysia (17 of 240) are marked whole, the other six filled their pools, re-pooled 2026-10-16 (#696). On 2026-10-16, 99 of the 910 names had an LC variant that splits them, and 96 agreed; the other three are Arabic names whose LC record names a person written the other way round (`Ibrāhīm, Ṣāliḥ` for Wikidata's `إبراهيم صالح`), so the P244 link or the order is suspect, not the split. LC writes most Han variants unspaced, so it confirms almost no Chinese or Japanese split. The 907 corpus lines (904 names new to the snapshot; three Ukrainian names were already in `corpus_wikidata.jsonl`) cost about 2.9 s in the corpus-reading test modules and about 0.6 s in the agreement tests, measured locally without coverage on 2026-10-16. Raw responses are cached under `$NAMEPARSER_TOOLS_CACHE`, in `wikidata/` and `lc/`. +**The native-script Wikidata source.** `build_wikidata_native_corpus.py` exists because every other real-name corpus is Latin script, apart from Wikidata's Russian patronymics, so the script-conditional readings (family-first Han, Hangul and kana; Korean surname segmentation) had been tested only against CLDR's invented names. No text holds an honorific, so the corpus does not exercise the CJK honorific peel; it shows only that the peel leaves plain names alone. It reuses `build_wikidata_corpus.py`'s fetches and token alignment and adds the following. Strata are a country and a language, not a property, and the query keeps people with a label in that language. A text is used only if every letter and combining mark is in the stratum's script (`mul` labels are Latin: `Anushka Sharma` would otherwise pass as a Hindi name). Unspaced text is aligned by character under the same exactly-one-part rule, and nothing is inferred: a Chinese item stating only its surname is left out rather than reading the rest of the text as the given name. Feminine surname forms are kept per language (Greek `Λελούδα` for `Λελούδας`, Bulgarian and Kazakh `-ова`, Ukrainian `-ська`), so the filter does not drop women. A Han, kana or Hangul text carries no shape, since the parser reads those scripts family-first by itself and that is what is being tested; pure katakana keeps the order it is written in, as the parser does. Each person gives one rendering, P1559 first, and its disagreements are in `wikidata_native_agreement.json`. On 2026-10-16 every spaced stratum agreed in every role except seven Arabic surnames of two words (`عبد العزيز`, `عز الدين`, `شكر الله`, `ذو الغنى`: the parser reads the first word as a middle name), a Persian double surname, and a Bulgarian name whose two stated family names include what is probably the father's name (`Динко Цветков Дерменджиев`); Hangul agreed on 120 of 120; Han read by default disagrees on every unspaced Chinese name and 49 of 60 Japanese ones, while the locale view reads 74/80 under `ZH` (six surnames it does not segment: `詹`, `卫`, `柯`, `茅`, `华`, `文`) and 59/60 under `JA`. Wikidata's P734 is a convention, not a fact, where the culture has none: an Arabic nisba (`البغدادي`) is entered as a family name, and the corpus scores it as one. `--lc-check` compares each name with the native-script variants of its Library of Congress record (P244), splitting by LC's comma or, in CJK, a space, and counts an LCCN LC no longer has; it runs after the build has written its files, and adds nothing to them and filters nothing from them. A pool that comes back shorter than its size stops the build unless its stratum is marked as the whole population (Thailand, Kazakhstan, Bangladesh), so a query that starts matching less cannot drift the sample unseen; a stratum so marked whose pool comes back full stops it too, since the mark is then stale. The check is `pooled()` in `build_wikidata_corpus.py`, which both Wikidata builders call; re-pooled on 2026-10-16 (#696), the parent's Iceland returned 270 of a pool of 600 and Malaysia 17 of 240, so both are marked whole, and the other six filled their pools (es 900, mx 600, co 480, ru 660, su 360, ri 240). On 2026-10-16, 99 of the 910 names had an LC variant that splits them, and 96 agreed; the other three are Arabic names whose LC record names a person written the other way round (`Ibrāhīm, Ṣāliḥ` for Wikidata's `إبراهيم صالح`), so the P244 link or the order is suspect, not the split. LC writes most Han variants unspaced, so it confirms almost no Chinese or Japanese split. The 907 corpus lines (904 names new to the snapshot; three Ukrainian names were already in `corpus_wikidata.jsonl`) cost about 2.9 s in the corpus-reading test modules and about 0.6 s in the agreement tests, measured locally without coverage on 2026-10-16. Raw responses are cached under `$NAMEPARSER_TOOLS_CACHE`, in `wikidata/` and `lc/`. **The conventions source.** `conventions_fields.jsonl` states the parts of the hand-written conventions corpus, from the `fields` of `conventions.py`'s rows, and its disagreements are in `conventions_agreement.json`. The names are invented, so the record is a worklist of conventions rather than a measure of real data: a patronymic read as a family name (`Hassan bin Yusof`), a title read as a given name (`U Kyaw Win`). It uses the Wikidata source's `patronymic`, written with its connective (`bin Yusof`), and adds `honorific`, a word of address written after the name (`Bey`, `ssi`, `san`), scored inside `suffix` as nameparser reads `さん` (Derek, 2026-10-16). A row no field can state honestly is left out of the fields file and stays in the corpus, which the snapshot still watches: the kunya `Abu Karim`, the `-né` married form `Szabóné Kiss Éva`, and `arap`/`ole`/`wa`, whose last word is a parent's name or a family name depending on the family. Such a row says why in its `unstated`, and the builder refuses a row with neither `fields` nor `unstated`, so this fields file states a subset of its corpus without a row's fields being lost unseen. Being hand-written, it is also held to two checks the generated sources pass anyway: its fields partition each text word for word (`test_the_fields_partition_their_rendering`), and the scorer raises on a field name no rule reads, for every source. diff --git a/tools/differential/build_wikidata_corpus.py b/tools/differential/build_wikidata_corpus.py index 7aa9e1ad..448b5bfb 100644 --- a/tools/differential/build_wikidata_corpus.py +++ b/tools/differential/build_wikidata_corpus.py @@ -189,11 +189,11 @@ def floor(self) -> int: Stratum("patronymic-ru", "Q159", "P5056", "ru", 110), Stratum("patronymic-su", "Q15180", "P5056", "ru", 60), Stratum("patronymic-ri", "Q34266", "P5056", "ru", 40), - # Icelandic public figures with a P5056 statement: 270 of a possible - # 600 on 2026-10-16, so the pool is the whole population. + # Icelandic public figures with a P5056 statement: 270 of a pool of + # 600 on 2026-10-16, so the population is smaller than the pool. Stratum("patronymic-is", "Q189", "P5056", "is", 100, whole=True), # 17 Malaysian public figures with a P5056 statement on 2026-10-16, - # 9 aligning: the pool is the limit, and the floor leaves 3 to lose. + # 9 aligning: the population is the limit, and the floor leaves 3 to lose. Stratum("patronymic-my", "Q833", "P5056", "ms", 40, least=6, whole=True), ) From a507459467722ab934e82e13db314d8dab7d302c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 16 Oct 2026 23:16:03 -0700 Subject: [PATCH 3/3] Tighten the LIMIT pin, drop the whole-marks pin, fix prose from the second review (#696) Co-Authored-By: Claude Sonnet 5.5 --- tests/v2/test_wikidata_corpus.py | 10 ++-------- tools/differential/README.md | 2 +- tools/differential/build_wikidata_corpus.py | 5 +++-- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/tests/v2/test_wikidata_corpus.py b/tests/v2/test_wikidata_corpus.py index 7f6c39c7..50c75b63 100644 --- a/tests/v2/test_wikidata_corpus.py +++ b/tests/v2/test_wikidata_corpus.py @@ -299,14 +299,8 @@ def test_build_stops_before_writing_when_a_stratum_falls_short( def test_a_query_takes_as_many_people_as_the_pool_check_expects() -> None: - assert f"LIMIT {build.POOL_FACTOR * _RU.quota}" in build.query(_RU) - assert _RU.size == build.POOL_FACTOR * _RU.quota - - -def test_the_strata_marked_whole_are_the_ones_measured_short() -> None: - # Pooled 2026-10-16: Iceland 270 of 600, Malaysia 17 of 240; the others - # filled. A live rebuild is the only other check of these marks. - assert {s.label for s in build.STRATA if s.whole} == {"patronymic-is", "patronymic-my"} + # endswith: "LIMIT 6" is also a prefix of "LIMIT 60" + assert build.query(_RU).endswith(f"LIMIT {_RU.size}") # -- the network and the cache ------------------------------------------------ diff --git a/tools/differential/README.md b/tools/differential/README.md index c00e5507..e9422eaa 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -423,7 +423,7 @@ A second view measures the locale packs (#672). `LOCALE_FILES` in `cldr_agreemen **Adding a source.** Map only what the source states. The comparison reads a missing field as a statement that the role is empty, so a field the source leaves unstated must be mapped so that its role is not compared at all: #688's first build wrote a prefixless GND surname as `surname-core`, and 19 renderings recorded the parser's `de` in `Vélez de Guevara` as a disagreement no fix could clear; its fix then missed the same gap inside a several-word core after a prefix (`Zuylen van Nijevelt, Julius van`), so test each way a field can be left unstated, not the one that was reported. A field must also occur in its rendering's text, word by word (`test_every_committed_field_occurs_in_its_rendering`), or it is expected empty too. A build must fail rather than shrink: give every stratum a floor, check a response's shape before caching it (an API error arrives as HTTP 200), and raise on a query that matches nothing. But tell a broken fetch from a fact about the data: #689's first fix raised on any item Wikidata reported missing, so one deleted name-part item anywhere in a pool would have stopped every rebuild, with `--refresh` advice that could not help; a missing part now only refuses the person stating it. A source sampled by script must check each text's script itself: Wikidata's `mul` labels are Latin, and the native-script build's first probe kept `Anushka Sharma` as a Hindi name. And give each refusal a test row that fails for the reason it names, not for another: that build's review found an all-Hebrew text standing for "a Hebrew point is not Arabic", which passed with the mark check deleted. A builder change has one oracle the suite cannot run, since it needs the local cache: rebuild offline from `$NAMEPARSER_TOOLS_CACHE` and diff the result against the committed fields file. The same review's mutation pass scored every surviving mutation that way, and four of them would have moved committed rows or the figures quoted from them with every test passing. The cache (`cache_dir`, `cached_json`, which checks a response before caching it and writes atomically) and the two output files (`corpus_lines`, `fields_lines`, `write_corpus`) are shared in `corpus_build.py`; a new builder imports them rather than copying another builder's. -**The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, a pool short of its size (unless the stratum is marked `whole`, which a full pool then contradicts) and a response that is not what was asked for each stop the build before anything is written; Iceland and Malaysia are marked `whole` (see the native paragraph below for the measurement); Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree, in a `wikidata/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. +**The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, a pool short of its size (unless the stratum is marked `whole`, which a full pool then contradicts) and a response that is not what was asked for each stop the build before anything is written; Iceland (270 of a pool of 600 on 2026-10-16, measured in the native paragraph below) and Malaysia are marked `whole`; Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree, in a `wikidata/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. **The native-script Wikidata source.** `build_wikidata_native_corpus.py` exists because every other real-name corpus is Latin script, apart from Wikidata's Russian patronymics, so the script-conditional readings (family-first Han, Hangul and kana; Korean surname segmentation) had been tested only against CLDR's invented names. No text holds an honorific, so the corpus does not exercise the CJK honorific peel; it shows only that the peel leaves plain names alone. It reuses `build_wikidata_corpus.py`'s fetches and token alignment and adds the following. Strata are a country and a language, not a property, and the query keeps people with a label in that language. A text is used only if every letter and combining mark is in the stratum's script (`mul` labels are Latin: `Anushka Sharma` would otherwise pass as a Hindi name). Unspaced text is aligned by character under the same exactly-one-part rule, and nothing is inferred: a Chinese item stating only its surname is left out rather than reading the rest of the text as the given name. Feminine surname forms are kept per language (Greek `Λελούδα` for `Λελούδας`, Bulgarian and Kazakh `-ова`, Ukrainian `-ська`), so the filter does not drop women. A Han, kana or Hangul text carries no shape, since the parser reads those scripts family-first by itself and that is what is being tested; pure katakana keeps the order it is written in, as the parser does. Each person gives one rendering, P1559 first, and its disagreements are in `wikidata_native_agreement.json`. On 2026-10-16 every spaced stratum agreed in every role except seven Arabic surnames of two words (`عبد العزيز`, `عز الدين`, `شكر الله`, `ذو الغنى`: the parser reads the first word as a middle name), a Persian double surname, and a Bulgarian name whose two stated family names include what is probably the father's name (`Динко Цветков Дерменджиев`); Hangul agreed on 120 of 120; Han read by default disagrees on every unspaced Chinese name and 49 of 60 Japanese ones, while the locale view reads 74/80 under `ZH` (six surnames it does not segment: `詹`, `卫`, `柯`, `茅`, `华`, `文`) and 59/60 under `JA`. Wikidata's P734 is a convention, not a fact, where the culture has none: an Arabic nisba (`البغدادي`) is entered as a family name, and the corpus scores it as one. `--lc-check` compares each name with the native-script variants of its Library of Congress record (P244), splitting by LC's comma or, in CJK, a space, and counts an LCCN LC no longer has; it runs after the build has written its files, and adds nothing to them and filters nothing from them. A pool that comes back shorter than its size stops the build unless its stratum is marked as the whole population (Thailand, Kazakhstan, Bangladesh), so a query that starts matching less cannot drift the sample unseen; a stratum so marked whose pool comes back full stops it too, since the mark is then stale. The check is `pooled()` in `build_wikidata_corpus.py`, which both Wikidata builders call; re-pooled on 2026-10-16 (#696), the parent's Iceland returned 270 of a pool of 600 and Malaysia 17 of 240, so both are marked whole, and the other six filled their pools (es 900, mx 600, co 480, ru 660, su 360, ri 240). On 2026-10-16, 99 of the 910 names had an LC variant that splits them, and 96 agreed; the other three are Arabic names whose LC record names a person written the other way round (`Ibrāhīm, Ṣāliḥ` for Wikidata's `إبراهيم صالح`), so the P244 link or the order is suspect, not the split. LC writes most Han variants unspaced, so it confirms almost no Chinese or Japanese split. The 907 corpus lines (904 names new to the snapshot; three Ukrainian names were already in `corpus_wikidata.jsonl`) cost about 2.9 s in the corpus-reading test modules and about 0.6 s in the agreement tests, measured locally without coverage on 2026-10-16. Raw responses are cached under `$NAMEPARSER_TOOLS_CACHE`, in `wikidata/` and `lc/`. diff --git a/tools/differential/build_wikidata_corpus.py b/tools/differential/build_wikidata_corpus.py index 448b5bfb..78abae37 100644 --- a/tools/differential/build_wikidata_corpus.py +++ b/tools/differential/build_wikidata_corpus.py @@ -166,10 +166,11 @@ class Stratum(NamedTuple): language: str # the language its people's names are written in quota: int #: The fewest names the build accepts; below it, it stops. Set below - #: the quota only where the pool is known to run short. + #: the quota only where the names that align are known to run short of it. least: int | None = None #: Whether the query's whole population is smaller than `size`; a - #: pool short of `size` stops the build unless this is set. + #: pool short of `size` stops the build unless this is set, and a full + #: pool stops it when it is. whole: bool = False @property