From aea229d54aa3e1a58e002b969e0a9c375ffd5700 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 17 Oct 2026 13:20:21 -0700 Subject: [PATCH 1/2] Report the given/middle split where a source states it (#702) CLDR and the conventions table state given and given2, but the merged given_names role passes a parse that splits them wrongly (Kim Min Ho). Add a report-only view counting those renderings, listed with --split. Not ratcheted: what middle means for a compound given name is open. Co-Authored-By: Claude Sonnet 5.5 --- tests/v2/test_cldr_agreement.py | 20 +++++++++ tools/differential/cldr_agreement.py | 62 +++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/tests/v2/test_cldr_agreement.py b/tests/v2/test_cldr_agreement.py index 6f522f52..d76571e0 100644 --- a/tests/v2/test_cldr_agreement.py +++ b/tests/v2/test_cldr_agreement.py @@ -528,3 +528,23 @@ def test_each_pack_still_agrees_with_each_source_where_it_did(source: str, pack: f"tools/differential/{locale.record.name} for {pack}.", f"If every one is intended, re-record with `{agreement.RECORD_COMMAND}` " "and say why in the commit."]) + + +def test_the_split_view_reports_a_compound_given_name_the_merged_role_passes() -> None: + # #702: Kim Min Ho reads given Min, middle Ho; the merged role agrees. + row = _row("Kim Min Ho", 4, given="Min Ho", surname="Kim") + assert agreement.disagreements([row]) == {} + assert agreement.split_disagreements([row]) == {"xx/nativeG/givenFirst": { + "text": "Kim Min Ho", "given": {"expected": "Min Ho", "got": "Min"}, + "middle": {"expected": "", "got": "Ho"}}} + # a split the source states and the parser reads the same way is silent + agreed = _row("Ana Maria Ruiz", None, given="Ana", given2="Maria", surname="Ruiz") + assert agreement.split_disagreements([agreed]) == {} + + +def test_the_split_view_is_reported_for_exactly_the_sources_that_state_it() -> None: + # a count line per source, and no listing without --split + out = _summary_run()[1] + assert out.count("given/middle split (report only") == len(agreement.SPLIT_SOURCES) + assert "CLDR given/middle split" in out and "Conventions given/middle split" in out + assert " conventions/Kim Min Ho" not in out diff --git a/tools/differential/cldr_agreement.py b/tools/differential/cldr_agreement.py index 95957c3d..a0823d9a 100644 --- a/tools/differential/cldr_agreement.py +++ b/tools/differential/cldr_agreement.py @@ -63,6 +63,15 @@ `honorific` is not pending: it is scored inside `suffix` by design, as nameparser reads `さん`, and ParsedName is not meant to gain it. +One view is report-only (#702): CLDR and the conventions table state the +split of the given names (`given` and `given2`), and the merged +`given_names` role cannot see a parse that splits them wrongly +(`Kim Min Ho` reads given `Min`, middle `Ho`). The summary counts the +renderings that agree merged but split differently, and `--split` lists +them. It is not a ratcheted role: GND and Wikidata record every given name +in one field and are never scored on it, and what `middle` means for a +compound given name is a design question. See split_disagreements. + A field name no rule reads raises: a mistyped `honorfic` would otherwise be expected in no role and pass as agreement. KNOWN_FIELDS is every name expected_roles reads, IGNORED_FIELDS what a source states @@ -221,6 +230,42 @@ def pending(role: str) -> tuple[str, ...]: return roles +#: Sources that state the given/middle split: CLDR (`given2`) and the +#: hand-written conventions table, where `given` holds a compound given +#: name whole (`Min Ho`). GND and Wikidata record every given name in one +#: field, so a split scored against them would call their convention a +#: disagreement (#702). +SPLIT_SOURCES = frozenset({"CLDR", "Conventions"}) + + +def split_disagreements(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """The report-only given/middle view (#702): renderings whose merged + `given_names` agrees but whose `given` and `middle` are split + differently from what the source states -- `given` expected from + `given`, `middle` from `given2`. Not ratcheted: most entries are the + parser's convention (first word given, the rest middle) against a + multi-word `given`, and what `middle` means for a compound given name + is a design question. A rendering stating a patronymic is skipped, + since the merged role holds it and the split has nowhere to put it.""" + found: dict[str, dict[str, Any]] = {} + for row in rows: + fields, text = row["fields"], row["text"] + if "patronymic" in fields: + continue + name = parse_rendering(text, row["shape"]) + if not same(expected_roles(fields, text)["given_names"], name.given_names, text): + continue + expected = {role: " ".join(v for n in names if n in fields + and occurs(v := fields[n], text)) + for role, names in (("given", ("given",)), ("middle", ("given2",)))} + got = {"given": name.given, "middle": name.middle} + bad = {role: {"expected": expected[role], "got": got[role]} + for role in expected if not same(expected[role], got[role], text)} + if bad: + found[rendering_key(row)] = {"text": text, **bad} + return dict(sorted(found.items())) + + def same(expected: str, got: str, text: str) -> bool: """Agreement, decided by the rendering as occurs() is: the same multiset of words where it has spaces (order within a role is free), @@ -381,7 +426,8 @@ def locale_sources() -> dict[str, LocaleSource]: def _summary(name: str, rows: list[dict[str, Any]], - current: dict[str, dict[str, Any]], record: Path) -> list[str]: + current: dict[str, dict[str, Any]], record: Path, + show_split: bool = False) -> list[str]: """Print one source's per-role agreement; return its changes.""" totals: Counter[str] = Counter() for row in rows: @@ -398,6 +444,16 @@ def _summary(name: str, rows: list[dict[str, Any]], f"yet (scored inside {merged})") problems = changes(read_record(record), current, {rendering_key(r) for r in rows}) print("\n".join(problems) or f"{name} view matches the record") + if name in SPLIT_SOURCES: + split = split_disagreements(rows) + print(f"{name} given/middle split (report only, #702): {len(split)} of " + f"{len(rows)} renderings agree on given_names but split it " + f"differently; --split lists them") + if show_split: + for key, entry in split.items(): + detail = "; ".join(f"{r} expected {entry[r]['expected']!r}, got " + f"{entry[r]['got']!r}" for r in entry if r != "text") + print(f" {key}: {entry['text']!r}: {detail}") return problems @@ -406,6 +462,8 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--record", action="store_true", help="rewrite every source's record and the locale record " "from the working tree") + ap.add_argument("--split", action="store_true", + help="list the renderings of the report-only given/middle view") args = ap.parse_args(argv) loaded = {name: (load_rows(fields), record) for name, (fields, record) in sources().items()} @@ -424,7 +482,7 @@ def main(argv: list[str] | None = None) -> int: return 0 problems: list[str] = [] for name, (source_rows, record) in loaded.items(): - problems += _summary(name, source_rows, current[name], record) + problems += _summary(name, source_rows, current[name], record, args.split) print() for name, source in locale_sources().items(): problems += _locale_summary(name, loaded[name][0], source, current[name]) From 3da9a513531914988eff7eba1a084d27902eea50 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 17 Oct 2026 13:22:32 -0700 Subject: [PATCH 2/2] Close the split view's test gaps and note its two populations (#702) Test the merged-disagreement and patronymic skips, the exact source set and the --split listing; document --split and that the report mixes compound given names with given2-before-given order swaps. Co-Authored-By: Claude Sonnet 5.5 --- tests/v2/test_cldr_agreement.py | 37 ++++++++++++++++++++++------ tools/differential/cldr_agreement.py | 5 +++- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/tests/v2/test_cldr_agreement.py b/tests/v2/test_cldr_agreement.py index d76571e0..a6c1016e 100644 --- a/tests/v2/test_cldr_agreement.py +++ b/tests/v2/test_cldr_agreement.py @@ -106,12 +106,12 @@ def test_each_pending_field_is_scored_inside_the_role_it_names(field: str) -> No @functools.cache -def _summary_run() -> tuple[int, str]: - """One `main([])` run and what it printed, shared by the tests that +def _summary_run(*args: str) -> tuple[int, str]: + """One `main(args)` run and what it printed, shared by the tests that read the summary: each run scores every source (about 0.7 s).""" out = io.StringIO() with contextlib.redirect_stdout(out): - code = agreement.main([]) + code = agreement.main(list(args)) return code, out.getvalue() @@ -542,9 +542,32 @@ def test_the_split_view_reports_a_compound_given_name_the_merged_role_passes() - assert agreement.split_disagreements([agreed]) == {} +def test_the_split_view_skips_a_rendering_whose_merged_role_already_disagrees() -> None: + # reported once, by the ratcheted role, not again by the split view + row = _row("Kim Min Ho", 4, given="Zed Qux", surname="Kim") + assert agreement.disagreements([row]) + assert agreement.split_disagreements([row]) == {} + + +def test_the_split_view_skips_a_rendering_that_states_a_patronymic() -> None: + # the merged role holds the patronymic; the split has nowhere to put it + text = "Anna Ivanovna Petrova" + row = _row(text, None, given="Anna", patronymic="Ivanovna", surname="Petrova") + assert agreement.parse_rendering(text, None).middle == "Ivanovna" + assert agreement.split_disagreements([row]) == {} + + def test_the_split_view_is_reported_for_exactly_the_sources_that_state_it() -> None: - # a count line per source, and no listing without --split + # GND and Wikidata record every given name in one field: never scored on it + assert agreement.SPLIT_SOURCES == {"CLDR", "Conventions"} + assert agreement.SPLIT_SOURCES <= set(agreement.sources()) out = _summary_run()[1] - assert out.count("given/middle split (report only") == len(agreement.SPLIT_SOURCES) - assert "CLDR given/middle split" in out and "Conventions given/middle split" in out - assert " conventions/Kim Min Ho" not in out + assert {line.split(" given/middle split")[0] for line in out.splitlines() + if "given/middle split" in line} == agreement.SPLIT_SOURCES + assert "expected '" not in out # no listing without --split + + +def test_split_lists_the_view_s_renderings() -> None: + out = _summary_run("--split")[1] + assert "conventions/Kim Min Ho/FAMILY_FIRST: 'Kim Min Ho': given expected " \ + "'Min Ho', got 'Min'" in out diff --git a/tools/differential/cldr_agreement.py b/tools/differential/cldr_agreement.py index a0823d9a..33ba7db9 100644 --- a/tools/differential/cldr_agreement.py +++ b/tools/differential/cldr_agreement.py @@ -29,6 +29,7 @@ uv run python tools/differential/cldr_agreement.py # summary uv run python tools/differential/cldr_agreement.py --record # re-record + uv run python tools/differential/cldr_agreement.py --split # list the given/middle view --record rewrites every record, each source's and the locale ones below, and first prints, per source, how many of each kind of change it is accepting, so a re-record made for one source cannot slip another @@ -68,7 +69,9 @@ `given_names` role cannot see a parse that splits them wrongly (`Kim Min Ho` reads given `Min`, middle `Ho`). The summary counts the renderings that agree merged but split differently, and `--split` lists -them. It is not a ratcheted role: GND and Wikidata record every given name +them (measured 2026-10-17: 251 of CLDR's 265 hold a multi-word `given`; the +rest write `given2` before `given`, an order swap that `FAMILY_FIRST_GIVEN_LAST` +reads, so the list mixes two populations). It is not a ratcheted role: GND and Wikidata record every given name in one field and are never scored on it, and what `middle` means for a compound given name is a design question. See split_disagreements.