diff --git a/AGENTS.md b/AGENTS.md index 256b66d9..c4f80d1f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -285,7 +285,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **A group-side reading of a family comma's given part that asks `tail_reading` disagrees with the given part's own walk, and the disagreement is accepted.** `tail_reading` reads the part as the comma-less name reads it; assign's walk over the given part after a family comma reads by its own predicates -- a shape-only numeral VI to X stays a name word there (`Smith, John van der Berg VI` reads middle 'van der Berg VI'), V before a third comma part is read strictly (#144), and P6 attaches a period-marked title-particle (`St.`) the comma-less reading takes as a title. So a bound or a decline group computes with `tail_reading` acts where the walk then reads no title: #623's chain bound (`Smith, John Mc Prof. VI`) and #637's join decline (`Smith, abdul Prof. VI` → middle 'Prof. VI', `Smith, abdul St.` → family 'St. Smith'). Both were recorded rather than narrowed (Derek: garbage input; decisions.md#H5 and #P5, rules.md#P5's Accepted clause). Before writing another group-side predictor of the given part, sweep those three shapes and decide, rather than rediscovering the gap in review. -**A comparison that runs both sides in one tree reports 0 differences, which is what success looks like.** `tools/differential/compare.py` has been hardened against this — it generates its baseline worker into a temp dir, strips `PYTHONPATH` from the child, and aborts unless both the version AND the resolved path check out on each side; its README's three invocation traps are the analysis behind that design, worth reading before changing the harness. **The exposure is ad-hoc comparisons you write yourself**, where the same collapse has a cause the harness cannot disarm for you: **the shell's working directory persists between tool calls**, so a two-tree comparison written as two `cd`s silently runs both halves in whichever tree it landed in. Pin absolute paths, and assert `nameparser.__file__` on both sides the way `compare.py` does. The general rule outlives any particular trap: before believing a null result, prove the harness can report a difference — a clean run and a broken harness are the same output. The converse trap is a mutation that moves nothing: it shows the code is unexercised by your grids, not that it is dead. #638 was filed calling group's chain re-ask dead on a grid with no title-particle head before a particle run and a dotted acronym; the next review found 400 moves (`Smith, Freiherr van Berg X.Y.Z.`) and the issue closed the same day. Before calling code dead, look for the input that reaches it. +**A comparison that runs both sides in one tree reports 0 differences, which is what success looks like.** `tools/differential/compare.py` has been hardened against this — it generates its baseline worker into a temp dir, strips `PYTHONPATH` from the child, and aborts unless both the version AND the resolved path check out on each side; its README's three invocation traps are the analysis behind that design, worth reading before changing the harness. **The exposure is ad-hoc comparisons you write yourself**, where the same collapse has a cause the harness cannot disarm for you: **the shell's working directory persists between tool calls**, so a two-tree comparison written as two `cd`s silently runs both halves in whichever tree it landed in. Pin absolute paths, and assert `nameparser.__file__` on both sides the way `compare.py` does. The general rule outlives any particular trap: before believing a null result, prove the harness can report a difference — a clean run and a broken harness are the same output. The proof has to come from the input class the null result is about: #683 moved `tools/name_usage.py` to a POST and checked it with `shehu`, which is ASCII and so cannot show that the server decodes a Cyrillic body. A misdecoded query still runs and matches nothing, so the Cyrillic zeros it was meant to vouch for proved nothing until `Иван` (17,119 given) went through the same path. The converse trap is a mutation that moves nothing: it shows the code is unexercised by your grids, not that it is dead. #638 was filed calling group's chain re-ask dead on a grid with no title-particle head before a particle run and a dotted acronym; the next review found 400 moves (`Smith, Freiherr van Berg X.Y.Z.`) and the issue closed the same day. Before calling code dead, look for the input that reaches it. **`SUFFIX_ACRONYMS` and `SUFFIX_WORDS` are asserted DISJOINT at import (since 2026-09-08) — a bulk post-nominal import that re-creates an overlap fails there, and the fix is to choose a set, not to relax the assert** — the two branches normalize differently (the word test strips only edge periods, the acronym test strips all of them), so a word in both is matched by two rules and which one fired is unreadable from outside. `esq` was the single overlap and left `SUFFIX_ACRONYMS`, which is what made the assert possible; the whole algebra that had made the overlap safe, and the criterion that decided which set a word belongs to, are in the `suffix-acronym-collisions` entry of `docs/design/decisions.md` if anyone wants them. diff --git a/tests/v2/test_name_usage_tool.py b/tests/v2/test_name_usage_tool.py index 23b68a95..7a9d5d41 100644 --- a/tests/v2/test_name_usage_tool.py +++ b/tests/v2/test_name_usage_tool.py @@ -16,6 +16,7 @@ import io import json import sys +import urllib.parse import urllib.request import zipfile from collections import Counter @@ -61,6 +62,27 @@ def test_the_network_guard_bites() -> None: tool._sparql("SELECT * WHERE {}") +def test_sparql_sends_the_query_in_the_body_not_the_url( + monkeypatch: pytest.MonkeyPatch) -> None: + # #680: a GET carried the query in the URL, and the endpoint answered + # HTTP 431 once it passed about 7 KB -- reached by a seven-letter + # Cyrillic word (Младший), each letter six bytes percent-encoded and + # repeated once per label language and spelling. + sent: list[urllib.request.Request] = [] + + def capture(request: urllib.request.Request, **kwargs: object) -> io.BytesIO: + sent.append(request) + return io.BytesIO(b'{"results": {"bindings": []}}') + monkeypatch.setattr(urllib.request, "urlopen", capture) + query = tool.wikidata_countries_query(tool.spellings("Младший")) + assert tool._sparql(query) == {"results": {"bindings": []}} + (request,) = sent + assert request.get_method() == "POST" + assert request.full_url == tool.WIKIDATA_ENDPOINT + assert isinstance(request.data, bytes) + assert urllib.parse.parse_qs(request.data.decode()) == {"query": [query]} + + _CENSUS = """name,rank,count,prop100k,cum_prop100k,pctwhite,pctblack,pctapi,pctaian,pct2prace,pcthispanic OBRIEN,258,118557,40.19,0,93.85,1.29,0.76,0.27,1.51,2.32 SHEHU,39212,561,0.19,0,94.47,3.92,(S),0,0.89,(S) diff --git a/tools/name_usage.py b/tools/name_usage.py index 760ec9a4..b711c7f3 100644 --- a/tools/name_usage.py +++ b/tools/name_usage.py @@ -420,9 +420,13 @@ def wikidata_usage(word: str, top: int) -> Usage: def _sparql(query: str) -> dict[str, Any]: - url = WIKIDATA_ENDPOINT + "?" + urllib.parse.urlencode({"query": query}) - request = urllib.request.Request(url, headers={ + # POSTed, not a GET: the VALUES block repeats every spelling once per + # label language, so a seven-letter Cyrillic word already pushes the + # URL past the endpoint's limit (HTTP 431, #680). + body = urllib.parse.urlencode({"query": query}).encode() + request = urllib.request.Request(WIKIDATA_ENDPOINT, data=body, headers={ "Accept": "application/sparql-results+json", + "Content-Type": "application/x-www-form-urlencoded; charset=utf-8", "User-Agent": USER_AGENT}) with urllib.request.urlopen(request, timeout=90) as response: result: dict[str, Any] = json.load(response)