From b9326616c6e478bd8973c59bb3dd7ec55614819f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 14 Oct 2026 00:58:58 -0700 Subject: [PATCH 1/3] POST Wikidata queries from name_usage.py instead of a GET MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The query's VALUES block repeats every spelling once per label language, so a seven-letter Cyrillic word (Младший, six bytes a letter percent-encoded) put the GET's URL past 7 KB and the endpoint answered HTTP 431. Measured: Младши's longer query is 6,909 bytes and succeeds, Младший's is 7,198 and failed. A form-encoded POST body has no such limit. Checked live: the seven words the issue names now answer, and shehu still reads 180 family as before. Closes #680 Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_name_usage_tool.py | 22 ++++++++++++++++++++++ tools/name_usage.py | 8 ++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/tests/v2/test_name_usage_tool.py b/tests/v2/test_name_usage_tool.py index 23b68a95..78633c87 100644 --- a/tests/v2/test_name_usage_tool.py +++ b/tests/v2/test_name_usage_tool.py @@ -16,6 +16,7 @@ import io import json import sys +import urllib.parse import urllib.request import zipfile from collections import Counter @@ -61,6 +62,27 @@ def test_the_network_guard_bites() -> None: tool._sparql("SELECT * WHERE {}") +def test_sparql_sends_the_query_in_the_body_not_the_url( + monkeypatch: pytest.MonkeyPatch) -> None: + # #680: a GET carried the query in the URL, and the endpoint answered + # HTTP 431 once it passed about 7 KB -- reached by a seven-letter + # Cyrillic word (Младший), each letter six bytes percent-encoded and + # repeated once per label language and spelling. + sent: list[urllib.request.Request] = [] + + def capture(request: urllib.request.Request, **kwargs: object) -> io.BytesIO: + sent.append(request) + return io.BytesIO(b'{"results": {"bindings": []}}') + monkeypatch.setattr(urllib.request, "urlopen", capture) + query = tool.wikidata_countries_query(tool.spellings("Младший")) + tool._sparql(query) + (request,) = sent + assert request.get_method() == "POST" + assert request.full_url == tool.WIKIDATA_ENDPOINT + assert isinstance(request.data, bytes) + assert urllib.parse.parse_qs(request.data.decode()) == {"query": [query]} + + _CENSUS = """name,rank,count,prop100k,cum_prop100k,pctwhite,pctblack,pctapi,pctaian,pct2prace,pcthispanic OBRIEN,258,118557,40.19,0,93.85,1.29,0.76,0.27,1.51,2.32 SHEHU,39212,561,0.19,0,94.47,3.92,(S),0,0.89,(S) diff --git a/tools/name_usage.py b/tools/name_usage.py index 760ec9a4..3811f5c1 100644 --- a/tools/name_usage.py +++ b/tools/name_usage.py @@ -420,9 +420,13 @@ def wikidata_usage(word: str, top: int) -> Usage: def _sparql(query: str) -> dict[str, Any]: - url = WIKIDATA_ENDPOINT + "?" + urllib.parse.urlencode({"query": query}) - request = urllib.request.Request(url, headers={ + # POSTed, not a GET: the VALUES block repeats every spelling once per + # label language, so a seven-letter Cyrillic word already pushes the + # URL past the endpoint's limit (HTTP 431, #680). + body = urllib.parse.urlencode({"query": query}).encode() + request = urllib.request.Request(WIKIDATA_ENDPOINT, data=body, headers={ "Accept": "application/sparql-results+json", + "Content-Type": "application/x-www-form-urlencoded", "User-Agent": USER_AGENT}) with urllib.request.urlopen(request, timeout=90) as response: result: dict[str, Any] = json.load(response) From c02d873a6390d342c86b18e5639f29c950ed985f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 14 Oct 2026 01:06:40 -0700 Subject: [PATCH 2/3] Declare the POST body's charset and check _sparql's return value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of #683: the body was correct UTF-8, but without a charset its decoding rested on the server's form default, and a misdecoded query still runs and matches nothing -- a false zero. Live control: Иван reads 17,119 given and Петров 964 family through the POST, with and without the declared charset. The test now also checks the response is read back from what urlopen returns. Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_name_usage_tool.py | 2 +- tools/name_usage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/v2/test_name_usage_tool.py b/tests/v2/test_name_usage_tool.py index 78633c87..7a9d5d41 100644 --- a/tests/v2/test_name_usage_tool.py +++ b/tests/v2/test_name_usage_tool.py @@ -75,7 +75,7 @@ def capture(request: urllib.request.Request, **kwargs: object) -> io.BytesIO: return io.BytesIO(b'{"results": {"bindings": []}}') monkeypatch.setattr(urllib.request, "urlopen", capture) query = tool.wikidata_countries_query(tool.spellings("Младший")) - tool._sparql(query) + assert tool._sparql(query) == {"results": {"bindings": []}} (request,) = sent assert request.get_method() == "POST" assert request.full_url == tool.WIKIDATA_ENDPOINT diff --git a/tools/name_usage.py b/tools/name_usage.py index 3811f5c1..b711c7f3 100644 --- a/tools/name_usage.py +++ b/tools/name_usage.py @@ -426,7 +426,7 @@ def _sparql(query: str) -> dict[str, Any]: body = urllib.parse.urlencode({"query": query}).encode() request = urllib.request.Request(WIKIDATA_ENDPOINT, data=body, headers={ "Accept": "application/sparql-results+json", - "Content-Type": "application/x-www-form-urlencoded", + "Content-Type": "application/x-www-form-urlencoded; charset=utf-8", "User-Agent": USER_AGENT}) with urllib.request.urlopen(request, timeout=90) as response: result: dict[str, Any] = json.load(response) From 54c6256a79be0b304734c52a2c118a8b2480b259 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 14 Oct 2026 01:07:04 -0700 Subject: [PATCH 3/3] Record in AGENTS.md that a positive control must share the null result's input class #683's first control for the POST was ASCII (shehu) and could not show the server decoding a Cyrillic body; review caught it. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 256b66d9..c4f80d1f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -285,7 +285,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **A group-side reading of a family comma's given part that asks `tail_reading` disagrees with the given part's own walk, and the disagreement is accepted.** `tail_reading` reads the part as the comma-less name reads it; assign's walk over the given part after a family comma reads by its own predicates -- a shape-only numeral VI to X stays a name word there (`Smith, John van der Berg VI` reads middle 'van der Berg VI'), V before a third comma part is read strictly (#144), and P6 attaches a period-marked title-particle (`St.`) the comma-less reading takes as a title. So a bound or a decline group computes with `tail_reading` acts where the walk then reads no title: #623's chain bound (`Smith, John Mc Prof. VI`) and #637's join decline (`Smith, abdul Prof. VI` → middle 'Prof. VI', `Smith, abdul St.` → family 'St. Smith'). Both were recorded rather than narrowed (Derek: garbage input; decisions.md#H5 and #P5, rules.md#P5's Accepted clause). Before writing another group-side predictor of the given part, sweep those three shapes and decide, rather than rediscovering the gap in review. -**A comparison that runs both sides in one tree reports 0 differences, which is what success looks like.** `tools/differential/compare.py` has been hardened against this — it generates its baseline worker into a temp dir, strips `PYTHONPATH` from the child, and aborts unless both the version AND the resolved path check out on each side; its README's three invocation traps are the analysis behind that design, worth reading before changing the harness. **The exposure is ad-hoc comparisons you write yourself**, where the same collapse has a cause the harness cannot disarm for you: **the shell's working directory persists between tool calls**, so a two-tree comparison written as two `cd`s silently runs both halves in whichever tree it landed in. Pin absolute paths, and assert `nameparser.__file__` on both sides the way `compare.py` does. The general rule outlives any particular trap: before believing a null result, prove the harness can report a difference — a clean run and a broken harness are the same output. The converse trap is a mutation that moves nothing: it shows the code is unexercised by your grids, not that it is dead. #638 was filed calling group's chain re-ask dead on a grid with no title-particle head before a particle run and a dotted acronym; the next review found 400 moves (`Smith, Freiherr van Berg X.Y.Z.`) and the issue closed the same day. Before calling code dead, look for the input that reaches it. +**A comparison that runs both sides in one tree reports 0 differences, which is what success looks like.** `tools/differential/compare.py` has been hardened against this — it generates its baseline worker into a temp dir, strips `PYTHONPATH` from the child, and aborts unless both the version AND the resolved path check out on each side; its README's three invocation traps are the analysis behind that design, worth reading before changing the harness. **The exposure is ad-hoc comparisons you write yourself**, where the same collapse has a cause the harness cannot disarm for you: **the shell's working directory persists between tool calls**, so a two-tree comparison written as two `cd`s silently runs both halves in whichever tree it landed in. Pin absolute paths, and assert `nameparser.__file__` on both sides the way `compare.py` does. The general rule outlives any particular trap: before believing a null result, prove the harness can report a difference — a clean run and a broken harness are the same output. The proof has to come from the input class the null result is about: #683 moved `tools/name_usage.py` to a POST and checked it with `shehu`, which is ASCII and so cannot show that the server decodes a Cyrillic body. A misdecoded query still runs and matches nothing, so the Cyrillic zeros it was meant to vouch for proved nothing until `Иван` (17,119 given) went through the same path. The converse trap is a mutation that moves nothing: it shows the code is unexercised by your grids, not that it is dead. #638 was filed calling group's chain re-ask dead on a grid with no title-particle head before a particle run and a dotted acronym; the next review found 400 moves (`Smith, Freiherr van Berg X.Y.Z.`) and the issue closed the same day. Before calling code dead, look for the input that reaches it. **`SUFFIX_ACRONYMS` and `SUFFIX_WORDS` are asserted DISJOINT at import (since 2026-09-08) — a bulk post-nominal import that re-creates an overlap fails there, and the fix is to choose a set, not to relax the assert** — the two branches normalize differently (the word test strips only edge periods, the acronym test strips all of them), so a word in both is matched by two rules and which one fired is unreadable from outside. `esq` was the single overlap and left `SUFFIX_ACRONYMS`, which is what made the assert possible; the whole algebra that had made the overlap safe, and the criterion that decided which set a word belongs to, are in the `suffix-acronym-collisions` entry of `docs/design/decisions.md` if anyone wants them.