diff --git a/tests/v2/_differential_fixtures.py b/tests/v2/_differential_fixtures.py index 8ccb6025..eba5999c 100644 --- a/tests/v2/_differential_fixtures.py +++ b/tests/v2/_differential_fixtures.py @@ -9,6 +9,7 @@ """ import importlib.util import json +import time import urllib.request from pathlib import Path from types import ModuleType @@ -75,15 +76,21 @@ def read(self) -> bytes: return self.body -def serve(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: bytes) -> list[str]: - """Answer every urlopen with `body`, cache into `tmp_path`, and return - the list each fetched URL is appended to: the build scripts' network, - offline.""" +def serve(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: bytes, *, + requests: list[urllib.request.Request] | None = None, + sleeps: list[float] | None = None) -> list[str]: + """Answer every urlopen with `body` and skip every sleep, cache into + `tmp_path`, and return the list each fetched URL is appended to: the + build scripts' network, offline. `requests` and `sleeps`, when given, + collect every Request sent and every sleep asked for.""" monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch.setattr(time, "sleep", (sleeps if sleeps is not None else []).append) fetched: list[str] = [] def urlopen(request: urllib.request.Request, timeout: int) -> _Response: fetched.append(request.full_url) + if requests is not None: + requests.append(request) return _Response(body) monkeypatch.setattr(urllib.request, "urlopen", urlopen) return fetched diff --git a/tests/v2/test_corpus_build.py b/tests/v2/test_corpus_build.py index d85f5cd7..53f0987d 100644 --- a/tests/v2/test_corpus_build.py +++ b/tests/v2/test_corpus_build.py @@ -9,11 +9,13 @@ import json import subprocess import sys +import time +import urllib.request from pathlib import Path import pytest -from ._differential_fixtures import _TOOLS, load_tool +from ._differential_fixtures import _TOOLS, _Response, load_tool corpus_build = load_tool("corpus_build") @@ -37,6 +39,59 @@ def test_each_builder_finds_the_shared_module_on_its_own( assert result.stdout.strip() == "True" +def _record_fetches(monkeypatch: pytest.MonkeyPatch) -> list[tuple[urllib.request.Request, int]]: + """Answer every urlopen with b"ok"; each entry is the request and its timeout.""" + seen: list[tuple[urllib.request.Request, int]] = [] + + def urlopen(request: urllib.request.Request, timeout: int) -> _Response: + seen.append((request, timeout)) + return _Response(b"ok") + monkeypatch.setattr(urllib.request, "urlopen", urlopen) + return seen + + +def test_fetch_bytes_sends_the_user_agent_and_the_extra_headers( + monkeypatch: pytest.MonkeyPatch) -> None: + seen = _record_fetches(monkeypatch) + body = corpus_build.fetch_bytes("https://x.test/a", user_agent="bot/1", timeout=7, + headers={"Accept": "application/json"}) + [(request, timeout)] = seen + assert body == b"ok" and timeout == 7 + assert request.get_method() == "GET" and request.data is None + assert request.get_header("User-agent") == "bot/1" + assert request.get_header("Accept") == "application/json" + + +def test_fetch_bytes_posts_when_given_data(monkeypatch: pytest.MonkeyPatch) -> None: + seen = _record_fetches(monkeypatch) + corpus_build.fetch_bytes("https://x.test/q", user_agent="bot/1", timeout=1, data=b"a=b") + [(request, _)] = seen + assert request.get_method() == "POST" and request.data == b"a=b" + + +def test_fetch_bytes_sleeps_only_when_given_a_delay( + monkeypatch: pytest.MonkeyPatch) -> None: + slept: list[float] = [] + _record_fetches(monkeypatch) + monkeypatch.setattr(time, "sleep", slept.append) + corpus_build.fetch_bytes("https://x.test/a", user_agent="bot/1", timeout=1) + assert slept == [] + corpus_build.fetch_bytes("https://x.test/a", user_agent="bot/1", timeout=1, delay=1) + assert slept == [1] + + +def test_a_failed_fetch_does_not_sleep(monkeypatch: pytest.MonkeyPatch) -> None: + slept: list[float] = [] + + def urlopen(request: urllib.request.Request, timeout: int) -> None: + raise OSError("down") + monkeypatch.setattr(urllib.request, "urlopen", urlopen) + monkeypatch.setattr(time, "sleep", slept.append) + with pytest.raises(OSError, match="down"): + corpus_build.fetch_bytes("https://x.test/a", user_agent="bot/1", timeout=1, delay=1) + assert slept == [] + + def test_a_shape_must_be_an_int_or_none() -> None: with pytest.raises(TypeError, match="shape must be an int or None, not '4'"): corpus_build.corpus_lines([{"text": "A", "shape": "4"}]) diff --git a/tests/v2/test_wikidata_corpus.py b/tests/v2/test_wikidata_corpus.py index 50c75b63..512f50ce 100644 --- a/tests/v2/test_wikidata_corpus.py +++ b/tests/v2/test_wikidata_corpus.py @@ -305,11 +305,6 @@ def test_a_query_takes_as_many_people_as_the_pool_check_expects() -> None: # -- the network and the cache ------------------------------------------------ -def _serve(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: bytes) -> list[str]: - monkeypatch.setattr(build.time, "sleep", lambda seconds: None) - return serve(monkeypatch, tmp_path, body) - - def _results(n: int) -> bytes: return json.dumps({"results": {"bindings": [ {"item": {"value": f"http://www.wikidata.org/entity/Q{7747 + i}"}} @@ -320,9 +315,22 @@ def _results(n: int) -> bytes: _RESULTS = _results(build.POOL_FACTOR) +def test_a_query_is_posted_as_json_and_the_fetch_waits_after_it( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + requests: list[Any] = [] + sleeps: list[float] = [] + serve(monkeypatch, tmp_path, _RESULTS, requests=requests, sleeps=sleeps) + build.pool(_RU, refresh=False) + [request] = requests + assert request.get_method() == "POST" and b"query=" in request.data + assert request.get_header("Accept") == "application/json" + assert request.get_header("User-agent") == build.USER_AGENT + assert sleeps == [1] + + def test_a_cached_response_is_read_without_fetching( monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - fetched = _serve(monkeypatch, tmp_path, _RESULTS) + fetched = serve(monkeypatch, tmp_path, _RESULTS) assert build.pool(_RU, refresh=False)[0] == "Q7747" assert len(build.pool(_RU, refresh=False)) == build.POOL_FACTOR assert len(fetched) == 1 @@ -336,7 +344,7 @@ def test_a_cached_response_is_read_without_fetching( b'Query timeout']) def test_a_response_that_is_not_what_was_asked_for_is_not_cached( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: bytes) -> None: - _serve(monkeypatch, tmp_path, body) + serve(monkeypatch, tmp_path, body) with pytest.raises(ValueError, match="not JSON|has no 'entities'.*maxlag"): build.entities(["Q1"], "labels", refresh=False) assert not list((tmp_path / "wikidata").iterdir()) @@ -344,7 +352,7 @@ def test_a_response_that_is_not_what_was_asked_for_is_not_cached( def test_a_damaged_cache_file_is_named( monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - _serve(monkeypatch, tmp_path, _RESULTS) + serve(monkeypatch, tmp_path, _RESULTS) build.pool(_RU, refresh=False) [cached] = (tmp_path / "wikidata").iterdir() cached.write_text('{"results": {"bind', encoding="utf-8") @@ -354,14 +362,14 @@ def test_a_damaged_cache_file_is_named( def test_a_query_matching_nothing_stops_the_build( monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - _serve(monkeypatch, tmp_path, b'{"results": {"bindings": []}}') + serve(monkeypatch, tmp_path, b'{"results": {"bindings": []}}') with pytest.raises(ValueError, match="patronymic-ru: the query matches nothing"): build.pool(_RU, refresh=False) def test_a_pool_short_of_its_size_stops_the_build_unless_it_is_the_whole( monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - _serve(monkeypatch, tmp_path, _results(3)) + serve(monkeypatch, tmp_path, _results(3)) with pytest.raises(ValueError, match="patronymic-ru: the query matched 3 people, " "fewer than its pool of 6"): build.pool(_RU, refresh=False) @@ -370,7 +378,7 @@ def test_a_pool_short_of_its_size_stops_the_build_unless_it_is_the_whole( def test_a_stratum_marked_whole_whose_pool_fills_stops_the_build( monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - _serve(monkeypatch, tmp_path, _results(build.POOL_FACTOR)) + serve(monkeypatch, tmp_path, _results(build.POOL_FACTOR)) with pytest.raises(ValueError, match="patronymic-ru: marked whole=True, but the " "query filled"): build.pool(_RU._replace(whole=True), refresh=False) @@ -378,7 +386,7 @@ def test_a_stratum_marked_whole_whose_pool_fills_stops_the_build( def test_an_item_the_response_leaves_out_stops_the_build( monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - _serve(monkeypatch, tmp_path, b'{"entities": {"Q1": {"id": "Q1"}}}') + serve(monkeypatch, tmp_path, b'{"entities": {"Q1": {"id": "Q1"}}}') with pytest.raises(ValueError, match="left out Q2"): build.entities(["Q1", "Q2"], "labels", refresh=False) diff --git a/tests/v2/test_wikidata_native_corpus.py b/tests/v2/test_wikidata_native_corpus.py index 1ee335ce..fbcb656a 100644 --- a/tests/v2/test_wikidata_native_corpus.py +++ b/tests/v2/test_wikidata_native_corpus.py @@ -264,6 +264,8 @@ def _stub_build(monkeypatch: pytest.MonkeyPatch, strata: tuple[Any, ...], for qs in pooled.values() for q in qs} if "Q11" in people: people["Q11"]["claims"]["P244"] = [_claim("n82128473")] + if "Q12" in people: # past the first stratum's quota: never taken, so never kept + people["Q12"]["claims"]["P244"] = [_claim("n99999999")] names = _names("ko", Q2="김", Q3="기영") monkeypatch.setattr(native, "STRATA", strata) monkeypatch.setattr(native, "pool", lambda stratum, refresh: pooled[stratum.label]) @@ -316,12 +318,6 @@ def test_a_query_names_the_strata_label_language() -> None: # -- the network and the cache ------------------------------------------------ -def _serve(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: bytes) -> list[str]: - # native imports build_wikidata_corpus by name, not as load_tool's copy - monkeypatch.setattr(native.wikidata.time, "sleep", lambda seconds: None) - return serve(monkeypatch, tmp_path, body) - - def _results(n: int) -> bytes: return json.dumps({"results": {"bindings": [ {"item": {"value": f"http://www.wikidata.org/entity/Q{i}"}} @@ -330,14 +326,14 @@ def _results(n: int) -> bytes: def test_a_query_matching_nothing_stops_the_build( monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - _serve(monkeypatch, tmp_path, _results(0)) + serve(monkeypatch, tmp_path, _results(0)) with pytest.raises(ValueError, match="ko-kr: the query matches nothing"): native.pool(_STRATA["ko-kr"], refresh=False) def test_a_pool_short_of_its_size_stops_the_build_unless_it_is_the_whole( monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - _serve(monkeypatch, tmp_path, _results(5)) + serve(monkeypatch, tmp_path, _results(5)) with pytest.raises(ValueError, match="ko-kr: the query matched 5 people, fewer " "than its pool of 300"): native.pool(_STRATA["ko-kr"], refresh=False) @@ -349,14 +345,14 @@ def test_a_stratum_marked_whole_whose_pool_fills_stops_the_build( # The population has outgrown the mark, which would otherwise go on # excusing a short pool. whole = native.Stratum("w", "Q869", "th", 1, 3, whole=True) - _serve(monkeypatch, tmp_path, _results(3)) + serve(monkeypatch, tmp_path, _results(3)) with pytest.raises(ValueError, match="w: marked whole=True, but the query filled"): native.pool(whole, refresh=False) def test_entities_asks_only_for_the_languages_labels( monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - fetched = _serve(monkeypatch, tmp_path, b'{"entities": {"Q1": {"id": "Q1"}}}') + fetched = serve(monkeypatch, tmp_path, b'{"entities": {"Q1": {"id": "Q1"}}}') native.entities(["Q1"], "zh", refresh=False) [url] = fetched assert "languages=zh%7Czh-hans%7Czh-hant%7Czh-cn%7Czh-tw%7Czh-hk%7Cmul" in url @@ -364,10 +360,12 @@ def test_entities_asks_only_for_the_languages_labels( def test_an_lc_record_is_cached_and_read_back( monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - fetched = _serve(monkeypatch, tmp_path, _MARC.encode()) + sleeps: list[float] = [] + fetched = serve(monkeypatch, tmp_path, _MARC.encode(), sleeps=sleeps) assert native.lc_variants("nr93008484", refresh=False) == ["הלוי, אהרן", "이 병기"] assert native.lc_variants("nr93008484", refresh=False) == ["הלוי, אהרן", "이 병기"] assert len(fetched) == 1 + assert sleeps == [1] # one fetch, one polite wait; the cached read waits for nothing def _fail(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, code: int) -> None: diff --git a/tools/differential/README.md b/tools/differential/README.md index e9422eaa..4d55ff81 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -421,7 +421,7 @@ A second view measures the locale packs (#672). `LOCALE_FILES` in `cldr_agreemen **The GND source.** `build_gnd_corpus.py` writes the GND's name parts in CLDR's field vocabulary (`forename` → `given`, `prefix` → `surname-prefix` with a one-word `surname` as `surname-core`, any other family → `surname` whole, `counting` → `generation`, and a `personalName` -- a person entered under the whole name, with no surname -- as `given` alone, so the family is expected empty), so `cldr_agreement.py` compares `gnd_fields.jsonl` by the same rules and records its disagreements in `gnd_agreement.json`, ratcheted by the same test. The locale view does not read it: no pack covers its names. A composed rendering says so (`composed: true`); where a particle is glued to the surname (`d'Arenberg`, `al-Ǧaʿfarī`) the composed line states the glued word as a whole `surname`, since the comparison matches fields word by word, and the index form, where the particle is a word of its own, keeps the split. A family with no `prefix`, or with a surname of several words, is stated whole for the same kind of reason: GND does not split a particle inside a surname (`Vélez de Guevara`, `Zuylen van Nijevelt`), so what it leaves unsplit means "not stated", not "no particle", and only `family` is compared there. Raw lobid responses are cached outside the tree, in a `gnd/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. The build stops before writing anything when a stratum falls below its floor, a query matches nothing, or a pool pages short, so a lobid change fails it rather than shrinking the corpus; `--record` prints, per source, how many changes of each kind it accepts. -**Adding a source.** Map only what the source states. The comparison reads a missing field as a statement that the role is empty, so a field the source leaves unstated must be mapped so that its role is not compared at all: #688's first build wrote a prefixless GND surname as `surname-core`, and 19 renderings recorded the parser's `de` in `Vélez de Guevara` as a disagreement no fix could clear; its fix then missed the same gap inside a several-word core after a prefix (`Zuylen van Nijevelt, Julius van`), so test each way a field can be left unstated, not the one that was reported. A field must also occur in its rendering's text, word by word (`test_every_committed_field_occurs_in_its_rendering`), or it is expected empty too. A build must fail rather than shrink: give every stratum a floor, check a response's shape before caching it (an API error arrives as HTTP 200), and raise on a query that matches nothing. But tell a broken fetch from a fact about the data: #689's first fix raised on any item Wikidata reported missing, so one deleted name-part item anywhere in a pool would have stopped every rebuild, with `--refresh` advice that could not help; a missing part now only refuses the person stating it. A source sampled by script must check each text's script itself: Wikidata's `mul` labels are Latin, and the native-script build's first probe kept `Anushka Sharma` as a Hindi name. And give each refusal a test row that fails for the reason it names, not for another: that build's review found an all-Hebrew text standing for "a Hebrew point is not Arabic", which passed with the mark check deleted. A builder change has one oracle the suite cannot run, since it needs the local cache: rebuild offline from `$NAMEPARSER_TOOLS_CACHE` and diff the result against the committed fields file. The same review's mutation pass scored every surviving mutation that way, and four of them would have moved committed rows or the figures quoted from them with every test passing. The cache (`cache_dir`, `cached_json`, which checks a response before caching it and writes atomically) and the two output files (`corpus_lines`, `fields_lines`, `write_corpus`) are shared in `corpus_build.py`; a new builder imports them rather than copying another builder's. +**Adding a source.** Map only what the source states. The comparison reads a missing field as a statement that the role is empty, so a field the source leaves unstated must be mapped so that its role is not compared at all: #688's first build wrote a prefixless GND surname as `surname-core`, and 19 renderings recorded the parser's `de` in `Vélez de Guevara` as a disagreement no fix could clear; its fix then missed the same gap inside a several-word core after a prefix (`Zuylen van Nijevelt, Julius van`), so test each way a field can be left unstated, not the one that was reported. A field must also occur in its rendering's text, word by word (`test_every_committed_field_occurs_in_its_rendering`), or it is expected empty too. A build must fail rather than shrink: give every stratum a floor, check a response's shape before caching it (an API error arrives as HTTP 200), and raise on a query that matches nothing. But tell a broken fetch from a fact about the data: #689's first fix raised on any item Wikidata reported missing, so one deleted name-part item anywhere in a pool would have stopped every rebuild, with `--refresh` advice that could not help; a missing part now only refuses the person stating it. A source sampled by script must check each text's script itself: Wikidata's `mul` labels are Latin, and the native-script build's first probe kept `Anushka Sharma` as a Hindi name. And give each refusal a test row that fails for the reason it names, not for another: that build's review found an all-Hebrew text standing for "a Hebrew point is not Arabic", which passed with the mark check deleted. A builder change has one oracle the suite cannot run, since it needs the local cache: rebuild offline from `$NAMEPARSER_TOOLS_CACHE` and diff the result against the committed fields file. The same review's mutation pass scored every surviving mutation that way, and four of them would have moved committed rows or the figures quoted from them with every test passing. The fetch (`fetch_bytes`: User-Agent, timeout, optional POST body and politeness delay), the cache (`cache_dir`, `cached_json`, which checks a response before caching it and writes atomically) and the two output files (`corpus_lines`, `fields_lines`, `write_corpus`) are shared in `corpus_build.py`, and the two Wikidata builders share their stratum loop (`build_strata` in `build_wikidata_corpus.py`: a person taken once across strata, the missing-person stop, the floor stop) as well as the pool check; a new builder imports them rather than copying another builder's, because a guard added to one copy is how #696 happened. **The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, a pool short of its size (unless the stratum is marked `whole`, which a full pool then contradicts) and a response that is not what was asked for each stop the build before anything is written; Iceland (270 of a pool of 600 on 2026-10-16, measured in the native paragraph below) and Malaysia are marked `whole`; Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree, in a `wikidata/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. diff --git a/tools/differential/build_cldr_corpus.py b/tools/differential/build_cldr_corpus.py index bc2bd52e..3c206d94 100644 --- a/tools/differential/build_cldr_corpus.py +++ b/tools/differential/build_cldr_corpus.py @@ -49,15 +49,15 @@ import argparse import json import sys -import urllib.request from pathlib import Path -# The shared cache and file helpers (corpus_build.py), found by putting +# The shared fetch, cache and file helpers (corpus_build.py), found by putting # this directory on sys.path: see corpus_build's docstring for why. _HERE = str(Path(__file__).resolve().parent) if _HERE not in sys.path: sys.path.append(_HERE) +import corpus_build # noqa: E402 from corpus_build import corpus_lines, fields_lines, write_corpus # noqa: E402,F401 HERE = Path(__file__).resolve().parent @@ -138,10 +138,7 @@ def parse_test_file(text: str, stem: str) -> list[dict[str, object]]: def _get(url: str) -> bytes: - request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) - with urllib.request.urlopen(request, timeout=60) as response: - data: bytes = response.read() - return data + return corpus_build.fetch_bytes(url, user_agent=USER_AGENT, timeout=60) def main() -> None: diff --git a/tools/differential/build_gnd_corpus.py b/tools/differential/build_gnd_corpus.py index f3c4e290..5ecbed2f 100644 --- a/tools/differential/build_gnd_corpus.py +++ b/tools/differential/build_gnd_corpus.py @@ -91,12 +91,11 @@ import hashlib import sys import urllib.parse -import urllib.request from collections.abc import Callable from pathlib import Path from typing import Any, NamedTuple -# The shared cache and file helpers (corpus_build.py), found by putting +# The shared fetch, cache and file helpers (corpus_build.py), found by putting # this directory on sys.path: see corpus_build's docstring for why. _HERE = str(Path(__file__).resolve().parent) if _HERE not in sys.path: @@ -251,10 +250,7 @@ def _get_json(params: dict[str, str | int], refresh: bool) -> dict[str, Any]: url = f"{SEARCH}?{urllib.parse.urlencode(params)}" def fetch() -> bytes: - request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) - with urllib.request.urlopen(request, timeout=120) as response: - body: bytes = response.read() - return body + return corpus_build.fetch_bytes(url, user_agent=USER_AGENT, timeout=120) return corpus_build.cached_json(corpus_build.cache_dir("gnd"), url, url, fetch, _checked, refresh) diff --git a/tools/differential/build_wikidata_corpus.py b/tools/differential/build_wikidata_corpus.py index 78abae37..417717b4 100644 --- a/tools/differential/build_wikidata_corpus.py +++ b/tools/differential/build_wikidata_corpus.py @@ -94,14 +94,12 @@ import argparse import sys -import time import urllib.parse -import urllib.request -from collections.abc import Iterable +from collections.abc import Callable, Iterable from pathlib import Path -from typing import Any, NamedTuple +from typing import Any, NamedTuple, Protocol, TypeVar -# The shared cache and file helpers (corpus_build.py), found by putting +# The shared fetch, cache and file helpers (corpus_build.py), found by putting # this directory on sys.path: see corpus_build's docstring for why. _HERE = str(Path(__file__).resolve().parent) if _HERE not in sys.path: @@ -212,12 +210,9 @@ def _checked(body: bytes | str, source: str, kind: str) -> dict[str, Any]: def _fetch(url: str, data: bytes | None, refresh: bool, kind: str) -> dict[str, Any]: def fetch() -> bytes: - headers = {"User-Agent": USER_AGENT, "Accept": "application/json"} - request = urllib.request.Request(url, data=data, headers=headers) - with urllib.request.urlopen(request, timeout=90) as response: - body: bytes = response.read() - time.sleep(1) # sequential, and gentle - return body + return corpus_build.fetch_bytes( + url, user_agent=USER_AGENT, timeout=90, data=data, + headers={"Accept": "application/json"}, delay=1) # sequential, and gentle return corpus_build.cached_json( corpus_build.cache_dir("wikidata"), url + "\n" + (data or b"").decode(), url, fetch, lambda body, source: _checked(body, source, kind), refresh) @@ -450,25 +445,54 @@ def renderings(qid: str, person: dict[str, Any], names: dict[str, Any], return rows -def build(refresh: bool) -> list[dict[str, Any]]: +class _Counted(Protocol): + """What the driver reads of a stratum; each builder's own NamedTuple. + Read-only properties, since a NamedTuple's fields cannot be set.""" + @property + def label(self) -> str: ... + @property + def quota(self) -> int: ... + @property + def floor(self) -> int: ... + + +S = TypeVar("S", bound=_Counted) + + +def build_strata(strata: Iterable[S], *, props: Iterable[str], label_width: int, + refresh: bool, + pool: Callable[[S, bool], list[str]], + entities: Callable[[list[str], S, bool], dict[str, Any]], + renderings: Callable[[str, dict[str, Any], dict[str, Any], S], + list[dict[str, Any]]], + keep: Callable[[str, dict[str, Any]], None] | None = None, + ) -> list[dict[str, Any]]: """Every stratum's rows, or ValueError naming each stratum that fell - below its floor -- raised before main() writes anything.""" + below its floor -- raised before main() writes anything -- or on a + pooled person Wikidata reports missing. Both Wikidata builders run + their strata through here, so a guard on the loop (a person taken + once across strata, the missing-person stop, the floor) is written + once; what differs is passed in: how a stratum is pooled, how + entities are fetched for it (`entities(ids, stratum, refresh)`), how + a person is rendered, and `keep(qid, person)`, called for each + person a stratum takes. `props` are the properties whose values are + the name-part items to fetch.""" rows: list[dict[str, Any]] = [] seen: set[str] = set() short: list[str] = [] - for stratum in STRATA: + for stratum in strata: pooled = pool(stratum, refresh) qids = [q for q in pooled if q not in seen] - people = entities(qids, "labels|claims", refresh) + people = entities(qids, stratum, refresh) # A pooled person must exist; a deleted name-part item is a fact # about the data, and only refuses the person stating it. stale = sorted(q for q in qids if "missing" in people[q]) if stale: raise ValueError(f"{stratum.label}: WDQS pooled items Wikidata reports " f"missing: {', '.join(stale[:10])}; rerun with --refresh") - parts = {v for p in people.values() for prop in PARTS + parts = {v for p in people.values() for prop in props for v in statements(p, prop)} - names = entities(parts, "labels|claims", refresh) + names = entities(sorted(parts), stratum, refresh) taken = refused = 0 for qid in qids: if taken == stratum.quota: @@ -479,8 +503,10 @@ def build(refresh: bool) -> list[dict[str, Any]]: continue seen.add(qid) rows.extend(mine) + if keep is not None: + keep(qid, people[qid]) taken += 1 - print(f"{stratum.label:15} {taken:3}/{stratum.quota} names " + print(f"{stratum.label:{label_width}} {taken:3}/{stratum.quota} names " f"({refused} refused, {len(pooled) - len(qids)} taken by an " f"earlier stratum, {len(pooled)} pooled)") if taken < stratum.floor: @@ -490,6 +516,14 @@ def build(refresh: bool) -> list[dict[str, Any]]: return rows +def build(refresh: bool) -> list[dict[str, Any]]: + """Every stratum's rows; see build_strata.""" + return build_strata( + STRATA, props=PARTS, label_width=15, refresh=refresh, pool=pool, + entities=lambda ids, stratum, refresh: entities(ids, "labels|claims", refresh), + renderings=renderings) + + def main() -> None: ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--refresh", action="store_true", diff --git a/tools/differential/build_wikidata_native_corpus.py b/tools/differential/build_wikidata_native_corpus.py index 9001e8e7..5f4149cc 100644 --- a/tools/differential/build_wikidata_native_corpus.py +++ b/tools/differential/build_wikidata_native_corpus.py @@ -102,7 +102,6 @@ import unicodedata import urllib.error import urllib.parse -import urllib.request import xml.etree.ElementTree as ET from collections import Counter from pathlib import Path @@ -359,43 +358,19 @@ def lccn(person: dict[str, Any]) -> str | None: def build(refresh: bool) -> tuple[list[dict[str, Any]], dict[str, str]]: """Every stratum's rows, and the LCCN of each person used who states - one. Raises ValueError, before main() writes anything, naming each - stratum below its floor, or on a pooled person Wikidata reports + one. The loop is `wikidata.build_strata`, shared with the parent + builder. Raises ValueError, before main() writes anything, naming + each stratum below its floor, or on a pooled person Wikidata reports missing, or (from pool()) on a pool that is empty or the wrong size.""" - rows: list[dict[str, Any]] = [] lccns: dict[str, str] = {} - seen: set[str] = set() - short: list[str] = [] - for stratum in STRATA: - pooled = pool(stratum, refresh) - qids = [q for q in pooled if q not in seen] - people = entities(qids, stratum.language, refresh) - stale = sorted(q for q in qids if "missing" in people[q]) - if stale: - raise ValueError(f"{stratum.label}: WDQS pooled items Wikidata reports " - f"missing: {', '.join(stale[:10])}; rerun with --refresh") - parts = {v for p in people.values() for prop in PARTS for v in statements(p, prop)} - names = entities(sorted(parts), stratum.language, refresh) - taken = refused = 0 - for qid in qids: - if taken == stratum.quota: - break - mine = renderings(qid, people[qid], names, stratum) - if not mine: - refused += 1 - continue - seen.add(qid) - rows.extend(mine) - if (found := lccn(people[qid])) is not None: - lccns[qid] = found - taken += 1 - print(f"{stratum.label:6} {taken:3}/{stratum.quota} names ({refused} refused, " - f"{len(pooled) - len(qids)} taken by an earlier stratum, " - f"{len(pooled)} pooled)") - if taken < stratum.floor: - short.append(f"{stratum.label} {taken}/{stratum.floor}") - if short: - raise ValueError(f"strata below their floor, nothing written: {', '.join(short)}") + + def keep(qid: str, person: dict[str, Any]) -> None: + if (found := lccn(person)) is not None: + lccns[qid] = found + rows = wikidata.build_strata( + STRATA, props=PARTS, label_width=6, refresh=refresh, pool=pool, + entities=lambda ids, stratum, refresh: entities(ids, stratum.language, refresh), + renderings=renderings, keep=keep) return rows, lccns @@ -421,11 +396,8 @@ def lc_variants(lccn: str, refresh: bool) -> list[str] | None: url = LC_RECORD.format(urllib.parse.quote(lccn)) def fetch() -> bytes: - request = urllib.request.Request(url, headers={"User-Agent": wikidata.USER_AGENT}) - with urllib.request.urlopen(request, timeout=60) as response: - body: bytes = response.read() - wikidata.time.sleep(1) - return body + return corpus_build.fetch_bytes(url, user_agent=wikidata.USER_AGENT, + timeout=60, delay=1) try: record = corpus_build.cached_json(corpus_build.cache_dir("lc"), url, url, fetch, _lc_checked, refresh) diff --git a/tools/differential/corpus_build.py b/tools/differential/corpus_build.py index 04ee55a7..0488951f 100644 --- a/tools/differential/corpus_build.py +++ b/tools/differential/corpus_build.py @@ -1,10 +1,11 @@ -"""What the labeled-corpus builders share: the cache, the files they write. +"""What the labeled-corpus builders share: the fetch, the cache, the files they write. -build_cldr_corpus.py, build_gnd_corpus.py and build_wikidata_corpus.py -each write a corpus file and a fields file in the same formats, and GND -and Wikidata, the two that cache, keep their raw responses outside the -repository the same way. Not a builder itself: it writes nothing on -import, and has no main(). +The builders (build_cldr_corpus.py, build_gnd_corpus.py, +build_wikidata_corpus.py and build_wikidata_native_corpus.py) each write +a corpus file and a fields file in the same formats; every one but CLDR +caches its raw responses outside the repository the same way. Every +builder's network fetch goes through `fetch_bytes`. Not a builder itself: it writes nothing on import, and has +no main(). A builder imports it by name, after appending its own directory to sys.path: that directory is usually sys.path[0] when a builder runs as a @@ -19,7 +20,9 @@ import hashlib import json import os -from collections.abc import Callable, Iterable +import time +import urllib.request +from collections.abc import Callable, Iterable, Mapping from pathlib import Path from typing import Any @@ -35,6 +38,23 @@ def cache_dir(name: str) -> Path: return path +def fetch_bytes(url: str, *, user_agent: str, timeout: int, data: bytes | None = None, + headers: Mapping[str, str] | None = None, delay: float = 0) -> bytes: + """The response body of `url` (a POST when `data` is given), sent + with the builder's User-Agent and any further `headers`, then + `delay` seconds of sleep: a builder that fetches in a row passes + one, so that it stays gentle on the source. Raises what urllib + raises. Looks up `urllib.request.urlopen` at call time, which is + what lets the tests answer every fetch from one stub.""" + request = urllib.request.Request( + url, data=data, headers={"User-Agent": user_agent, **(headers or {})}) + with urllib.request.urlopen(request, timeout=timeout) as response: + body: bytes = response.read() + if delay: + time.sleep(delay) + return body + + def parse_json(body: bytes | str, source: str) -> object: """`body` parsed, or ValueError naming where it came from.""" try: