Share the fetch and the stratum loop across the corpus builders

tests/v2/_differential_fixtures.py

@@ -9,6 +9,7 @@
 """
 import importlib.util
 import json
+import time
 import urllib.request
 from pathlib import Path
 from types import ModuleType
@@ -75,15 +76,21 @@ def read(self) -> bytes:
         return self.body
 
 
-def serve(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: bytes) -> list[str]:
-    """Answer every urlopen with `body`, cache into `tmp_path`, and return
-    the list each fetched URL is appended to: the build scripts' network,
-    offline."""
+def serve(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: bytes, *,
+          requests: list[urllib.request.Request] | None = None,
+          sleeps: list[float] | None = None) -> list[str]:
+    """Answer every urlopen with `body` and skip every sleep, cache into
+    `tmp_path`, and return the list each fetched URL is appended to: the
+    build scripts' network, offline. `requests` and `sleeps`, when given,
+    collect every Request sent and every sleep asked for."""
     monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path))
+    monkeypatch.setattr(time, "sleep", (sleeps if sleeps is not None else []).append)
     fetched: list[str] = []
 
     def urlopen(request: urllib.request.Request, timeout: int) -> _Response:
         fetched.append(request.full_url)
+        if requests is not None:
+            requests.append(request)
         return _Response(body)
     monkeypatch.setattr(urllib.request, "urlopen", urlopen)
     return fetched

tests/v2/test_corpus_build.py

@@ -9,11 +9,13 @@
 import json
 import subprocess
 import sys
+import time
+import urllib.request
 from pathlib import Path
 
 import pytest
 
-from ._differential_fixtures import _TOOLS, load_tool
+from ._differential_fixtures import _TOOLS, _Response, load_tool
 
 corpus_build = load_tool("corpus_build")
 
@@ -37,6 +39,59 @@ def test_each_builder_finds_the_shared_module_on_its_own(
     assert result.stdout.strip() == "True"
 
 
+def _record_fetches(monkeypatch: pytest.MonkeyPatch) -> list[tuple[urllib.request.Request, int]]:
+    """Answer every urlopen with b"ok"; each entry is the request and its timeout."""
+    seen: list[tuple[urllib.request.Request, int]] = []
+
+    def urlopen(request: urllib.request.Request, timeout: int) -> _Response:
+        seen.append((request, timeout))
+        return _Response(b"ok")
+    monkeypatch.setattr(urllib.request, "urlopen", urlopen)
+    return seen
+
+
+def test_fetch_bytes_sends_the_user_agent_and_the_extra_headers(
+        monkeypatch: pytest.MonkeyPatch) -> None:
+    seen = _record_fetches(monkeypatch)
+    body = corpus_build.fetch_bytes("https://x.test/a", user_agent="bot/1", timeout=7,
+                                    headers={"Accept": "application/json"})
+    [(request, timeout)] = seen
+    assert body == b"ok" and timeout == 7
+    assert request.get_method() == "GET" and request.data is None
+    assert request.get_header("User-agent") == "bot/1"
+    assert request.get_header("Accept") == "application/json"
+
+
+def test_fetch_bytes_posts_when_given_data(monkeypatch: pytest.MonkeyPatch) -> None:
+    seen = _record_fetches(monkeypatch)
+    corpus_build.fetch_bytes("https://x.test/q", user_agent="bot/1", timeout=1, data=b"a=b")
+    [(request, _)] = seen
+    assert request.get_method() == "POST" and request.data == b"a=b"
+
+
+def test_fetch_bytes_sleeps_only_when_given_a_delay(
+        monkeypatch: pytest.MonkeyPatch) -> None:
+    slept: list[float] = []
+    _record_fetches(monkeypatch)
+    monkeypatch.setattr(time, "sleep", slept.append)
+    corpus_build.fetch_bytes("https://x.test/a", user_agent="bot/1", timeout=1)
+    assert slept == []
+    corpus_build.fetch_bytes("https://x.test/a", user_agent="bot/1", timeout=1, delay=1)
+    assert slept == [1]
+
+
+def test_a_failed_fetch_does_not_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
+    slept: list[float] = []
+
+    def urlopen(request: urllib.request.Request, timeout: int) -> None:
+        raise OSError("down")
+    monkeypatch.setattr(urllib.request, "urlopen", urlopen)
+    monkeypatch.setattr(time, "sleep", slept.append)
+    with pytest.raises(OSError, match="down"):
+        corpus_build.fetch_bytes("https://x.test/a", user_agent="bot/1", timeout=1, delay=1)
+    assert slept == []
+
+
 def test_a_shape_must_be_an_int_or_none() -> None:
     with pytest.raises(TypeError, match="shape must be an int or None, not '4'"):
         corpus_build.corpus_lines([{"text": "A", "shape": "4"}])

tests/v2/test_wikidata_corpus.py

@@ -305,11 +305,6 @@ def test_a_query_takes_as_many_people_as_the_pool_check_expects() -> None:
 
 # -- the network and the cache ------------------------------------------------
 
-def _serve(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: bytes) -> list[str]:
-    monkeypatch.setattr(build.time, "sleep", lambda seconds: None)
-    return serve(monkeypatch, tmp_path, body)
-
-
 def _results(n: int) -> bytes:
     return json.dumps({"results": {"bindings": [
         {"item": {"value": f"http://www.wikidata.org/entity/Q{7747 + i}"}}
@@ -320,9 +315,22 @@ def _results(n: int) -> bytes:
 _RESULTS = _results(build.POOL_FACTOR)
 
 
+def test_a_query_is_posted_as_json_and_the_fetch_waits_after_it(
+        monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    requests: list[Any] = []
+    sleeps: list[float] = []
+    serve(monkeypatch, tmp_path, _RESULTS, requests=requests, sleeps=sleeps)
+    build.pool(_RU, refresh=False)
+    [request] = requests
+    assert request.get_method() == "POST" and b"query=" in request.data
+    assert request.get_header("Accept") == "application/json"
+    assert request.get_header("User-agent") == build.USER_AGENT
+    assert sleeps == [1]
+
+
 def test_a_cached_response_is_read_without_fetching(
         monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    fetched = _serve(monkeypatch, tmp_path, _RESULTS)
+    fetched = serve(monkeypatch, tmp_path, _RESULTS)
     assert build.pool(_RU, refresh=False)[0] == "Q7747"
     assert len(build.pool(_RU, refresh=False)) == build.POOL_FACTOR
     assert len(fetched) == 1
@@ -336,15 +344,15 @@ def test_a_cached_response_is_read_without_fetching(
     b'<html>Query timeout</html>'])
 def test_a_response_that_is_not_what_was_asked_for_is_not_cached(
         monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: bytes) -> None:
-    _serve(monkeypatch, tmp_path, body)
+    serve(monkeypatch, tmp_path, body)
     with pytest.raises(ValueError, match="not JSON|has no 'entities'.*maxlag"):
         build.entities(["Q1"], "labels", refresh=False)
     assert not list((tmp_path / "wikidata").iterdir())
 
 
 def test_a_damaged_cache_file_is_named(
         monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    _serve(monkeypatch, tmp_path, _RESULTS)
+    serve(monkeypatch, tmp_path, _RESULTS)
     build.pool(_RU, refresh=False)
     [cached] = (tmp_path / "wikidata").iterdir()
     cached.write_text('{"results": {"bind', encoding="utf-8")
@@ -354,14 +362,14 @@ def test_a_damaged_cache_file_is_named(
 
 def test_a_query_matching_nothing_stops_the_build(
         monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    _serve(monkeypatch, tmp_path, b'{"results": {"bindings": []}}')
+    serve(monkeypatch, tmp_path, b'{"results": {"bindings": []}}')
     with pytest.raises(ValueError, match="patronymic-ru: the query matches nothing"):
         build.pool(_RU, refresh=False)
 
 
 def test_a_pool_short_of_its_size_stops_the_build_unless_it_is_the_whole(
         monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    _serve(monkeypatch, tmp_path, _results(3))
+    serve(monkeypatch, tmp_path, _results(3))
     with pytest.raises(ValueError, match="patronymic-ru: the query matched 3 people, "
                                          "fewer than its pool of 6"):
         build.pool(_RU, refresh=False)
@@ -370,15 +378,15 @@ def test_a_pool_short_of_its_size_stops_the_build_unless_it_is_the_whole(
 
 def test_a_stratum_marked_whole_whose_pool_fills_stops_the_build(
         monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    _serve(monkeypatch, tmp_path, _results(build.POOL_FACTOR))
+    serve(monkeypatch, tmp_path, _results(build.POOL_FACTOR))
     with pytest.raises(ValueError, match="patronymic-ru: marked whole=True, but the "
                                          "query filled"):
         build.pool(_RU._replace(whole=True), refresh=False)
 
 
 def test_an_item_the_response_leaves_out_stops_the_build(
         monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    _serve(monkeypatch, tmp_path, b'{"entities": {"Q1": {"id": "Q1"}}}')
+    serve(monkeypatch, tmp_path, b'{"entities": {"Q1": {"id": "Q1"}}}')
     with pytest.raises(ValueError, match="left out Q2"):
         build.entities(["Q1", "Q2"], "labels", refresh=False)
 

tests/v2/test_wikidata_native_corpus.py

@@ -264,6 +264,8 @@ def _stub_build(monkeypatch: pytest.MonkeyPatch, strata: tuple[Any, ...],
               for qs in pooled.values() for q in qs}
     if "Q11" in people:
         people["Q11"]["claims"]["P244"] = [_claim("n82128473")]
+    if "Q12" in people:  # past the first stratum's quota: never taken, so never kept
+        people["Q12"]["claims"]["P244"] = [_claim("n99999999")]
     names = _names("ko", Q2="김", Q3="기영")
     monkeypatch.setattr(native, "STRATA", strata)
     monkeypatch.setattr(native, "pool", lambda stratum, refresh: pooled[stratum.label])
@@ -316,12 +318,6 @@ def test_a_query_names_the_strata_label_language() -> None:
 
 # -- the network and the cache ------------------------------------------------
 
-def _serve(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: bytes) -> list[str]:
-    # native imports build_wikidata_corpus by name, not as load_tool's copy
-    monkeypatch.setattr(native.wikidata.time, "sleep", lambda seconds: None)
-    return serve(monkeypatch, tmp_path, body)
-
-
 def _results(n: int) -> bytes:
     return json.dumps({"results": {"bindings": [
         {"item": {"value": f"http://www.wikidata.org/entity/Q{i}"}}
@@ -330,14 +326,14 @@ def _results(n: int) -> bytes:
 
 def test_a_query_matching_nothing_stops_the_build(
         monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    _serve(monkeypatch, tmp_path, _results(0))
+    serve(monkeypatch, tmp_path, _results(0))
     with pytest.raises(ValueError, match="ko-kr: the query matches nothing"):
         native.pool(_STRATA["ko-kr"], refresh=False)
 
 
 def test_a_pool_short_of_its_size_stops_the_build_unless_it_is_the_whole(
         monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    _serve(monkeypatch, tmp_path, _results(5))
+    serve(monkeypatch, tmp_path, _results(5))
     with pytest.raises(ValueError, match="ko-kr: the query matched 5 people, fewer "
                                          "than its pool of 300"):
         native.pool(_STRATA["ko-kr"], refresh=False)
@@ -349,25 +345,27 @@ def test_a_stratum_marked_whole_whose_pool_fills_stops_the_build(
     # The population has outgrown the mark, which would otherwise go on
     # excusing a short pool.
     whole = native.Stratum("w", "Q869", "th", 1, 3, whole=True)
-    _serve(monkeypatch, tmp_path, _results(3))
+    serve(monkeypatch, tmp_path, _results(3))
     with pytest.raises(ValueError, match="w: marked whole=True, but the query filled"):
         native.pool(whole, refresh=False)
 
 
 def test_entities_asks_only_for_the_languages_labels(
         monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    fetched = _serve(monkeypatch, tmp_path, b'{"entities": {"Q1": {"id": "Q1"}}}')
+    fetched = serve(monkeypatch, tmp_path, b'{"entities": {"Q1": {"id": "Q1"}}}')
     native.entities(["Q1"], "zh", refresh=False)
     [url] = fetched
     assert "languages=zh%7Czh-hans%7Czh-hant%7Czh-cn%7Czh-tw%7Czh-hk%7Cmul" in url
 
 
 def test_an_lc_record_is_cached_and_read_back(
         monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    fetched = _serve(monkeypatch, tmp_path, _MARC.encode())
+    sleeps: list[float] = []
+    fetched = serve(monkeypatch, tmp_path, _MARC.encode(), sleeps=sleeps)
     assert native.lc_variants("nr93008484", refresh=False) == ["הלוי, אהרן", "이 병기"]
     assert native.lc_variants("nr93008484", refresh=False) == ["הלוי, אהרן", "이 병기"]
     assert len(fetched) == 1
+    assert sleeps == [1]  # one fetch, one polite wait; the cached read waits for nothing
 
 
 def _fail(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, code: int) -> None: