fix: harden Apec search and run ids

This commit is contained in:
Antoine 2026-06-03 22:09:45 +02:00
parent cdae23e303
commit 40b3619578
5 changed files with 235 additions and 39 deletions

View File

@ -16,6 +16,8 @@ _SEARCH_INPUT_SELECTOR = 'input[name="keywords"]'
_RESULT_LINK_SELECTOR = "a[href*='/candidat/recherche-emploi.html/emploi/detail-offre/']" _RESULT_LINK_SELECTOR = "a[href*='/candidat/recherche-emploi.html/emploi/detail-offre/']"
_DETAIL_JOB_ID_PATTERN = re.compile(r"/detail-offre/([^/?#]+)") _DETAIL_JOB_ID_PATTERN = re.compile(r"/detail-offre/([^/?#]+)")
_APEC_HOSTS = {"apec.fr", "www.apec.fr"} _APEC_HOSTS = {"apec.fr", "www.apec.fr"}
_MAX_PAGES_PER_QUERY = 50
_MAX_CONSECUTIVE_NO_PROGRESS_PAGES = 10
@dataclass(slots=True) @dataclass(slots=True)
@ -84,6 +86,16 @@ def _accept_cookies_if_present(page) -> None:
return return
def _goto_and_wait(page, url: str) -> bool:
try:
page.goto(url, wait_until="domcontentloaded")
page.wait_for_load_state("domcontentloaded")
except Exception:
return False
return True
def _is_public_apec_detail_url(url: str) -> bool: def _is_public_apec_detail_url(url: str) -> bool:
parsed_url = urlparse(url) parsed_url = urlparse(url)
return ( return (
@ -111,9 +123,10 @@ class ApecAdapter:
if len(results) >= self.max_listings: if len(results) >= self.max_listings:
break break
page.goto(_search_url(query, search_filters), wait_until="domcontentloaded") if not _goto_and_wait(page, _search_url(query, search_filters)):
continue
_accept_cookies_if_present(page) _accept_cookies_if_present(page)
page.wait_for_load_state("domcontentloaded")
try: try:
page.wait_for_selector(_SEARCH_INPUT_SELECTOR, timeout=5_000) page.wait_for_selector(_SEARCH_INPUT_SELECTOR, timeout=5_000)
@ -129,20 +142,24 @@ class ApecAdapter:
result_page_url = page.url result_page_url = page.url
seen_page_urls: set[str] = {result_page_url} seen_page_urls: set[str] = {result_page_url}
page_number = 0 no_progress_pages = 0
for page_number in range(_MAX_PAGES_PER_QUERY):
if len(results) >= self.max_listings:
break
while len(results) < self.max_listings:
if page_number > 0: if page_number > 0:
next_page_url = _search_results_url(result_page_url, page_number) next_page_url = _search_results_url(result_page_url, page_number)
if next_page_url in seen_page_urls: if next_page_url in seen_page_urls:
break break
page.goto(next_page_url, wait_until="domcontentloaded") if not _goto_and_wait(page, next_page_url):
break
try: try:
page.wait_for_selector(_RESULT_LINK_SELECTOR, timeout=5_000) page.wait_for_selector(_RESULT_LINK_SELECTOR, timeout=5_000)
except PlaywrightTimeoutError: except PlaywrightTimeoutError:
break break
current_page_url = page.url current_page_url = page.url
if page_number > 0 and current_page_url in seen_page_urls: if page_number > 0 and current_page_url in seen_page_urls:
@ -154,8 +171,13 @@ class ApecAdapter:
"nodes => nodes.map(node => node.href)" "nodes => nodes.map(node => node.href)"
) )
if not hrefs: if not hrefs:
break no_progress_pages += 1
if no_progress_pages >= _MAX_CONSECUTIVE_NO_PROGRESS_PAGES:
break
continue
added_any_result = False
for href in hrefs: for href in hrefs:
source_job_id = _extract_source_job_id(href) source_job_id = _extract_source_job_id(href)
dedupe_key = source_job_id or href dedupe_key = source_job_id or href
@ -164,11 +186,17 @@ class ApecAdapter:
seen_keys.add(dedupe_key) seen_keys.add(dedupe_key)
results.append(ApecSearchResult(url=href, source_job_id=source_job_id)) results.append(ApecSearchResult(url=href, source_job_id=source_job_id))
added_any_result = True
if len(results) >= self.max_listings: if len(results) >= self.max_listings:
break break
page_number += 1 if added_any_result:
no_progress_pages = 0
else:
no_progress_pages += 1
if no_progress_pages >= _MAX_CONSECUTIVE_NO_PROGRESS_PAGES:
break
if not usable_search_page_seen: if not usable_search_page_seen:
raise ApecSearchError("Apec search page was not reachable for any query") raise ApecSearchError("Apec search page was not reachable for any query")

View File

@ -134,9 +134,9 @@ def fetch_apec(
contract_type=derived_search_filters.contract_type or "CDI", contract_type=derived_search_filters.contract_type or "CDI",
) )
current = _utc_now().astimezone(timezone.utc).replace(microsecond=0) current = _utc_now().astimezone(timezone.utc)
run_id = current.strftime("%Y-%m-%dT%H-%M-%SZ") run_id = current.strftime("%Y-%m-%dT%H-%M-%S-%fZ")
run_started_at = current.strftime("%Y-%m-%dT%H:%M:%SZ") run_started_at = current.replace(microsecond=0).strftime("%Y-%m-%dT%H:%M:%SZ")
fetched_at = run_started_at fetched_at = run_started_at
adapter = ApecAdapter(max_listings=50) adapter = ApecAdapter(max_listings=50)

View File

@ -163,6 +163,112 @@ def test_search_continues_past_duplicate_only_pages_until_a_later_hit(monkeypatc
assert any("page=3" in url for url in page.goto_urls) assert any("page=3" in url for url in page.goto_urls)
def test_search_continues_after_query_and_pagination_navigation_failures(monkeypatch) -> None:
first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=beta&page=0&selectedIndex=0"
second_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=beta&page=1&selectedIndex=0"
page = _FakeDetailPage(
{
"beta": {0: [first_result], 1: [second_result]},
}
)
original_goto = page.goto
goto_calls = 0
def flaky_goto(url: str, wait_until: str | None = None) -> None:
nonlocal goto_calls
goto_calls += 1
if goto_calls == 1:
raise RuntimeError("navigation boom")
original_goto(url, wait_until=wait_until)
original_wait_for_load_state = page.wait_for_load_state
def flaky_wait_for_load_state(state: str) -> None:
if page.current_page == 1:
raise RuntimeError("load boom")
original_wait_for_load_state(state)
monkeypatch.setattr(page, "goto", flaky_goto)
monkeypatch.setattr(page, "wait_for_load_state", flaky_wait_for_load_state)
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
results = ApecAdapter(max_listings=10).search(
["alpha", "beta"],
search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
)
assert [result.url for result in results] == [first_result]
assert [result.source_job_id for result in results] == ["111"]
def test_search_stops_after_max_page_count(monkeypatch) -> None:
page = _FakeDetailPage(
{
"alpha": {
0: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"],
1: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=alpha&page=1&selectedIndex=0"],
2: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/333?motsCles=alpha&page=2&selectedIndex=0"],
3: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/444?motsCles=alpha&page=3&selectedIndex=0"],
}
}
)
original_goto = page.goto
def bounded_goto(url: str, wait_until: str | None = None) -> None:
original_goto(url, wait_until=wait_until)
if page.current_page >= 3:
raise AssertionError("pagination should stop before page 3")
monkeypatch.setattr(page, "goto", bounded_goto)
monkeypatch.setattr(adapter_module, "_MAX_PAGES_PER_QUERY", 3)
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
results = ApecAdapter(max_listings=10).search(
["alpha"],
search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
)
assert [result.source_job_id for result in results] == ["111", "222", "333"]
assert not any("page=3" in url for url in page.goto_urls)
def test_search_stops_after_consecutive_no_progress_pages(monkeypatch) -> None:
page = _FakeDetailPage(
{
"alpha": {
0: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"],
1: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=1&selectedIndex=0"],
2: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=2&selectedIndex=0"],
3: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=alpha&page=3&selectedIndex=0"],
}
}
)
original_goto = page.goto
def bounded_goto(url: str, wait_until: str | None = None) -> None:
original_goto(url, wait_until=wait_until)
if page.current_page >= 3:
raise AssertionError("pagination should stop before page 3")
monkeypatch.setattr(page, "goto", bounded_goto)
monkeypatch.setattr(adapter_module, "_MAX_CONSECUTIVE_NO_PROGRESS_PAGES", 2)
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
results = ApecAdapter(max_listings=10).search(
["alpha"],
search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
)
assert [result.source_job_id for result in results] == ["111"]
assert not any("page=3" in url for url in page.goto_urls)
def test_search_stops_when_result_page_url_repeats(monkeypatch) -> None: def test_search_stops_when_result_page_url_repeats(monkeypatch) -> None:
first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0" first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"
page = _FakeDetailPage( page = _FakeDetailPage(

View File

@ -12,7 +12,10 @@ from job_research.cli import app
def _fixed_now() -> datetime: def _fixed_now() -> datetime:
return datetime(2026, 6, 1, 10, 0, 0, tzinfo=timezone.utc) return datetime(2026, 6, 1, 10, 0, 0, 123456, tzinfo=timezone.utc)
FIXED_RUN_ID = _fixed_now().strftime("%Y-%m-%dT%H-%M-%S-%fZ")
def _apec_detail_html( def _apec_detail_html(
@ -271,7 +274,7 @@ def test_fetch_apec_reports_snapshot_directory_creation_failures_cleanly(
assert adapter.search_calls == [["Role From YAML"]] assert adapter.search_calls == [["Role From YAML"]]
assert adapter.fetch_calls == [] assert adapter.fetch_calls == []
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
assert not (run_dir / "snapshots").exists() assert not (run_dir / "snapshots").exists()
@ -320,7 +323,7 @@ def test_fetch_apec_succeeds_with_empty_search_results(tmp_path, monkeypatch) ->
assert adapter.search_calls == [["Role From YAML"]] assert adapter.search_calls == [["Role From YAML"]]
assert adapter.fetch_calls == [] assert adapter.fetch_calls == []
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
assert (run_dir / "snapshots").exists() assert (run_dir / "snapshots").exists()
assert list((run_dir / "snapshots").iterdir()) == [] assert list((run_dir / "snapshots").iterdir()) == []
@ -329,7 +332,7 @@ def test_fetch_apec_succeeds_with_empty_search_results(tmp_path, monkeypatch) ->
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload == { assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z", "run_id": FIXED_RUN_ID,
"run_started_at": "2026-06-01T10:00:00Z", "run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"], "derived_queries": ["Role From YAML"],
"snapshots": [], "snapshots": [],
@ -341,6 +344,62 @@ def test_fetch_apec_succeeds_with_empty_search_results(tmp_path, monkeypatch) ->
} }
def test_fetch_apec_uses_unique_run_dirs_for_runs_started_within_same_second(
tmp_path,
monkeypatch,
) -> None:
data_root = tmp_path / "data"
data_root.mkdir()
(data_root / "candidate-profile.yaml").write_text(
dedent(
"""
target_roles:
- Role From YAML
"""
).strip(),
encoding="utf-8",
)
class EmptyResultAdapter:
instances: list["EmptyResultAdapter"] = []
def __init__(self, max_listings: int = 50) -> None:
self.max_listings = max_listings
self.search_calls: list[list[str]] = []
self.fetch_calls: list[str] = []
EmptyResultAdapter.instances.append(self)
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
self.search_calls.append(list(queries))
return []
def fetch_listing_html(self, url: str) -> str:
self.fetch_calls.append(url)
raise AssertionError("fetch should not be called when search returns no results")
timestamps = [
datetime(2026, 6, 1, 10, 0, 0, 111111, tzinfo=timezone.utc),
datetime(2026, 6, 1, 10, 0, 0, 222222, tzinfo=timezone.utc),
]
expected_run_ids = [timestamp.strftime("%Y-%m-%dT%H-%M-%S-%fZ") for timestamp in timestamps]
def next_now() -> datetime:
return timestamps.pop(0)
monkeypatch.setattr("job_research.cli.ApecAdapter", EmptyResultAdapter)
monkeypatch.setattr("job_research.cli._utc_now", next_now)
runner = CliRunner()
first = runner.invoke(app, ["fetch-apec", "--data-root", str(data_root)])
second = runner.invoke(app, ["fetch-apec", "--data-root", str(data_root)])
assert first.exit_code == 0
assert second.exit_code == 0
runs_root = data_root / "apec" / "runs"
assert sorted(path.name for path in runs_root.iterdir()) == expected_run_ids
@pytest.mark.parametrize("failing_filename", ["listings.yaml", "run-meta.yaml"]) @pytest.mark.parametrize("failing_filename", ["listings.yaml", "run-meta.yaml"])
def test_fetch_apec_reports_final_artifact_write_failures_cleanly( def test_fetch_apec_reports_final_artifact_write_failures_cleanly(
tmp_path, tmp_path,
@ -406,7 +465,7 @@ def test_fetch_apec_reports_final_artifact_write_failures_cleanly(
assert adapter.search_calls == [["Role From YAML"]] assert adapter.search_calls == [["Role From YAML"]]
assert adapter.fetch_calls == ["https://example.test/job/123"] assert adapter.fetch_calls == ["https://example.test/job/123"]
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
if failing_filename == "listings.yaml": if failing_filename == "listings.yaml":
assert not (run_dir / "listings.yaml").exists() assert not (run_dir / "listings.yaml").exists()
assert (run_dir / "run-meta.yaml").exists() assert (run_dir / "run-meta.yaml").exists()
@ -499,7 +558,7 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload == { assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z", "run_id": FIXED_RUN_ID,
"run_started_at": "2026-06-01T10:00:00Z", "run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"], "derived_queries": ["Role From YAML"],
"snapshots": [ "snapshots": [
@ -572,7 +631,7 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings(
assert adapter.search_calls == [["Role From YAML"]] assert adapter.search_calls == [["Role From YAML"]]
assert adapter.fetch_calls == [good_result.url, bad_result.url] assert adapter.fetch_calls == [good_result.url, bad_result.url]
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
snapshot_files = list((run_dir / "snapshots").glob("*.html")) snapshot_files = list((run_dir / "snapshots").glob("*.html"))
assert [snapshot.name for snapshot in snapshot_files] == ["job-123.html"] assert [snapshot.name for snapshot in snapshot_files] == ["job-123.html"]
assert snapshot_files[0].read_text(encoding="utf-8") == html assert snapshot_files[0].read_text(encoding="utf-8") == html
@ -596,7 +655,7 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings(
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload == { assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z", "run_id": FIXED_RUN_ID,
"run_started_at": "2026-06-01T10:00:00Z", "run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"], "derived_queries": ["Role From YAML"],
"snapshots": [ "snapshots": [
@ -679,7 +738,7 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_li
assert "deduplicated=2" in result.stdout assert "deduplicated=2" in result.stdout
assert "failed=1" in result.stdout assert "failed=1" in result.stdout
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
snapshot_files = sorted((run_dir / "snapshots").glob("*.html")) snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
assert [snapshot.name for snapshot in snapshot_files] == ["job-456.html"] assert [snapshot.name for snapshot in snapshot_files] == ["job-456.html"]
@ -715,7 +774,7 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_li
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload == { assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z", "run_id": FIXED_RUN_ID,
"run_started_at": "2026-06-01T10:00:00Z", "run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"], "derived_queries": ["Role From YAML"],
"snapshots": [ "snapshots": [
@ -799,7 +858,7 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails(
assert "deduplicated=2" in result.stdout assert "deduplicated=2" in result.stdout
assert "failed=2" in result.stdout assert "failed=2" in result.stdout
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
snapshot_files = sorted((run_dir / "snapshots").glob("*.html")) snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
assert snapshot_files == [] assert snapshot_files == []
@ -835,7 +894,7 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails(
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload == { assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z", "run_id": FIXED_RUN_ID,
"run_started_at": "2026-06-01T10:00:00Z", "run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"], "derived_queries": ["Role From YAML"],
"snapshots": [ "snapshots": [
@ -931,14 +990,14 @@ def test_fetch_apec_counts_each_failed_url_once_even_when_multiple_errors_are_re
assert "deduplicated=1" in result.stdout assert "deduplicated=1" in result.stdout
assert "failed=1" in result.stdout assert "failed=1" in result.stdout
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
snapshot_files = sorted((run_dir / "snapshots").glob("*.html")) snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
assert [snapshot.name for snapshot in snapshot_files] == ["job-456.html"] assert [snapshot.name for snapshot in snapshot_files] == ["job-456.html"]
assert snapshot_files[0].read_text(encoding="utf-8") == html assert snapshot_files[0].read_text(encoding="utf-8") == html
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload == { assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z", "run_id": FIXED_RUN_ID,
"run_started_at": "2026-06-01T10:00:00Z", "run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"], "derived_queries": ["Role From YAML"],
"snapshots": [ "snapshots": [
@ -1015,7 +1074,7 @@ def test_fetch_apec_preserves_snapshot_and_run_meta_when_normalization_fails(
assert "deduplicated=0" in result.stdout assert "deduplicated=0" in result.stdout
assert "failed=1" in result.stdout assert "failed=1" in result.stdout
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
snapshot_file = run_dir / "snapshots" / "job-123.html" snapshot_file = run_dir / "snapshots" / "job-123.html"
assert snapshot_file.exists() assert snapshot_file.exists()
assert snapshot_file.read_text(encoding="utf-8") == html assert snapshot_file.read_text(encoding="utf-8") == html
@ -1025,7 +1084,7 @@ def test_fetch_apec_preserves_snapshot_and_run_meta_when_normalization_fails(
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload == { assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z", "run_id": FIXED_RUN_ID,
"run_started_at": "2026-06-01T10:00:00Z", "run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"], "derived_queries": ["Role From YAML"],
"snapshots": [ "snapshots": [
@ -1097,7 +1156,7 @@ def test_fetch_apec_writes_artifacts_when_every_normalization_attempt_fails(
assert "deduplicated=0" in result.stdout assert "deduplicated=0" in result.stdout
assert "failed=2" in result.stdout assert "failed=2" in result.stdout
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
snapshot_files = sorted((run_dir / "snapshots").glob("*.html")) snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
assert [snapshot.name for snapshot in snapshot_files] == ["job-123.html", "job-456.html"] assert [snapshot.name for snapshot in snapshot_files] == ["job-123.html", "job-456.html"]
assert [snapshot.read_text(encoding="utf-8") for snapshot in snapshot_files] == [html, html] assert [snapshot.read_text(encoding="utf-8") for snapshot in snapshot_files] == [html, html]
@ -1107,7 +1166,7 @@ def test_fetch_apec_writes_artifacts_when_every_normalization_attempt_fails(
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload == { assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z", "run_id": FIXED_RUN_ID,
"run_started_at": "2026-06-01T10:00:00Z", "run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"], "derived_queries": ["Role From YAML"],
"snapshots": [ "snapshots": [
@ -1214,7 +1273,7 @@ def test_fetch_apec_processes_only_the_first_fifty_search_results(tmp_path, monk
assert adapter.search_calls == [["Role From YAML"]] assert adapter.search_calls == [["Role From YAML"]]
assert adapter.fetch_calls == [result.url for result in results[:50]] assert adapter.fetch_calls == [result.url for result in results[:50]]
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
snapshot_files = sorted((run_dir / "snapshots").glob("*.html")) snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
assert len(snapshot_files) == 50 assert len(snapshot_files) == 50
@ -1271,6 +1330,6 @@ def test_fetch_apec_fails_when_every_listing_attempt_fails(tmp_path, monkeypatch
assert adapter.search_calls == [["Role From YAML"]] assert adapter.search_calls == [["Role From YAML"]]
assert adapter.fetch_calls == [result_item.url for result_item in results] assert adapter.fetch_calls == [result_item.url for result_item in results]
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
assert not (run_dir / "listings.yaml").exists() assert not (run_dir / "listings.yaml").exists()
assert not (run_dir / "run-meta.yaml").exists() assert not (run_dir / "run-meta.yaml").exists()

View File

@ -4,6 +4,9 @@ from job_research.models import ApecListing, ApecRunMeta, ApecSnapshotMeta, List
from job_research.storage import apec_run_paths from job_research.storage import apec_run_paths
FIXED_RUN_ID = "2026-06-01T10-00-00-123456Z"
def test_apec_models_serialize_expected_listing_shape() -> None: def test_apec_models_serialize_expected_listing_shape() -> None:
listing = ApecListing( listing = ApecListing(
source="apec", source="apec",
@ -24,7 +27,7 @@ def test_apec_models_serialize_expected_listing_shape() -> None:
], ],
) )
run_meta = ApecRunMeta( run_meta = ApecRunMeta(
run_id="2026-06-01T10-00-00Z", run_id=FIXED_RUN_ID,
run_started_at="2026-06-01T10:00:00Z", run_started_at="2026-06-01T10:00:00Z",
derived_queries=["Data Engineer"], derived_queries=["Data Engineer"],
snapshots=[ snapshots=[
@ -44,7 +47,7 @@ def test_apec_models_serialize_expected_listing_shape() -> None:
assert listing.model_dump()["source"] == "apec" assert listing.model_dump()["source"] == "apec"
assert listing.model_dump()["warnings"][0]["field"] == "location" assert listing.model_dump()["warnings"][0]["field"] == "location"
assert run_meta.model_dump()["run_id"] == "2026-06-01T10-00-00Z" assert run_meta.model_dump()["run_id"] == FIXED_RUN_ID
assert run_meta.model_dump()["run_started_at"] == "2026-06-01T10:00:00Z" assert run_meta.model_dump()["run_started_at"] == "2026-06-01T10:00:00Z"
assert run_meta.model_dump()["derived_queries"] == ["Data Engineer"] assert run_meta.model_dump()["derived_queries"] == ["Data Engineer"]
assert run_meta.model_dump(mode="json")["snapshots"] == [ assert run_meta.model_dump(mode="json")["snapshots"] == [
@ -58,8 +61,8 @@ def test_apec_models_serialize_expected_listing_shape() -> None:
def test_apec_run_paths_builds_expected_layout(tmp_path: Path) -> None: def test_apec_run_paths_builds_expected_layout(tmp_path: Path) -> None:
paths = apec_run_paths(tmp_path, run_id="2026-06-01T10-00-00Z") paths = apec_run_paths(tmp_path, run_id=FIXED_RUN_ID)
run_dir = tmp_path / "apec" / "runs" / "2026-06-01T10-00-00Z" run_dir = tmp_path / "apec" / "runs" / FIXED_RUN_ID
assert paths["run_dir"] == run_dir assert paths["run_dir"] == run_dir
assert paths["listings"] == run_dir / "listings.yaml" assert paths["listings"] == run_dir / "listings.yaml"
@ -68,7 +71,7 @@ def test_apec_run_paths_builds_expected_layout(tmp_path: Path) -> None:
def test_apec_run_artifacts_include_snapshot_and_meta(tmp_path: Path) -> None: def test_apec_run_artifacts_include_snapshot_and_meta(tmp_path: Path) -> None:
paths = apec_run_paths(tmp_path, run_id="2026-06-01T10-00-00Z") paths = apec_run_paths(tmp_path, run_id=FIXED_RUN_ID)
paths["snapshots"].mkdir(parents=True, exist_ok=True) paths["snapshots"].mkdir(parents=True, exist_ok=True)
snapshot = paths["snapshots"] / "job-123.html" snapshot = paths["snapshots"] / "job-123.html"