fix: harden Apec search and run ids
This commit is contained in:
parent
cdae23e303
commit
40b3619578
@ -16,6 +16,8 @@ _SEARCH_INPUT_SELECTOR = 'input[name="keywords"]'
|
|||||||
_RESULT_LINK_SELECTOR = "a[href*='/candidat/recherche-emploi.html/emploi/detail-offre/']"
|
_RESULT_LINK_SELECTOR = "a[href*='/candidat/recherche-emploi.html/emploi/detail-offre/']"
|
||||||
_DETAIL_JOB_ID_PATTERN = re.compile(r"/detail-offre/([^/?#]+)")
|
_DETAIL_JOB_ID_PATTERN = re.compile(r"/detail-offre/([^/?#]+)")
|
||||||
_APEC_HOSTS = {"apec.fr", "www.apec.fr"}
|
_APEC_HOSTS = {"apec.fr", "www.apec.fr"}
|
||||||
|
_MAX_PAGES_PER_QUERY = 50
|
||||||
|
_MAX_CONSECUTIVE_NO_PROGRESS_PAGES = 10
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
@dataclass(slots=True)
|
||||||
@ -84,6 +86,16 @@ def _accept_cookies_if_present(page) -> None:
|
|||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
|
def _goto_and_wait(page, url: str) -> bool:
|
||||||
|
try:
|
||||||
|
page.goto(url, wait_until="domcontentloaded")
|
||||||
|
page.wait_for_load_state("domcontentloaded")
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def _is_public_apec_detail_url(url: str) -> bool:
|
def _is_public_apec_detail_url(url: str) -> bool:
|
||||||
parsed_url = urlparse(url)
|
parsed_url = urlparse(url)
|
||||||
return (
|
return (
|
||||||
@ -111,9 +123,10 @@ class ApecAdapter:
|
|||||||
if len(results) >= self.max_listings:
|
if len(results) >= self.max_listings:
|
||||||
break
|
break
|
||||||
|
|
||||||
page.goto(_search_url(query, search_filters), wait_until="domcontentloaded")
|
if not _goto_and_wait(page, _search_url(query, search_filters)):
|
||||||
|
continue
|
||||||
|
|
||||||
_accept_cookies_if_present(page)
|
_accept_cookies_if_present(page)
|
||||||
page.wait_for_load_state("domcontentloaded")
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
page.wait_for_selector(_SEARCH_INPUT_SELECTOR, timeout=5_000)
|
page.wait_for_selector(_SEARCH_INPUT_SELECTOR, timeout=5_000)
|
||||||
@ -129,20 +142,24 @@ class ApecAdapter:
|
|||||||
|
|
||||||
result_page_url = page.url
|
result_page_url = page.url
|
||||||
seen_page_urls: set[str] = {result_page_url}
|
seen_page_urls: set[str] = {result_page_url}
|
||||||
page_number = 0
|
no_progress_pages = 0
|
||||||
|
|
||||||
|
for page_number in range(_MAX_PAGES_PER_QUERY):
|
||||||
|
if len(results) >= self.max_listings:
|
||||||
|
break
|
||||||
|
|
||||||
while len(results) < self.max_listings:
|
|
||||||
if page_number > 0:
|
if page_number > 0:
|
||||||
next_page_url = _search_results_url(result_page_url, page_number)
|
next_page_url = _search_results_url(result_page_url, page_number)
|
||||||
if next_page_url in seen_page_urls:
|
if next_page_url in seen_page_urls:
|
||||||
break
|
break
|
||||||
|
|
||||||
page.goto(next_page_url, wait_until="domcontentloaded")
|
if not _goto_and_wait(page, next_page_url):
|
||||||
|
break
|
||||||
|
|
||||||
try:
|
try:
|
||||||
page.wait_for_selector(_RESULT_LINK_SELECTOR, timeout=5_000)
|
page.wait_for_selector(_RESULT_LINK_SELECTOR, timeout=5_000)
|
||||||
except PlaywrightTimeoutError:
|
except PlaywrightTimeoutError:
|
||||||
break
|
break
|
||||||
|
|
||||||
current_page_url = page.url
|
current_page_url = page.url
|
||||||
if page_number > 0 and current_page_url in seen_page_urls:
|
if page_number > 0 and current_page_url in seen_page_urls:
|
||||||
@ -154,8 +171,13 @@ class ApecAdapter:
|
|||||||
"nodes => nodes.map(node => node.href)"
|
"nodes => nodes.map(node => node.href)"
|
||||||
)
|
)
|
||||||
if not hrefs:
|
if not hrefs:
|
||||||
break
|
no_progress_pages += 1
|
||||||
|
if no_progress_pages >= _MAX_CONSECUTIVE_NO_PROGRESS_PAGES:
|
||||||
|
break
|
||||||
|
|
||||||
|
continue
|
||||||
|
|
||||||
|
added_any_result = False
|
||||||
for href in hrefs:
|
for href in hrefs:
|
||||||
source_job_id = _extract_source_job_id(href)
|
source_job_id = _extract_source_job_id(href)
|
||||||
dedupe_key = source_job_id or href
|
dedupe_key = source_job_id or href
|
||||||
@ -164,11 +186,17 @@ class ApecAdapter:
|
|||||||
|
|
||||||
seen_keys.add(dedupe_key)
|
seen_keys.add(dedupe_key)
|
||||||
results.append(ApecSearchResult(url=href, source_job_id=source_job_id))
|
results.append(ApecSearchResult(url=href, source_job_id=source_job_id))
|
||||||
|
added_any_result = True
|
||||||
|
|
||||||
if len(results) >= self.max_listings:
|
if len(results) >= self.max_listings:
|
||||||
break
|
break
|
||||||
|
|
||||||
page_number += 1
|
if added_any_result:
|
||||||
|
no_progress_pages = 0
|
||||||
|
else:
|
||||||
|
no_progress_pages += 1
|
||||||
|
if no_progress_pages >= _MAX_CONSECUTIVE_NO_PROGRESS_PAGES:
|
||||||
|
break
|
||||||
|
|
||||||
if not usable_search_page_seen:
|
if not usable_search_page_seen:
|
||||||
raise ApecSearchError("Apec search page was not reachable for any query")
|
raise ApecSearchError("Apec search page was not reachable for any query")
|
||||||
|
|||||||
@ -134,9 +134,9 @@ def fetch_apec(
|
|||||||
contract_type=derived_search_filters.contract_type or "CDI",
|
contract_type=derived_search_filters.contract_type or "CDI",
|
||||||
)
|
)
|
||||||
|
|
||||||
current = _utc_now().astimezone(timezone.utc).replace(microsecond=0)
|
current = _utc_now().astimezone(timezone.utc)
|
||||||
run_id = current.strftime("%Y-%m-%dT%H-%M-%SZ")
|
run_id = current.strftime("%Y-%m-%dT%H-%M-%S-%fZ")
|
||||||
run_started_at = current.strftime("%Y-%m-%dT%H:%M:%SZ")
|
run_started_at = current.replace(microsecond=0).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
fetched_at = run_started_at
|
fetched_at = run_started_at
|
||||||
|
|
||||||
adapter = ApecAdapter(max_listings=50)
|
adapter = ApecAdapter(max_listings=50)
|
||||||
|
|||||||
@ -163,6 +163,112 @@ def test_search_continues_past_duplicate_only_pages_until_a_later_hit(monkeypatc
|
|||||||
assert any("page=3" in url for url in page.goto_urls)
|
assert any("page=3" in url for url in page.goto_urls)
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_continues_after_query_and_pagination_navigation_failures(monkeypatch) -> None:
|
||||||
|
first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=beta&page=0&selectedIndex=0"
|
||||||
|
second_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=beta&page=1&selectedIndex=0"
|
||||||
|
page = _FakeDetailPage(
|
||||||
|
{
|
||||||
|
"beta": {0: [first_result], 1: [second_result]},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
original_goto = page.goto
|
||||||
|
goto_calls = 0
|
||||||
|
|
||||||
|
def flaky_goto(url: str, wait_until: str | None = None) -> None:
|
||||||
|
nonlocal goto_calls
|
||||||
|
|
||||||
|
goto_calls += 1
|
||||||
|
if goto_calls == 1:
|
||||||
|
raise RuntimeError("navigation boom")
|
||||||
|
|
||||||
|
original_goto(url, wait_until=wait_until)
|
||||||
|
|
||||||
|
original_wait_for_load_state = page.wait_for_load_state
|
||||||
|
|
||||||
|
def flaky_wait_for_load_state(state: str) -> None:
|
||||||
|
if page.current_page == 1:
|
||||||
|
raise RuntimeError("load boom")
|
||||||
|
|
||||||
|
original_wait_for_load_state(state)
|
||||||
|
|
||||||
|
monkeypatch.setattr(page, "goto", flaky_goto)
|
||||||
|
monkeypatch.setattr(page, "wait_for_load_state", flaky_wait_for_load_state)
|
||||||
|
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
||||||
|
|
||||||
|
results = ApecAdapter(max_listings=10).search(
|
||||||
|
["alpha", "beta"],
|
||||||
|
search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert [result.url for result in results] == [first_result]
|
||||||
|
assert [result.source_job_id for result in results] == ["111"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_stops_after_max_page_count(monkeypatch) -> None:
|
||||||
|
page = _FakeDetailPage(
|
||||||
|
{
|
||||||
|
"alpha": {
|
||||||
|
0: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"],
|
||||||
|
1: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=alpha&page=1&selectedIndex=0"],
|
||||||
|
2: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/333?motsCles=alpha&page=2&selectedIndex=0"],
|
||||||
|
3: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/444?motsCles=alpha&page=3&selectedIndex=0"],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
original_goto = page.goto
|
||||||
|
|
||||||
|
def bounded_goto(url: str, wait_until: str | None = None) -> None:
|
||||||
|
original_goto(url, wait_until=wait_until)
|
||||||
|
if page.current_page >= 3:
|
||||||
|
raise AssertionError("pagination should stop before page 3")
|
||||||
|
|
||||||
|
monkeypatch.setattr(page, "goto", bounded_goto)
|
||||||
|
monkeypatch.setattr(adapter_module, "_MAX_PAGES_PER_QUERY", 3)
|
||||||
|
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
||||||
|
|
||||||
|
results = ApecAdapter(max_listings=10).search(
|
||||||
|
["alpha"],
|
||||||
|
search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert [result.source_job_id for result in results] == ["111", "222", "333"]
|
||||||
|
assert not any("page=3" in url for url in page.goto_urls)
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_stops_after_consecutive_no_progress_pages(monkeypatch) -> None:
|
||||||
|
page = _FakeDetailPage(
|
||||||
|
{
|
||||||
|
"alpha": {
|
||||||
|
0: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"],
|
||||||
|
1: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=1&selectedIndex=0"],
|
||||||
|
2: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=2&selectedIndex=0"],
|
||||||
|
3: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=alpha&page=3&selectedIndex=0"],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
original_goto = page.goto
|
||||||
|
|
||||||
|
def bounded_goto(url: str, wait_until: str | None = None) -> None:
|
||||||
|
original_goto(url, wait_until=wait_until)
|
||||||
|
if page.current_page >= 3:
|
||||||
|
raise AssertionError("pagination should stop before page 3")
|
||||||
|
|
||||||
|
monkeypatch.setattr(page, "goto", bounded_goto)
|
||||||
|
monkeypatch.setattr(adapter_module, "_MAX_CONSECUTIVE_NO_PROGRESS_PAGES", 2)
|
||||||
|
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
||||||
|
|
||||||
|
results = ApecAdapter(max_listings=10).search(
|
||||||
|
["alpha"],
|
||||||
|
search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert [result.source_job_id for result in results] == ["111"]
|
||||||
|
assert not any("page=3" in url for url in page.goto_urls)
|
||||||
|
|
||||||
|
|
||||||
def test_search_stops_when_result_page_url_repeats(monkeypatch) -> None:
|
def test_search_stops_when_result_page_url_repeats(monkeypatch) -> None:
|
||||||
first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"
|
first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"
|
||||||
page = _FakeDetailPage(
|
page = _FakeDetailPage(
|
||||||
|
|||||||
@ -12,7 +12,10 @@ from job_research.cli import app
|
|||||||
|
|
||||||
|
|
||||||
def _fixed_now() -> datetime:
|
def _fixed_now() -> datetime:
|
||||||
return datetime(2026, 6, 1, 10, 0, 0, tzinfo=timezone.utc)
|
return datetime(2026, 6, 1, 10, 0, 0, 123456, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
FIXED_RUN_ID = _fixed_now().strftime("%Y-%m-%dT%H-%M-%S-%fZ")
|
||||||
|
|
||||||
|
|
||||||
def _apec_detail_html(
|
def _apec_detail_html(
|
||||||
@ -271,7 +274,7 @@ def test_fetch_apec_reports_snapshot_directory_creation_failures_cleanly(
|
|||||||
assert adapter.search_calls == [["Role From YAML"]]
|
assert adapter.search_calls == [["Role From YAML"]]
|
||||||
assert adapter.fetch_calls == []
|
assert adapter.fetch_calls == []
|
||||||
|
|
||||||
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
|
||||||
assert not (run_dir / "snapshots").exists()
|
assert not (run_dir / "snapshots").exists()
|
||||||
|
|
||||||
|
|
||||||
@ -320,7 +323,7 @@ def test_fetch_apec_succeeds_with_empty_search_results(tmp_path, monkeypatch) ->
|
|||||||
assert adapter.search_calls == [["Role From YAML"]]
|
assert adapter.search_calls == [["Role From YAML"]]
|
||||||
assert adapter.fetch_calls == []
|
assert adapter.fetch_calls == []
|
||||||
|
|
||||||
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
|
||||||
assert (run_dir / "snapshots").exists()
|
assert (run_dir / "snapshots").exists()
|
||||||
assert list((run_dir / "snapshots").iterdir()) == []
|
assert list((run_dir / "snapshots").iterdir()) == []
|
||||||
|
|
||||||
@ -329,7 +332,7 @@ def test_fetch_apec_succeeds_with_empty_search_results(tmp_path, monkeypatch) ->
|
|||||||
|
|
||||||
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
||||||
assert run_meta_payload == {
|
assert run_meta_payload == {
|
||||||
"run_id": "2026-06-01T10-00-00Z",
|
"run_id": FIXED_RUN_ID,
|
||||||
"run_started_at": "2026-06-01T10:00:00Z",
|
"run_started_at": "2026-06-01T10:00:00Z",
|
||||||
"derived_queries": ["Role From YAML"],
|
"derived_queries": ["Role From YAML"],
|
||||||
"snapshots": [],
|
"snapshots": [],
|
||||||
@ -341,6 +344,62 @@ def test_fetch_apec_succeeds_with_empty_search_results(tmp_path, monkeypatch) ->
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_apec_uses_unique_run_dirs_for_runs_started_within_same_second(
|
||||||
|
tmp_path,
|
||||||
|
monkeypatch,
|
||||||
|
) -> None:
|
||||||
|
data_root = tmp_path / "data"
|
||||||
|
data_root.mkdir()
|
||||||
|
(data_root / "candidate-profile.yaml").write_text(
|
||||||
|
dedent(
|
||||||
|
"""
|
||||||
|
target_roles:
|
||||||
|
- Role From YAML
|
||||||
|
"""
|
||||||
|
).strip(),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
class EmptyResultAdapter:
|
||||||
|
instances: list["EmptyResultAdapter"] = []
|
||||||
|
|
||||||
|
def __init__(self, max_listings: int = 50) -> None:
|
||||||
|
self.max_listings = max_listings
|
||||||
|
self.search_calls: list[list[str]] = []
|
||||||
|
self.fetch_calls: list[str] = []
|
||||||
|
EmptyResultAdapter.instances.append(self)
|
||||||
|
|
||||||
|
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||||
|
self.search_calls.append(list(queries))
|
||||||
|
return []
|
||||||
|
|
||||||
|
def fetch_listing_html(self, url: str) -> str:
|
||||||
|
self.fetch_calls.append(url)
|
||||||
|
raise AssertionError("fetch should not be called when search returns no results")
|
||||||
|
|
||||||
|
timestamps = [
|
||||||
|
datetime(2026, 6, 1, 10, 0, 0, 111111, tzinfo=timezone.utc),
|
||||||
|
datetime(2026, 6, 1, 10, 0, 0, 222222, tzinfo=timezone.utc),
|
||||||
|
]
|
||||||
|
expected_run_ids = [timestamp.strftime("%Y-%m-%dT%H-%M-%S-%fZ") for timestamp in timestamps]
|
||||||
|
|
||||||
|
def next_now() -> datetime:
|
||||||
|
return timestamps.pop(0)
|
||||||
|
|
||||||
|
monkeypatch.setattr("job_research.cli.ApecAdapter", EmptyResultAdapter)
|
||||||
|
monkeypatch.setattr("job_research.cli._utc_now", next_now)
|
||||||
|
|
||||||
|
runner = CliRunner()
|
||||||
|
first = runner.invoke(app, ["fetch-apec", "--data-root", str(data_root)])
|
||||||
|
second = runner.invoke(app, ["fetch-apec", "--data-root", str(data_root)])
|
||||||
|
|
||||||
|
assert first.exit_code == 0
|
||||||
|
assert second.exit_code == 0
|
||||||
|
|
||||||
|
runs_root = data_root / "apec" / "runs"
|
||||||
|
assert sorted(path.name for path in runs_root.iterdir()) == expected_run_ids
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("failing_filename", ["listings.yaml", "run-meta.yaml"])
|
@pytest.mark.parametrize("failing_filename", ["listings.yaml", "run-meta.yaml"])
|
||||||
def test_fetch_apec_reports_final_artifact_write_failures_cleanly(
|
def test_fetch_apec_reports_final_artifact_write_failures_cleanly(
|
||||||
tmp_path,
|
tmp_path,
|
||||||
@ -406,7 +465,7 @@ def test_fetch_apec_reports_final_artifact_write_failures_cleanly(
|
|||||||
assert adapter.search_calls == [["Role From YAML"]]
|
assert adapter.search_calls == [["Role From YAML"]]
|
||||||
assert adapter.fetch_calls == ["https://example.test/job/123"]
|
assert adapter.fetch_calls == ["https://example.test/job/123"]
|
||||||
|
|
||||||
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
|
||||||
if failing_filename == "listings.yaml":
|
if failing_filename == "listings.yaml":
|
||||||
assert not (run_dir / "listings.yaml").exists()
|
assert not (run_dir / "listings.yaml").exists()
|
||||||
assert (run_dir / "run-meta.yaml").exists()
|
assert (run_dir / "run-meta.yaml").exists()
|
||||||
@ -499,7 +558,7 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch
|
|||||||
|
|
||||||
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
||||||
assert run_meta_payload == {
|
assert run_meta_payload == {
|
||||||
"run_id": "2026-06-01T10-00-00Z",
|
"run_id": FIXED_RUN_ID,
|
||||||
"run_started_at": "2026-06-01T10:00:00Z",
|
"run_started_at": "2026-06-01T10:00:00Z",
|
||||||
"derived_queries": ["Role From YAML"],
|
"derived_queries": ["Role From YAML"],
|
||||||
"snapshots": [
|
"snapshots": [
|
||||||
@ -572,7 +631,7 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings(
|
|||||||
assert adapter.search_calls == [["Role From YAML"]]
|
assert adapter.search_calls == [["Role From YAML"]]
|
||||||
assert adapter.fetch_calls == [good_result.url, bad_result.url]
|
assert adapter.fetch_calls == [good_result.url, bad_result.url]
|
||||||
|
|
||||||
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
|
||||||
snapshot_files = list((run_dir / "snapshots").glob("*.html"))
|
snapshot_files = list((run_dir / "snapshots").glob("*.html"))
|
||||||
assert [snapshot.name for snapshot in snapshot_files] == ["job-123.html"]
|
assert [snapshot.name for snapshot in snapshot_files] == ["job-123.html"]
|
||||||
assert snapshot_files[0].read_text(encoding="utf-8") == html
|
assert snapshot_files[0].read_text(encoding="utf-8") == html
|
||||||
@ -596,7 +655,7 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings(
|
|||||||
|
|
||||||
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
||||||
assert run_meta_payload == {
|
assert run_meta_payload == {
|
||||||
"run_id": "2026-06-01T10-00-00Z",
|
"run_id": FIXED_RUN_ID,
|
||||||
"run_started_at": "2026-06-01T10:00:00Z",
|
"run_started_at": "2026-06-01T10:00:00Z",
|
||||||
"derived_queries": ["Role From YAML"],
|
"derived_queries": ["Role From YAML"],
|
||||||
"snapshots": [
|
"snapshots": [
|
||||||
@ -679,7 +738,7 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_li
|
|||||||
assert "deduplicated=2" in result.stdout
|
assert "deduplicated=2" in result.stdout
|
||||||
assert "failed=1" in result.stdout
|
assert "failed=1" in result.stdout
|
||||||
|
|
||||||
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
|
||||||
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
|
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
|
||||||
assert [snapshot.name for snapshot in snapshot_files] == ["job-456.html"]
|
assert [snapshot.name for snapshot in snapshot_files] == ["job-456.html"]
|
||||||
|
|
||||||
@ -715,7 +774,7 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_li
|
|||||||
|
|
||||||
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
||||||
assert run_meta_payload == {
|
assert run_meta_payload == {
|
||||||
"run_id": "2026-06-01T10-00-00Z",
|
"run_id": FIXED_RUN_ID,
|
||||||
"run_started_at": "2026-06-01T10:00:00Z",
|
"run_started_at": "2026-06-01T10:00:00Z",
|
||||||
"derived_queries": ["Role From YAML"],
|
"derived_queries": ["Role From YAML"],
|
||||||
"snapshots": [
|
"snapshots": [
|
||||||
@ -799,7 +858,7 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails(
|
|||||||
assert "deduplicated=2" in result.stdout
|
assert "deduplicated=2" in result.stdout
|
||||||
assert "failed=2" in result.stdout
|
assert "failed=2" in result.stdout
|
||||||
|
|
||||||
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
|
||||||
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
|
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
|
||||||
assert snapshot_files == []
|
assert snapshot_files == []
|
||||||
|
|
||||||
@ -835,7 +894,7 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails(
|
|||||||
|
|
||||||
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
||||||
assert run_meta_payload == {
|
assert run_meta_payload == {
|
||||||
"run_id": "2026-06-01T10-00-00Z",
|
"run_id": FIXED_RUN_ID,
|
||||||
"run_started_at": "2026-06-01T10:00:00Z",
|
"run_started_at": "2026-06-01T10:00:00Z",
|
||||||
"derived_queries": ["Role From YAML"],
|
"derived_queries": ["Role From YAML"],
|
||||||
"snapshots": [
|
"snapshots": [
|
||||||
@ -931,14 +990,14 @@ def test_fetch_apec_counts_each_failed_url_once_even_when_multiple_errors_are_re
|
|||||||
assert "deduplicated=1" in result.stdout
|
assert "deduplicated=1" in result.stdout
|
||||||
assert "failed=1" in result.stdout
|
assert "failed=1" in result.stdout
|
||||||
|
|
||||||
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
|
||||||
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
|
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
|
||||||
assert [snapshot.name for snapshot in snapshot_files] == ["job-456.html"]
|
assert [snapshot.name for snapshot in snapshot_files] == ["job-456.html"]
|
||||||
assert snapshot_files[0].read_text(encoding="utf-8") == html
|
assert snapshot_files[0].read_text(encoding="utf-8") == html
|
||||||
|
|
||||||
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
||||||
assert run_meta_payload == {
|
assert run_meta_payload == {
|
||||||
"run_id": "2026-06-01T10-00-00Z",
|
"run_id": FIXED_RUN_ID,
|
||||||
"run_started_at": "2026-06-01T10:00:00Z",
|
"run_started_at": "2026-06-01T10:00:00Z",
|
||||||
"derived_queries": ["Role From YAML"],
|
"derived_queries": ["Role From YAML"],
|
||||||
"snapshots": [
|
"snapshots": [
|
||||||
@ -1015,7 +1074,7 @@ def test_fetch_apec_preserves_snapshot_and_run_meta_when_normalization_fails(
|
|||||||
assert "deduplicated=0" in result.stdout
|
assert "deduplicated=0" in result.stdout
|
||||||
assert "failed=1" in result.stdout
|
assert "failed=1" in result.stdout
|
||||||
|
|
||||||
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
|
||||||
snapshot_file = run_dir / "snapshots" / "job-123.html"
|
snapshot_file = run_dir / "snapshots" / "job-123.html"
|
||||||
assert snapshot_file.exists()
|
assert snapshot_file.exists()
|
||||||
assert snapshot_file.read_text(encoding="utf-8") == html
|
assert snapshot_file.read_text(encoding="utf-8") == html
|
||||||
@ -1025,7 +1084,7 @@ def test_fetch_apec_preserves_snapshot_and_run_meta_when_normalization_fails(
|
|||||||
|
|
||||||
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
||||||
assert run_meta_payload == {
|
assert run_meta_payload == {
|
||||||
"run_id": "2026-06-01T10-00-00Z",
|
"run_id": FIXED_RUN_ID,
|
||||||
"run_started_at": "2026-06-01T10:00:00Z",
|
"run_started_at": "2026-06-01T10:00:00Z",
|
||||||
"derived_queries": ["Role From YAML"],
|
"derived_queries": ["Role From YAML"],
|
||||||
"snapshots": [
|
"snapshots": [
|
||||||
@ -1097,7 +1156,7 @@ def test_fetch_apec_writes_artifacts_when_every_normalization_attempt_fails(
|
|||||||
assert "deduplicated=0" in result.stdout
|
assert "deduplicated=0" in result.stdout
|
||||||
assert "failed=2" in result.stdout
|
assert "failed=2" in result.stdout
|
||||||
|
|
||||||
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
|
||||||
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
|
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
|
||||||
assert [snapshot.name for snapshot in snapshot_files] == ["job-123.html", "job-456.html"]
|
assert [snapshot.name for snapshot in snapshot_files] == ["job-123.html", "job-456.html"]
|
||||||
assert [snapshot.read_text(encoding="utf-8") for snapshot in snapshot_files] == [html, html]
|
assert [snapshot.read_text(encoding="utf-8") for snapshot in snapshot_files] == [html, html]
|
||||||
@ -1107,7 +1166,7 @@ def test_fetch_apec_writes_artifacts_when_every_normalization_attempt_fails(
|
|||||||
|
|
||||||
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
||||||
assert run_meta_payload == {
|
assert run_meta_payload == {
|
||||||
"run_id": "2026-06-01T10-00-00Z",
|
"run_id": FIXED_RUN_ID,
|
||||||
"run_started_at": "2026-06-01T10:00:00Z",
|
"run_started_at": "2026-06-01T10:00:00Z",
|
||||||
"derived_queries": ["Role From YAML"],
|
"derived_queries": ["Role From YAML"],
|
||||||
"snapshots": [
|
"snapshots": [
|
||||||
@ -1214,7 +1273,7 @@ def test_fetch_apec_processes_only_the_first_fifty_search_results(tmp_path, monk
|
|||||||
assert adapter.search_calls == [["Role From YAML"]]
|
assert adapter.search_calls == [["Role From YAML"]]
|
||||||
assert adapter.fetch_calls == [result.url for result in results[:50]]
|
assert adapter.fetch_calls == [result.url for result in results[:50]]
|
||||||
|
|
||||||
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
|
||||||
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
|
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
|
||||||
assert len(snapshot_files) == 50
|
assert len(snapshot_files) == 50
|
||||||
|
|
||||||
@ -1271,6 +1330,6 @@ def test_fetch_apec_fails_when_every_listing_attempt_fails(tmp_path, monkeypatch
|
|||||||
assert adapter.search_calls == [["Role From YAML"]]
|
assert adapter.search_calls == [["Role From YAML"]]
|
||||||
assert adapter.fetch_calls == [result_item.url for result_item in results]
|
assert adapter.fetch_calls == [result_item.url for result_item in results]
|
||||||
|
|
||||||
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID
|
||||||
assert not (run_dir / "listings.yaml").exists()
|
assert not (run_dir / "listings.yaml").exists()
|
||||||
assert not (run_dir / "run-meta.yaml").exists()
|
assert not (run_dir / "run-meta.yaml").exists()
|
||||||
|
|||||||
@ -4,6 +4,9 @@ from job_research.models import ApecListing, ApecRunMeta, ApecSnapshotMeta, List
|
|||||||
from job_research.storage import apec_run_paths
|
from job_research.storage import apec_run_paths
|
||||||
|
|
||||||
|
|
||||||
|
FIXED_RUN_ID = "2026-06-01T10-00-00-123456Z"
|
||||||
|
|
||||||
|
|
||||||
def test_apec_models_serialize_expected_listing_shape() -> None:
|
def test_apec_models_serialize_expected_listing_shape() -> None:
|
||||||
listing = ApecListing(
|
listing = ApecListing(
|
||||||
source="apec",
|
source="apec",
|
||||||
@ -24,7 +27,7 @@ def test_apec_models_serialize_expected_listing_shape() -> None:
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
run_meta = ApecRunMeta(
|
run_meta = ApecRunMeta(
|
||||||
run_id="2026-06-01T10-00-00Z",
|
run_id=FIXED_RUN_ID,
|
||||||
run_started_at="2026-06-01T10:00:00Z",
|
run_started_at="2026-06-01T10:00:00Z",
|
||||||
derived_queries=["Data Engineer"],
|
derived_queries=["Data Engineer"],
|
||||||
snapshots=[
|
snapshots=[
|
||||||
@ -44,7 +47,7 @@ def test_apec_models_serialize_expected_listing_shape() -> None:
|
|||||||
|
|
||||||
assert listing.model_dump()["source"] == "apec"
|
assert listing.model_dump()["source"] == "apec"
|
||||||
assert listing.model_dump()["warnings"][0]["field"] == "location"
|
assert listing.model_dump()["warnings"][0]["field"] == "location"
|
||||||
assert run_meta.model_dump()["run_id"] == "2026-06-01T10-00-00Z"
|
assert run_meta.model_dump()["run_id"] == FIXED_RUN_ID
|
||||||
assert run_meta.model_dump()["run_started_at"] == "2026-06-01T10:00:00Z"
|
assert run_meta.model_dump()["run_started_at"] == "2026-06-01T10:00:00Z"
|
||||||
assert run_meta.model_dump()["derived_queries"] == ["Data Engineer"]
|
assert run_meta.model_dump()["derived_queries"] == ["Data Engineer"]
|
||||||
assert run_meta.model_dump(mode="json")["snapshots"] == [
|
assert run_meta.model_dump(mode="json")["snapshots"] == [
|
||||||
@ -58,8 +61,8 @@ def test_apec_models_serialize_expected_listing_shape() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_apec_run_paths_builds_expected_layout(tmp_path: Path) -> None:
|
def test_apec_run_paths_builds_expected_layout(tmp_path: Path) -> None:
|
||||||
paths = apec_run_paths(tmp_path, run_id="2026-06-01T10-00-00Z")
|
paths = apec_run_paths(tmp_path, run_id=FIXED_RUN_ID)
|
||||||
run_dir = tmp_path / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
run_dir = tmp_path / "apec" / "runs" / FIXED_RUN_ID
|
||||||
|
|
||||||
assert paths["run_dir"] == run_dir
|
assert paths["run_dir"] == run_dir
|
||||||
assert paths["listings"] == run_dir / "listings.yaml"
|
assert paths["listings"] == run_dir / "listings.yaml"
|
||||||
@ -68,7 +71,7 @@ def test_apec_run_paths_builds_expected_layout(tmp_path: Path) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_apec_run_artifacts_include_snapshot_and_meta(tmp_path: Path) -> None:
|
def test_apec_run_artifacts_include_snapshot_and_meta(tmp_path: Path) -> None:
|
||||||
paths = apec_run_paths(tmp_path, run_id="2026-06-01T10-00-00Z")
|
paths = apec_run_paths(tmp_path, run_id=FIXED_RUN_ID)
|
||||||
paths["snapshots"].mkdir(parents=True, exist_ok=True)
|
paths["snapshots"].mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
snapshot = paths["snapshots"] / "job-123.html"
|
snapshot = paths["snapshots"] / "job-123.html"
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user