diff --git a/src/job_research/apec/adapter.py b/src/job_research/apec/adapter.py index bf72f78..b0a37ba 100644 --- a/src/job_research/apec/adapter.py +++ b/src/job_research/apec/adapter.py @@ -16,6 +16,8 @@ _SEARCH_INPUT_SELECTOR = 'input[name="keywords"]' _RESULT_LINK_SELECTOR = "a[href*='/candidat/recherche-emploi.html/emploi/detail-offre/']" _DETAIL_JOB_ID_PATTERN = re.compile(r"/detail-offre/([^/?#]+)") _APEC_HOSTS = {"apec.fr", "www.apec.fr"} +_MAX_PAGES_PER_QUERY = 50 +_MAX_CONSECUTIVE_NO_PROGRESS_PAGES = 10 @dataclass(slots=True) @@ -84,6 +86,16 @@ def _accept_cookies_if_present(page) -> None: return +def _goto_and_wait(page, url: str) -> bool: + try: + page.goto(url, wait_until="domcontentloaded") + page.wait_for_load_state("domcontentloaded") + except Exception: + return False + + return True + + def _is_public_apec_detail_url(url: str) -> bool: parsed_url = urlparse(url) return ( @@ -111,9 +123,10 @@ class ApecAdapter: if len(results) >= self.max_listings: break - page.goto(_search_url(query, search_filters), wait_until="domcontentloaded") + if not _goto_and_wait(page, _search_url(query, search_filters)): + continue + _accept_cookies_if_present(page) - page.wait_for_load_state("domcontentloaded") try: page.wait_for_selector(_SEARCH_INPUT_SELECTOR, timeout=5_000) @@ -129,20 +142,24 @@ class ApecAdapter: result_page_url = page.url seen_page_urls: set[str] = {result_page_url} - page_number = 0 + no_progress_pages = 0 + + for page_number in range(_MAX_PAGES_PER_QUERY): + if len(results) >= self.max_listings: + break - while len(results) < self.max_listings: if page_number > 0: next_page_url = _search_results_url(result_page_url, page_number) if next_page_url in seen_page_urls: break - page.goto(next_page_url, wait_until="domcontentloaded") + if not _goto_and_wait(page, next_page_url): + break - try: - page.wait_for_selector(_RESULT_LINK_SELECTOR, timeout=5_000) - except PlaywrightTimeoutError: - break + try: + page.wait_for_selector(_RESULT_LINK_SELECTOR, timeout=5_000) + except PlaywrightTimeoutError: + break current_page_url = page.url if page_number > 0 and current_page_url in seen_page_urls: @@ -154,8 +171,13 @@ class ApecAdapter: "nodes => nodes.map(node => node.href)" ) if not hrefs: - break + no_progress_pages += 1 + if no_progress_pages >= _MAX_CONSECUTIVE_NO_PROGRESS_PAGES: + break + continue + + added_any_result = False for href in hrefs: source_job_id = _extract_source_job_id(href) dedupe_key = source_job_id or href @@ -164,11 +186,17 @@ class ApecAdapter: seen_keys.add(dedupe_key) results.append(ApecSearchResult(url=href, source_job_id=source_job_id)) + added_any_result = True if len(results) >= self.max_listings: break - page_number += 1 + if added_any_result: + no_progress_pages = 0 + else: + no_progress_pages += 1 + if no_progress_pages >= _MAX_CONSECUTIVE_NO_PROGRESS_PAGES: + break if not usable_search_page_seen: raise ApecSearchError("Apec search page was not reachable for any query") diff --git a/src/job_research/cli.py b/src/job_research/cli.py index 9899e92..f6114b7 100644 --- a/src/job_research/cli.py +++ b/src/job_research/cli.py @@ -134,9 +134,9 @@ def fetch_apec( contract_type=derived_search_filters.contract_type or "CDI", ) - current = _utc_now().astimezone(timezone.utc).replace(microsecond=0) - run_id = current.strftime("%Y-%m-%dT%H-%M-%SZ") - run_started_at = current.strftime("%Y-%m-%dT%H:%M:%SZ") + current = _utc_now().astimezone(timezone.utc) + run_id = current.strftime("%Y-%m-%dT%H-%M-%S-%fZ") + run_started_at = current.replace(microsecond=0).strftime("%Y-%m-%dT%H:%M:%SZ") fetched_at = run_started_at adapter = ApecAdapter(max_listings=50) diff --git a/tests/apec/test_adapter.py b/tests/apec/test_adapter.py index ee5152d..e0654c0 100644 --- a/tests/apec/test_adapter.py +++ b/tests/apec/test_adapter.py @@ -163,6 +163,112 @@ def test_search_continues_past_duplicate_only_pages_until_a_later_hit(monkeypatc assert any("page=3" in url for url in page.goto_urls) +def test_search_continues_after_query_and_pagination_navigation_failures(monkeypatch) -> None: + first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=beta&page=0&selectedIndex=0" + second_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=beta&page=1&selectedIndex=0" + page = _FakeDetailPage( + { + "beta": {0: [first_result], 1: [second_result]}, + } + ) + + original_goto = page.goto + goto_calls = 0 + + def flaky_goto(url: str, wait_until: str | None = None) -> None: + nonlocal goto_calls + + goto_calls += 1 + if goto_calls == 1: + raise RuntimeError("navigation boom") + + original_goto(url, wait_until=wait_until) + + original_wait_for_load_state = page.wait_for_load_state + + def flaky_wait_for_load_state(state: str) -> None: + if page.current_page == 1: + raise RuntimeError("load boom") + + original_wait_for_load_state(state) + + monkeypatch.setattr(page, "goto", flaky_goto) + monkeypatch.setattr(page, "wait_for_load_state", flaky_wait_for_load_state) + monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page)) + + results = ApecAdapter(max_listings=10).search( + ["alpha", "beta"], + search_filters=ApecSearchFilters(location="France", contract_type="CDI"), + ) + + assert [result.url for result in results] == [first_result] + assert [result.source_job_id for result in results] == ["111"] + + +def test_search_stops_after_max_page_count(monkeypatch) -> None: + page = _FakeDetailPage( + { + "alpha": { + 0: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"], + 1: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=alpha&page=1&selectedIndex=0"], + 2: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/333?motsCles=alpha&page=2&selectedIndex=0"], + 3: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/444?motsCles=alpha&page=3&selectedIndex=0"], + } + } + ) + + original_goto = page.goto + + def bounded_goto(url: str, wait_until: str | None = None) -> None: + original_goto(url, wait_until=wait_until) + if page.current_page >= 3: + raise AssertionError("pagination should stop before page 3") + + monkeypatch.setattr(page, "goto", bounded_goto) + monkeypatch.setattr(adapter_module, "_MAX_PAGES_PER_QUERY", 3) + monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page)) + + results = ApecAdapter(max_listings=10).search( + ["alpha"], + search_filters=ApecSearchFilters(location="France", contract_type="CDI"), + ) + + assert [result.source_job_id for result in results] == ["111", "222", "333"] + assert not any("page=3" in url for url in page.goto_urls) + + +def test_search_stops_after_consecutive_no_progress_pages(monkeypatch) -> None: + page = _FakeDetailPage( + { + "alpha": { + 0: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"], + 1: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=1&selectedIndex=0"], + 2: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=2&selectedIndex=0"], + 3: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=alpha&page=3&selectedIndex=0"], + } + } + ) + + original_goto = page.goto + + def bounded_goto(url: str, wait_until: str | None = None) -> None: + original_goto(url, wait_until=wait_until) + if page.current_page >= 3: + raise AssertionError("pagination should stop before page 3") + + monkeypatch.setattr(page, "goto", bounded_goto) + monkeypatch.setattr(adapter_module, "_MAX_CONSECUTIVE_NO_PROGRESS_PAGES", 2) + monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page)) + + results = ApecAdapter(max_listings=10).search( + ["alpha"], + search_filters=ApecSearchFilters(location="France", contract_type="CDI"), + ) + + assert [result.source_job_id for result in results] == ["111"] + assert not any("page=3" in url for url in page.goto_urls) + + def test_search_stops_when_result_page_url_repeats(monkeypatch) -> None: first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0" page = _FakeDetailPage( diff --git a/tests/test_apec_cli.py b/tests/test_apec_cli.py index 8e58817..ee06490 100644 --- a/tests/test_apec_cli.py +++ b/tests/test_apec_cli.py @@ -12,7 +12,10 @@ from job_research.cli import app def _fixed_now() -> datetime: - return datetime(2026, 6, 1, 10, 0, 0, tzinfo=timezone.utc) + return datetime(2026, 6, 1, 10, 0, 0, 123456, tzinfo=timezone.utc) + + +FIXED_RUN_ID = _fixed_now().strftime("%Y-%m-%dT%H-%M-%S-%fZ") def _apec_detail_html( @@ -271,7 +274,7 @@ def test_fetch_apec_reports_snapshot_directory_creation_failures_cleanly( assert adapter.search_calls == [["Role From YAML"]] assert adapter.fetch_calls == [] - run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID assert not (run_dir / "snapshots").exists() @@ -320,7 +323,7 @@ def test_fetch_apec_succeeds_with_empty_search_results(tmp_path, monkeypatch) -> assert adapter.search_calls == [["Role From YAML"]] assert adapter.fetch_calls == [] - run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID assert (run_dir / "snapshots").exists() assert list((run_dir / "snapshots").iterdir()) == [] @@ -329,7 +332,7 @@ def test_fetch_apec_succeeds_with_empty_search_results(tmp_path, monkeypatch) -> run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) assert run_meta_payload == { - "run_id": "2026-06-01T10-00-00Z", + "run_id": FIXED_RUN_ID, "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], "snapshots": [], @@ -341,6 +344,62 @@ def test_fetch_apec_succeeds_with_empty_search_results(tmp_path, monkeypatch) -> } +def test_fetch_apec_uses_unique_run_dirs_for_runs_started_within_same_second( + tmp_path, + monkeypatch, +) -> None: + data_root = tmp_path / "data" + data_root.mkdir() + (data_root / "candidate-profile.yaml").write_text( + dedent( + """ + target_roles: + - Role From YAML + """ + ).strip(), + encoding="utf-8", + ) + + class EmptyResultAdapter: + instances: list["EmptyResultAdapter"] = [] + + def __init__(self, max_listings: int = 50) -> None: + self.max_listings = max_listings + self.search_calls: list[list[str]] = [] + self.fetch_calls: list[str] = [] + EmptyResultAdapter.instances.append(self) + + def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]: + self.search_calls.append(list(queries)) + return [] + + def fetch_listing_html(self, url: str) -> str: + self.fetch_calls.append(url) + raise AssertionError("fetch should not be called when search returns no results") + + timestamps = [ + datetime(2026, 6, 1, 10, 0, 0, 111111, tzinfo=timezone.utc), + datetime(2026, 6, 1, 10, 0, 0, 222222, tzinfo=timezone.utc), + ] + expected_run_ids = [timestamp.strftime("%Y-%m-%dT%H-%M-%S-%fZ") for timestamp in timestamps] + + def next_now() -> datetime: + return timestamps.pop(0) + + monkeypatch.setattr("job_research.cli.ApecAdapter", EmptyResultAdapter) + monkeypatch.setattr("job_research.cli._utc_now", next_now) + + runner = CliRunner() + first = runner.invoke(app, ["fetch-apec", "--data-root", str(data_root)]) + second = runner.invoke(app, ["fetch-apec", "--data-root", str(data_root)]) + + assert first.exit_code == 0 + assert second.exit_code == 0 + + runs_root = data_root / "apec" / "runs" + assert sorted(path.name for path in runs_root.iterdir()) == expected_run_ids + + @pytest.mark.parametrize("failing_filename", ["listings.yaml", "run-meta.yaml"]) def test_fetch_apec_reports_final_artifact_write_failures_cleanly( tmp_path, @@ -406,7 +465,7 @@ def test_fetch_apec_reports_final_artifact_write_failures_cleanly( assert adapter.search_calls == [["Role From YAML"]] assert adapter.fetch_calls == ["https://example.test/job/123"] - run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID if failing_filename == "listings.yaml": assert not (run_dir / "listings.yaml").exists() assert (run_dir / "run-meta.yaml").exists() @@ -499,7 +558,7 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) assert run_meta_payload == { - "run_id": "2026-06-01T10-00-00Z", + "run_id": FIXED_RUN_ID, "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], "snapshots": [ @@ -572,7 +631,7 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings( assert adapter.search_calls == [["Role From YAML"]] assert adapter.fetch_calls == [good_result.url, bad_result.url] - run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID snapshot_files = list((run_dir / "snapshots").glob("*.html")) assert [snapshot.name for snapshot in snapshot_files] == ["job-123.html"] assert snapshot_files[0].read_text(encoding="utf-8") == html @@ -596,7 +655,7 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings( run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) assert run_meta_payload == { - "run_id": "2026-06-01T10-00-00Z", + "run_id": FIXED_RUN_ID, "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], "snapshots": [ @@ -679,7 +738,7 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_li assert "deduplicated=2" in result.stdout assert "failed=1" in result.stdout - run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID snapshot_files = sorted((run_dir / "snapshots").glob("*.html")) assert [snapshot.name for snapshot in snapshot_files] == ["job-456.html"] @@ -715,7 +774,7 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_li run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) assert run_meta_payload == { - "run_id": "2026-06-01T10-00-00Z", + "run_id": FIXED_RUN_ID, "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], "snapshots": [ @@ -799,7 +858,7 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails( assert "deduplicated=2" in result.stdout assert "failed=2" in result.stdout - run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID snapshot_files = sorted((run_dir / "snapshots").glob("*.html")) assert snapshot_files == [] @@ -835,7 +894,7 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails( run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) assert run_meta_payload == { - "run_id": "2026-06-01T10-00-00Z", + "run_id": FIXED_RUN_ID, "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], "snapshots": [ @@ -931,14 +990,14 @@ def test_fetch_apec_counts_each_failed_url_once_even_when_multiple_errors_are_re assert "deduplicated=1" in result.stdout assert "failed=1" in result.stdout - run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID snapshot_files = sorted((run_dir / "snapshots").glob("*.html")) assert [snapshot.name for snapshot in snapshot_files] == ["job-456.html"] assert snapshot_files[0].read_text(encoding="utf-8") == html run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) assert run_meta_payload == { - "run_id": "2026-06-01T10-00-00Z", + "run_id": FIXED_RUN_ID, "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], "snapshots": [ @@ -1015,7 +1074,7 @@ def test_fetch_apec_preserves_snapshot_and_run_meta_when_normalization_fails( assert "deduplicated=0" in result.stdout assert "failed=1" in result.stdout - run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID snapshot_file = run_dir / "snapshots" / "job-123.html" assert snapshot_file.exists() assert snapshot_file.read_text(encoding="utf-8") == html @@ -1025,7 +1084,7 @@ def test_fetch_apec_preserves_snapshot_and_run_meta_when_normalization_fails( run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) assert run_meta_payload == { - "run_id": "2026-06-01T10-00-00Z", + "run_id": FIXED_RUN_ID, "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], "snapshots": [ @@ -1097,7 +1156,7 @@ def test_fetch_apec_writes_artifacts_when_every_normalization_attempt_fails( assert "deduplicated=0" in result.stdout assert "failed=2" in result.stdout - run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID snapshot_files = sorted((run_dir / "snapshots").glob("*.html")) assert [snapshot.name for snapshot in snapshot_files] == ["job-123.html", "job-456.html"] assert [snapshot.read_text(encoding="utf-8") for snapshot in snapshot_files] == [html, html] @@ -1107,7 +1166,7 @@ def test_fetch_apec_writes_artifacts_when_every_normalization_attempt_fails( run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) assert run_meta_payload == { - "run_id": "2026-06-01T10-00-00Z", + "run_id": FIXED_RUN_ID, "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], "snapshots": [ @@ -1214,7 +1273,7 @@ def test_fetch_apec_processes_only_the_first_fifty_search_results(tmp_path, monk assert adapter.search_calls == [["Role From YAML"]] assert adapter.fetch_calls == [result.url for result in results[:50]] - run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID snapshot_files = sorted((run_dir / "snapshots").glob("*.html")) assert len(snapshot_files) == 50 @@ -1271,6 +1330,6 @@ def test_fetch_apec_fails_when_every_listing_attempt_fails(tmp_path, monkeypatch assert adapter.search_calls == [["Role From YAML"]] assert adapter.fetch_calls == [result_item.url for result_item in results] - run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + run_dir = data_root / "apec" / "runs" / FIXED_RUN_ID assert not (run_dir / "listings.yaml").exists() assert not (run_dir / "run-meta.yaml").exists() diff --git a/tests/test_apec_storage.py b/tests/test_apec_storage.py index caa77c3..801e2ad 100644 --- a/tests/test_apec_storage.py +++ b/tests/test_apec_storage.py @@ -4,6 +4,9 @@ from job_research.models import ApecListing, ApecRunMeta, ApecSnapshotMeta, List from job_research.storage import apec_run_paths +FIXED_RUN_ID = "2026-06-01T10-00-00-123456Z" + + def test_apec_models_serialize_expected_listing_shape() -> None: listing = ApecListing( source="apec", @@ -24,7 +27,7 @@ def test_apec_models_serialize_expected_listing_shape() -> None: ], ) run_meta = ApecRunMeta( - run_id="2026-06-01T10-00-00Z", + run_id=FIXED_RUN_ID, run_started_at="2026-06-01T10:00:00Z", derived_queries=["Data Engineer"], snapshots=[ @@ -44,7 +47,7 @@ def test_apec_models_serialize_expected_listing_shape() -> None: assert listing.model_dump()["source"] == "apec" assert listing.model_dump()["warnings"][0]["field"] == "location" - assert run_meta.model_dump()["run_id"] == "2026-06-01T10-00-00Z" + assert run_meta.model_dump()["run_id"] == FIXED_RUN_ID assert run_meta.model_dump()["run_started_at"] == "2026-06-01T10:00:00Z" assert run_meta.model_dump()["derived_queries"] == ["Data Engineer"] assert run_meta.model_dump(mode="json")["snapshots"] == [ @@ -58,8 +61,8 @@ def test_apec_models_serialize_expected_listing_shape() -> None: def test_apec_run_paths_builds_expected_layout(tmp_path: Path) -> None: - paths = apec_run_paths(tmp_path, run_id="2026-06-01T10-00-00Z") - run_dir = tmp_path / "apec" / "runs" / "2026-06-01T10-00-00Z" + paths = apec_run_paths(tmp_path, run_id=FIXED_RUN_ID) + run_dir = tmp_path / "apec" / "runs" / FIXED_RUN_ID assert paths["run_dir"] == run_dir assert paths["listings"] == run_dir / "listings.yaml" @@ -68,7 +71,7 @@ def test_apec_run_paths_builds_expected_layout(tmp_path: Path) -> None: def test_apec_run_artifacts_include_snapshot_and_meta(tmp_path: Path) -> None: - paths = apec_run_paths(tmp_path, run_id="2026-06-01T10-00-00Z") + paths = apec_run_paths(tmp_path, run_id=FIXED_RUN_ID) paths["snapshots"].mkdir(parents=True, exist_ok=True) snapshot = paths["snapshots"] / "job-123.html"