diff --git a/src/job_research/apec/adapter.py b/src/job_research/apec/adapter.py index 2dbda7c..20a6fe8 100644 --- a/src/job_research/apec/adapter.py +++ b/src/job_research/apec/adapter.py @@ -245,9 +245,19 @@ class ApecAdapter: seen_page_urls.add(current_page_url) - hrefs = page.locator(_RESULT_LINK_SELECTOR).evaluate_all( - "nodes => nodes.map(node => node.href)" - ) + try: + hrefs = page.locator(_RESULT_LINK_SELECTOR).evaluate_all( + "nodes => nodes.map(node => node.href)" + ) + except Exception: + self._record_search_error( + query, + search_filters, + f"page {page_number} result links could not be evaluated", + url=current_page_url, + ) + break + if not hrefs: no_progress_pages += 1 if no_progress_pages >= _MAX_CONSECUTIVE_NO_PROGRESS_PAGES: diff --git a/src/job_research/apec/normalize.py b/src/job_research/apec/normalize.py index b5e6884..68388bb 100644 --- a/src/job_research/apec/normalize.py +++ b/src/job_research/apec/normalize.py @@ -12,6 +12,7 @@ from job_research.models import ApecListing, ListingWarning _PUBLISHED_AT_PATTERN = re.compile(r"Publi[ée]e le (\d{2}/\d{2}/\d{4})") _REFRESHED_AT_PATTERN = re.compile(r"Actualis[ée]e le (\d{2}/\d{2}/\d{4})") +_DETAIL_JOB_ID_PATTERN = re.compile(r"/detail-offre/([^/?#]+)") _SOURCE_JOB_ID_PATTERN = re.compile(r"Ref\. Apec\s*:\s*([A-Z0-9]+)") _CONTRACT_PATTERN = re.compile(r"\b(CDI|CDD|Alternance|Intérim|Stage|Freelance|Indépendant)\b") _HEADING_TAG_NAMES = {"h1", "h2", "h3", "h4", "h5", "h6"} @@ -103,6 +104,14 @@ def _warning(field: str, message: str) -> ListingWarning: return ListingWarning(field=field, message=message) +def _extract_source_job_id_from_url(url: str) -> str | None: + match = _DETAIL_JOB_ID_PATTERN.search(url) + if match is None: + return None + + return match.group(1) + + def _extract_listing_date( soup: BeautifulSoup, pattern: re.Pattern[str], @@ -132,10 +141,7 @@ def _extract_listing_date( return None -def _extract_source_job_id(soup: BeautifulSoup, source_job_id: str | None) -> str | None: - if source_job_id is not None: - return source_job_id - +def _extract_source_job_id(soup: BeautifulSoup) -> str | None: ref = soup.select_one(".ref-offre") if ref is None: return None @@ -244,19 +250,29 @@ def normalize_apec_listing( warnings.append(_warning("description_text", "Description is empty or placeholder text")) description_text = None + requested_source_job_id = _extract_source_job_id_from_url(url) + ref_source_job_id = _extract_source_job_id(soup) + if source_job_id is not None: - normalized_source_job_id = source_job_id - else: - ref = soup.select_one(".ref-offre") - if ref is None: - warnings.append(_warning("source_job_id", "Source job id missing from Apec listing")) - normalized_source_job_id = None + if ( + requested_source_job_id is not None + and ref_source_job_id is not None + and requested_source_job_id != ref_source_job_id + ): + warnings.append(_warning("source_job_id", "Recovered source job id from ref-offre fallback")) + normalized_source_job_id = ref_source_job_id else: - ref_source_job_id = _extract_source_job_id(soup, None) - if ref_source_job_id is None: - warnings.append(_warning("source_job_id", "Source job id missing from ref-offre")) + normalized_source_job_id = source_job_id + else: + if ref_source_job_id is None: + if requested_source_job_id is None: + warnings.append(_warning("source_job_id", "Source job id missing from Apec listing")) + normalized_source_job_id = None else: - warnings.append(_warning("source_job_id", "Recovered source job id from ref-offre fallback")) + warnings.append(_warning("source_job_id", "Recovered source job id from detail URL fallback")) + normalized_source_job_id = requested_source_job_id + else: + warnings.append(_warning("source_job_id", "Recovered source job id from ref-offre fallback")) normalized_source_job_id = ref_source_job_id company = soup.select_one(".card-ents .ents-name") diff --git a/tests/apec/test_adapter.py b/tests/apec/test_adapter.py index 870278e..290a1a3 100644 --- a/tests/apec/test_adapter.py +++ b/tests/apec/test_adapter.py @@ -466,6 +466,59 @@ def test_search_records_pagination_render_failures(monkeypatch) -> None: assert adapter.search_errors[0].message == "page 1 results did not render" +def test_search_records_evaluate_all_failures_and_continues_to_next_query(monkeypatch) -> None: + first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0" + second_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=beta&page=0&selectedIndex=0" + page = _FakeDetailPage( + { + "alpha": { + 0: [first_result], + 1: [first_result], + }, + "beta": { + 0: [second_result], + 1: [second_result], + }, + } + ) + + original_locator = page.locator + + class _FlakyLocator: + def __init__(self, locator) -> None: + self._locator = locator + + def evaluate_all(self, function: str): + if ( + self._locator.selector == _RESULT_LINK_SELECTOR + and page.current_query == "alpha" + and page.current_page == 1 + ): + raise RuntimeError("evaluate boom") + + return self._locator.evaluate_all(function) + + def __getattr__(self, name: str): + return getattr(self._locator, name) + + def flaky_locator(selector: str): + return _FlakyLocator(original_locator(selector)) + + monkeypatch.setattr(page, "locator", flaky_locator) + monkeypatch.setattr(adapter_module, "_MAX_CONSECUTIVE_NO_PROGRESS_PAGES", 1) + monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page)) + + adapter = ApecAdapter(max_listings=10) + results = adapter.search( + ["alpha", "beta"], + search_filters=ApecSearchFilters(location="France", contract_type="CDI"), + ) + + assert [result.url for result in results] == [first_result, second_result] + assert [error.stage for error in adapter.search_errors] == ["search"] + assert "page=1" in adapter.search_errors[0].url + + def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> None: page = _FakeDetailPage({}, rendered_html="rendered offer") monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page)) diff --git a/tests/apec/test_normalize.py b/tests/apec/test_normalize.py index bb4eb16..09fa59a 100644 --- a/tests/apec/test_normalize.py +++ b/tests/apec/test_normalize.py @@ -71,6 +71,45 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None: assert listing.fetched_at == "2026-06-01T10:00:00Z" +def test_normalize_apec_listing_prefers_final_source_job_id_from_detail_page() -> None: + html = """ + + +
+

Data Engineer F/H

+
+
Ref. Apec : FINAL456
+ +

Publiée le 20/04/2026 Actualisée le 02/06/2026

+
+
+
+ CLOUD TEMPLE +
+
+
+

Descriptif du poste

+

Build pipelines

+
+
+ + + """ + + listing = normalize_apec_listing( + url="https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/REQUESTED123", + html=html, + fetched_at="2026-06-01T10:00:00Z", + source_job_id="REQUESTED123", + ) + + assert listing.source_job_id == "FINAL456" + + def test_normalize_apec_listing_warns_and_returns_none_for_invalid_dates() -> None: html = """