diff --git a/src/job_research/apec/adapter.py b/src/job_research/apec/adapter.py index 2f67f1d..7c808d2 100644 --- a/src/job_research/apec/adapter.py +++ b/src/job_research/apec/adapter.py @@ -145,9 +145,16 @@ class ApecAdapter: finally: page.close() - def _record_search_error(self, query: str, search_filters: ApecSearchFilters, message: str) -> None: + def _record_search_error( + self, + query: str, + search_filters: ApecSearchFilters, + message: str, + *, + url: str | None = None, + ) -> None: self.search_errors.append( - ListingError(url=_search_url(query, search_filters), stage="search", message=message) + ListingError(url=url or _search_url(query, search_filters), stage="search", message=message) ) @staticmethod @@ -206,11 +213,23 @@ class ApecAdapter: break if not _goto_and_wait(page, next_page_url): + self._record_search_error( + query, + search_filters, + f"page {page_number} navigation failed", + url=next_page_url, + ) break try: page.wait_for_selector(_RESULT_LINK_SELECTOR, timeout=5_000) except PlaywrightTimeoutError: + self._record_search_error( + query, + search_filters, + f"page {page_number} results did not render", + url=next_page_url, + ) break current_page_url = page.url diff --git a/src/job_research/apec/normalize.py b/src/job_research/apec/normalize.py index 8191fc3..527bef6 100644 --- a/src/job_research/apec/normalize.py +++ b/src/job_research/apec/normalize.py @@ -2,6 +2,7 @@ from __future__ import annotations import re from datetime import datetime +import unicodedata from bs4 import BeautifulSoup from bs4.element import NavigableString @@ -13,6 +14,17 @@ _PUBLISHED_AT_PATTERN = re.compile(r"Publi[ée]e le (\d{2}/\d{2}/\d{4})") _SOURCE_JOB_ID_PATTERN = re.compile(r"Ref\. Apec\s*:\s*([A-Z0-9]+)") _CONTRACT_PATTERN = re.compile(r"\b(CDI|CDD|Alternance|Intérim|Stage|Freelance|Indépendant)\b") _HEADING_TAG_NAMES = {"h1", "h2", "h3", "h4", "h5", "h6"} +_PLACEHOLDER_TEXT_TOKENS = { + "na", + "nr", + "none", + "null", + "unknown", + "tbd", + "nonrenseigne", + "nonrenseignee", + "nondisponible", +} def _clean_text(value: str | None) -> str | None: @@ -23,6 +35,20 @@ def _clean_text(value: str | None) -> str | None: return cleaned or None +def _text_token(value: str) -> str: + normalized = unicodedata.normalize("NFKD", value) + return re.sub(r"[^a-z0-9]+", "", normalized.casefold()) + + +def _has_useful_text(value: str | None) -> bool: + cleaned = _clean_text(value) + if cleaned is None: + return False + + token = _text_token(cleaned) + return bool(token) and token not in _PLACEHOLDER_TEXT_TOKENS + + def _text_before_heading(node) -> str | None: if isinstance(node, NavigableString): return _clean_text(str(node)) @@ -155,6 +181,9 @@ def normalize_apec_listing( warnings.append(_warning("title", "Recovered title from generic h1 fallback")) else: warnings.append(_warning("title", "Title missing from Apec listing")) + title_text = _clean_text(title.get_text(" ", strip=True)) if title is not None else None + if title is not None and not _has_useful_text(title_text): + warnings.append(_warning("title", "Title is empty or placeholder text")) details_offer_list = soup.select_one(".details-offer-list") @@ -163,7 +192,9 @@ def normalize_apec_listing( if details_offer_list is not None: location_item = details_offer_list.select_one("li:nth-of-type(3)") if location_item is not None: - location = location_item.get_text(" ", strip=True) + location = _clean_text(location_item.get_text(" ", strip=True)) + if not _has_useful_text(location): + warnings.append(_warning("location", "Location is empty or placeholder text")) else: warnings.append(_warning("location", "Location missing from details-offer list")) @@ -174,6 +205,8 @@ def normalize_apec_listing( span = contract_item.find("span") if span is not None: contract_type = _clean_text(span.get_text(" ", strip=True)) + if not _has_useful_text(contract_type): + warnings.append(_warning("contract_type", "Contract type is empty or placeholder text")) else: match = _CONTRACT_PATTERN.search(contract_item.get_text(" ", strip=True)) if match is not None: @@ -227,10 +260,10 @@ def normalize_apec_listing( source="apec", source_job_id=normalized_source_job_id, url=url, - title=_clean_text(title.get_text(" ", strip=True)) if title else None, + title=title_text if _has_useful_text(title_text) else None, company=company_text, - location=_clean_text(location) if location else None, - contract_type=contract_type, + location=location if _has_useful_text(location) else None, + contract_type=contract_type if _has_useful_text(contract_type) else None, description_text=description_text or None, published_at=published_at_value, fetched_at=fetched_at, diff --git a/tests/apec/test_adapter.py b/tests/apec/test_adapter.py index d9915ac..221dec4 100644 --- a/tests/apec/test_adapter.py +++ b/tests/apec/test_adapter.py @@ -363,6 +363,75 @@ def test_search_raises_when_every_query_renders_broken_search_shell(monkeypatch) assert [error.stage for error in adapter.search_errors] == ["search", "search"] +def test_search_records_pagination_navigation_failures(monkeypatch) -> None: + first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0" + page = _FakeDetailPage( + { + "alpha": { + 0: [first_result], + 1: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=alpha&page=1&selectedIndex=0"], + }, + } + ) + + original_goto = page.goto + + def flaky_goto(url: str, wait_until: str | None = None) -> None: + if "page=1" in url: + raise RuntimeError("navigation boom") + + original_goto(url, wait_until=wait_until) + + monkeypatch.setattr(page, "goto", flaky_goto) + monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page)) + + adapter = ApecAdapter(max_listings=10) + results = adapter.search( + ["alpha"], + search_filters=ApecSearchFilters(location="France", contract_type="CDI"), + ) + + assert [result.url for result in results] == [first_result] + assert [error.stage for error in adapter.search_errors] == ["search"] + assert "page=1" in adapter.search_errors[0].url + assert adapter.search_errors[0].message == "page 1 navigation failed" + + +def test_search_records_pagination_render_failures(monkeypatch) -> None: + first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0" + second_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=alpha&page=1&selectedIndex=0" + page = _FakeDetailPage( + { + "alpha": { + 0: [first_result], + 1: [second_result], + }, + } + ) + + original_wait_for_selector = page.wait_for_selector + + def flaky_wait_for_selector(selector: str, timeout: int | None = None) -> None: + if selector == _RESULT_LINK_SELECTOR and page.current_page == 1: + raise PlaywrightTimeoutError(f"selector not found: {selector}") + + original_wait_for_selector(selector, timeout=timeout) + + monkeypatch.setattr(page, "wait_for_selector", flaky_wait_for_selector) + monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page)) + + adapter = ApecAdapter(max_listings=10) + results = adapter.search( + ["alpha"], + search_filters=ApecSearchFilters(location="France", contract_type="CDI"), + ) + + assert [result.url for result in results] == [first_result] + assert [error.stage for error in adapter.search_errors] == ["search"] + assert "page=1" in adapter.search_errors[0].url + assert adapter.search_errors[0].message == "page 1 results did not render" + + def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> None: page = _FakeDetailPage({}, rendered_html="rendered offer") monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page)) diff --git a/tests/apec/test_normalize.py b/tests/apec/test_normalize.py index 213aaf7..77ebff6 100644 --- a/tests/apec/test_normalize.py +++ b/tests/apec/test_normalize.py @@ -165,3 +165,45 @@ def test_normalize_apec_listing_records_warnings_for_fallback_and_missing_fields "company", "published_at", ] + + +def test_normalize_apec_listing_records_warnings_for_placeholder_text_values() -> None: + html = """ + + +
+

N/A

+
+
Ref. Apec : 178554452W
+ +

Publiée le 20/04/2026 Actualisée le 02/06/2026

+
+
+
+ Example Corp +
+
+
+

Descriptif du poste

+

Build pipelines

+
+
+ + + """ + + listing = normalize_apec_listing( + url="https://example.test/job/123", + html=html, + fetched_at="2026-06-01T10:00:00Z", + source_job_id="178554452W", + ) + + assert listing.title is None + assert listing.location is None + assert listing.contract_type is None + assert [warning.field for warning in listing.warnings] == ["title", "location", "contract_type"]