diff --git a/src/job_research/apec/adapter.py b/src/job_research/apec/adapter.py index 7c808d2..cae593a 100644 --- a/src/job_research/apec/adapter.py +++ b/src/job_research/apec/adapter.py @@ -84,9 +84,16 @@ def _search_url(query: str, search_filters: ApecSearchFilters, page_number: int def _accept_cookies_if_present(page) -> None: try: - page.get_by_role("button", name="Accepter tous les cookies").click(timeout=2_000) - except PlaywrightTimeoutError: - return + page.locator('input[name="cguAcceptees"]').check(timeout=2_000) + except (AttributeError, PlaywrightTimeoutError): + pass + + for button_name in ("ACCEPTER", "Accepter tous les cookies"): + try: + page.get_by_role("button", name=button_name).click(timeout=2_000) + return + except (AttributeError, PlaywrightTimeoutError): + continue def _goto_and_wait(page, url: str) -> bool: @@ -280,6 +287,7 @@ class ApecAdapter: with self._open_page() as page: page.goto(url, wait_until="domcontentloaded") + _accept_cookies_if_present(page) page.wait_for_function( """ () => { diff --git a/src/job_research/apec/dedupe.py b/src/job_research/apec/dedupe.py index 65866c5..d9a2f74 100644 --- a/src/job_research/apec/dedupe.py +++ b/src/job_research/apec/dedupe.py @@ -9,6 +9,7 @@ _MERGEABLE_FIELDS = ( "contract_type", "description_text", "published_at", + "refreshed_at", ) diff --git a/src/job_research/apec/normalize.py b/src/job_research/apec/normalize.py index 527bef6..5670ec4 100644 --- a/src/job_research/apec/normalize.py +++ b/src/job_research/apec/normalize.py @@ -1,8 +1,8 @@ from __future__ import annotations import re -from datetime import datetime import unicodedata +from datetime import datetime from bs4 import BeautifulSoup from bs4.element import NavigableString @@ -11,6 +11,7 @@ from job_research.models import ApecListing, ListingWarning _PUBLISHED_AT_PATTERN = re.compile(r"Publi[ée]e le (\d{2}/\d{2}/\d{4})") +_REFRESHED_AT_PATTERN = re.compile(r"Actualis[ée]e le (\d{2}/\d{2}/\d{4})") _SOURCE_JOB_ID_PATTERN = re.compile(r"Ref\. Apec\s*:\s*([A-Z0-9]+)") _CONTRACT_PATTERN = re.compile(r"\b(CDI|CDD|Alternance|Intérim|Stage|Freelance|Indépendant)\b") _HEADING_TAG_NAMES = {"h1", "h2", "h3", "h4", "h5", "h6"} @@ -114,6 +115,18 @@ def _extract_published_at(soup: BeautifulSoup) -> str | None: return datetime.strptime(match.group(1), "%d/%m/%Y").date().isoformat() +def _extract_refreshed_at(soup: BeautifulSoup) -> str | None: + card_offer = soup.select_one(".card-offer") + if card_offer is None: + return None + + match = _REFRESHED_AT_PATTERN.search(card_offer.get_text(" ", strip=True)) + if match is None: + return None + + return datetime.strptime(match.group(1), "%d/%m/%Y").date().isoformat() + + def _extract_source_job_id(soup: BeautifulSoup, source_job_id: str | None) -> str | None: if source_job_id is not None: return source_job_id @@ -170,6 +183,7 @@ def normalize_apec_listing( *, source_job_id: str | None = None, published_at: str | None = None, + refreshed_at: str | None = None, ) -> ApecListing: soup = BeautifulSoup(html, "html.parser") warnings: list[ListingWarning] = [] @@ -251,11 +265,16 @@ def normalize_apec_listing( company_text = _clean_text(company.get_text(" ", strip=True)) if company is not None else None if company_text is None: warnings.append(_warning("company", "Company missing from Apec listing")) + elif not _has_useful_text(company_text): + warnings.append(_warning("company", "Company is empty or placeholder text")) + company_text = None published_at_value = published_at or _extract_published_at(soup) if published_at_value is None: warnings.append(_warning("published_at", "Published date missing from Apec listing")) + refreshed_at_value = refreshed_at or _extract_refreshed_at(soup) + return ApecListing( source="apec", source_job_id=normalized_source_job_id, @@ -266,6 +285,7 @@ def normalize_apec_listing( contract_type=contract_type if _has_useful_text(contract_type) else None, description_text=description_text or None, published_at=published_at_value, + refreshed_at=refreshed_at_value, fetched_at=fetched_at, warnings=warnings, ) diff --git a/src/job_research/models.py b/src/job_research/models.py index df68499..3ebc5b2 100644 --- a/src/job_research/models.py +++ b/src/job_research/models.py @@ -49,6 +49,7 @@ class ApecListing(BaseModel): contract_type: str | None = None description_text: str | None = None published_at: str | None = None + refreshed_at: str | None = None fetched_at: str warnings: list[ListingWarning] = Field(default_factory=list) diff --git a/tests/apec/test_adapter.py b/tests/apec/test_adapter.py index 221dec4..567e510 100644 --- a/tests/apec/test_adapter.py +++ b/tests/apec/test_adapter.py @@ -24,6 +24,9 @@ class _FakeResultButton: f"?motsCles={quote_plus(self.page.current_query)}&page=0" ) self.page.current_page = 0 + elif self.name in {"ACCEPTER", "Accepter tous les cookies"}: + self.page.consent_button_clicks.append(self.name) + self.page.consent_accepted = True class _FakeLocator: @@ -34,6 +37,13 @@ class _FakeLocator: def fill(self, value: str) -> None: self.page.current_query = value + def check(self, timeout: int | None = None) -> None: + if self.selector == 'input[name="cguAcceptees"]': + self.page.cgu_checkbox_checked = True + return None + + raise PlaywrightTimeoutError(f"selector not found: {self.selector}") + def evaluate_all(self, function: str): if self.selector == _RESULT_LINK_SELECTOR: return list(self.page.current_results()) @@ -49,6 +59,7 @@ class _FakeDetailPage: rendered_html: str = "rendered", search_ready: bool = True, zero_result_queries: set[str] | None = None, + consent_required: bool = False, ) -> None: self.result_pages = result_pages self.rendered_html = rendered_html @@ -56,6 +67,10 @@ class _FakeDetailPage: self.waited_functions: list[tuple[str, int | None]] = [] self.search_ready = search_ready self.zero_result_queries = zero_result_queries or set() + self.consent_required = consent_required + self.cgu_checkbox_checked = False + self.consent_button_clicks: list[str] = [] + self.consent_accepted = not consent_required self.goto_urls: list[str] = [] self.current_query = "" self.current_page = 0 @@ -91,13 +106,16 @@ class _FakeDetailPage: self.default_timeout = timeout def wait_for_function(self, function: str, polling: int | None = None, timeout: int | None = None) -> None: + if self.consent_required and not self.consent_accepted: + raise PlaywrightTimeoutError("consent not accepted") + self.waited_functions.append((function, polling)) self.rendered = True return None def wait_for_selector(self, selector: str, timeout: int | None = None) -> None: if selector == _SEARCH_INPUT_SELECTOR: - if self.search_ready: + if self.search_ready and self.consent_accepted: return None raise PlaywrightTimeoutError(f"selector not found: {selector}") @@ -363,6 +381,22 @@ def test_search_raises_when_every_query_renders_broken_search_shell(monkeypatch) assert [error.stage for error in adapter.search_errors] == ["search", "search"] +def test_search_accepts_current_cgu_popin_before_waiting_for_results(monkeypatch) -> None: + first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0" + page = _FakeDetailPage({"alpha": {0: [first_result]}}, consent_required=True) + monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page)) + + results = ApecAdapter(max_listings=10).search( + ["alpha"], + search_filters=ApecSearchFilters(location="France", contract_type="CDI"), + ) + + assert [result.url for result in results] == [first_result] + assert page.cgu_checkbox_checked is True + assert page.consent_button_clicks == ["ACCEPTER"] + assert page.consent_accepted is True + + def test_search_records_pagination_navigation_failures(monkeypatch) -> None: first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0" page = _FakeDetailPage( @@ -449,6 +483,20 @@ def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> Non assert page.waited_functions[0][1] == 1000 +def test_fetch_listing_html_accepts_current_cgu_popin_before_waiting_for_detail_content(monkeypatch) -> None: + page = _FakeDetailPage({}, rendered_html="rendered offer", consent_required=True) + monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page)) + + html = ApecAdapter().fetch_listing_html( + "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111" + ) + + assert html == "rendered offer" + assert page.cgu_checkbox_checked is True + assert page.consent_button_clicks == ["ACCEPTER"] + assert page.consent_accepted is True + + def test_fetch_listing_html_uses_explicit_company_fallback_chain(monkeypatch) -> None: page = _FakeDetailPage({}, rendered_html="rendered offer") monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page)) diff --git a/tests/apec/test_dedupe.py b/tests/apec/test_dedupe.py index b514891..189ab86 100644 --- a/tests/apec/test_dedupe.py +++ b/tests/apec/test_dedupe.py @@ -87,6 +87,7 @@ def test_dedupe_apec_listings_merges_metadata_from_duplicate_rows() -> None: source="apec", source_job_id=None, published_at=None, + refreshed_at=None, url="url1", fetched_at="2026-06-01T10:00:00Z", ) @@ -94,6 +95,7 @@ def test_dedupe_apec_listings_merges_metadata_from_duplicate_rows() -> None: source="apec", source_job_id="job-123", published_at="2026-06-01", + refreshed_at="2026-06-02", url="url1", fetched_at="2026-06-01T10:01:00Z", ) @@ -104,6 +106,7 @@ def test_dedupe_apec_listings_merges_metadata_from_duplicate_rows() -> None: assert deduped[0].url == "url1" assert deduped[0].source_job_id == "job-123" assert deduped[0].published_at == "2026-06-01" + assert deduped[0].refreshed_at == "2026-06-02" def test_dedupe_apec_listings_merges_metadata_through_alias_chain() -> None: diff --git a/tests/apec/test_normalize.py b/tests/apec/test_normalize.py index 77ebff6..292f0ff 100644 --- a/tests/apec/test_normalize.py +++ b/tests/apec/test_normalize.py @@ -67,6 +67,7 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None: assert listing.contract_type == "CDI" assert listing.description_text == "Build pipelines" assert listing.published_at == "2026-04-20" + assert listing.refreshed_at == "2026-06-02" assert listing.fetched_at == "2026-06-01T10:00:00Z" @@ -124,6 +125,7 @@ def test_normalize_apec_listing_uses_details_offer_list_company_fallback() -> No assert listing.company == "Fallback Company" assert listing.description_text == "Build pipelines" + assert listing.refreshed_at == "2026-06-02" def test_normalize_apec_listing_records_warnings_for_fallback_and_missing_fields() -> None: @@ -207,3 +209,44 @@ def test_normalize_apec_listing_records_warnings_for_placeholder_text_values() - assert listing.location is None assert listing.contract_type is None assert [warning.field for warning in listing.warnings] == ["title", "location", "contract_type"] + + +def test_normalize_apec_listing_records_warning_for_placeholder_company() -> None: + html = """ + + +
+

Data Engineer F/H

+
+
Ref. Apec : 178554452W
+ +

Publiée le 20/04/2026 Actualisée le 02/06/2026

+
+
+
+ N/A +
+
+
+

Descriptif du poste

+

Build pipelines

+
+
+ + + """ + + listing = normalize_apec_listing( + url="https://example.test/job/123", + html=html, + fetched_at="2026-06-01T10:00:00Z", + source_job_id="178554452W", + ) + + assert listing.company is None + assert listing.refreshed_at == "2026-06-02" + assert [warning.field for warning in listing.warnings] == ["company"] diff --git a/tests/test_apec_cli.py b/tests/test_apec_cli.py index 4d6fb7d..704301c 100644 --- a/tests/test_apec_cli.py +++ b/tests/test_apec_cli.py @@ -611,6 +611,7 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch "contract_type": "CDI", "description_text": "Build pipelines", "published_at": "2026-04-20", + "refreshed_at": "2026-06-02", "fetched_at": "2026-06-01T10:00:00Z", "warnings": [], } @@ -774,6 +775,7 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings( "contract_type": "CDI", "description_text": "Build pipelines", "published_at": "2026-04-20", + "refreshed_at": "2026-06-02", "fetched_at": "2026-06-01T10:00:00Z", "warnings": [], } @@ -880,6 +882,7 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_li "contract_type": "CDI", "description_text": "Build pipelines", "published_at": "2026-04-20", + "refreshed_at": "2026-06-02", "fetched_at": "2026-06-01T10:00:00Z", "warnings": [], }, @@ -893,6 +896,7 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_li "contract_type": "CDI", "description_text": "Build pipelines", "published_at": "2026-04-20", + "refreshed_at": "2026-06-02", "fetched_at": "2026-06-01T10:00:00Z", "warnings": [], }, @@ -1000,6 +1004,7 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails( "contract_type": "CDI", "description_text": "Build pipelines", "published_at": "2026-04-20", + "refreshed_at": "2026-06-02", "fetched_at": "2026-06-01T10:00:00Z", "warnings": [], }, @@ -1013,6 +1018,7 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails( "contract_type": "CDI", "description_text": "Build pipelines", "published_at": "2026-04-20", + "refreshed_at": "2026-06-02", "fetched_at": "2026-06-01T10:00:00Z", "warnings": [], }, diff --git a/tests/test_apec_storage.py b/tests/test_apec_storage.py index 801e2ad..34b6a1b 100644 --- a/tests/test_apec_storage.py +++ b/tests/test_apec_storage.py @@ -18,6 +18,7 @@ def test_apec_models_serialize_expected_listing_shape() -> None: contract_type="CDI", description_text="Build pipelines", published_at="2026-06-01", + refreshed_at="2026-06-02", fetched_at="2026-06-01T10:00:00Z", warnings=[ ListingWarning( @@ -47,6 +48,7 @@ def test_apec_models_serialize_expected_listing_shape() -> None: assert listing.model_dump()["source"] == "apec" assert listing.model_dump()["warnings"][0]["field"] == "location" + assert listing.model_dump()["refreshed_at"] == "2026-06-02" assert run_meta.model_dump()["run_id"] == FIXED_RUN_ID assert run_meta.model_dump()["run_started_at"] == "2026-06-01T10:00:00Z" assert run_meta.model_dump()["derived_queries"] == ["Data Engineer"]