fix: refine Apec normalization metadata

src/job_research/apec/adapter.py

@@ -84,9 +84,16 @@ def _search_url(query: str, search_filters: ApecSearchFilters, page_number: int
 
 def _accept_cookies_if_present(page) -> None:
     try:
-        page.get_by_role("button", name="Accepter tous les cookies").click(timeout=2_000)
-    except PlaywrightTimeoutError:
-        return
+        page.locator('input[name="cguAcceptees"]').check(timeout=2_000)
+    except (AttributeError, PlaywrightTimeoutError):
+        pass
+
+    for button_name in ("ACCEPTER", "Accepter tous les cookies"):
+        try:
+            page.get_by_role("button", name=button_name).click(timeout=2_000)
+            return
+        except (AttributeError, PlaywrightTimeoutError):
+            continue
 
 
 def _goto_and_wait(page, url: str) -> bool:
@@ -280,6 +287,7 @@ class ApecAdapter:
 
         with self._open_page() as page:
             page.goto(url, wait_until="domcontentloaded")
+            _accept_cookies_if_present(page)
             page.wait_for_function(
                 """
                 () => {

src/job_research/apec/dedupe.py

@@ -9,6 +9,7 @@ _MERGEABLE_FIELDS = (
     "contract_type",
     "description_text",
     "published_at",
+    "refreshed_at",
 )
 
 

src/job_research/apec/normalize.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import re
-from datetime import datetime
 import unicodedata
+from datetime import datetime
 
 from bs4 import BeautifulSoup
 from bs4.element import NavigableString
@@ -11,6 +11,7 @@ from job_research.models import ApecListing, ListingWarning
 
 
 _PUBLISHED_AT_PATTERN = re.compile(r"Publi[ée]e le (\d{2}/\d{2}/\d{4})")
+_REFRESHED_AT_PATTERN = re.compile(r"Actualis[ée]e le (\d{2}/\d{2}/\d{4})")
 _SOURCE_JOB_ID_PATTERN = re.compile(r"Ref\. Apec\s*:\s*([A-Z0-9]+)")
 _CONTRACT_PATTERN = re.compile(r"\b(CDI|CDD|Alternance|Intérim|Stage|Freelance|Indépendant)\b")
 _HEADING_TAG_NAMES = {"h1", "h2", "h3", "h4", "h5", "h6"}
@@ -114,6 +115,18 @@ def _extract_published_at(soup: BeautifulSoup) -> str | None:
     return datetime.strptime(match.group(1), "%d/%m/%Y").date().isoformat()
 
 
+def _extract_refreshed_at(soup: BeautifulSoup) -> str | None:
+    card_offer = soup.select_one(".card-offer")
+    if card_offer is None:
+        return None
+
+    match = _REFRESHED_AT_PATTERN.search(card_offer.get_text(" ", strip=True))
+    if match is None:
+        return None
+
+    return datetime.strptime(match.group(1), "%d/%m/%Y").date().isoformat()
+
+
 def _extract_source_job_id(soup: BeautifulSoup, source_job_id: str | None) -> str | None:
     if source_job_id is not None:
         return source_job_id
@@ -170,6 +183,7 @@ def normalize_apec_listing(
     *,
     source_job_id: str | None = None,
     published_at: str | None = None,
+    refreshed_at: str | None = None,
 ) -> ApecListing:
     soup = BeautifulSoup(html, "html.parser")
     warnings: list[ListingWarning] = []
@@ -251,11 +265,16 @@ def normalize_apec_listing(
     company_text = _clean_text(company.get_text(" ", strip=True)) if company is not None else None
     if company_text is None:
         warnings.append(_warning("company", "Company missing from Apec listing"))
+    elif not _has_useful_text(company_text):
+        warnings.append(_warning("company", "Company is empty or placeholder text"))
+        company_text = None
 
     published_at_value = published_at or _extract_published_at(soup)
     if published_at_value is None:
         warnings.append(_warning("published_at", "Published date missing from Apec listing"))
 
+    refreshed_at_value = refreshed_at or _extract_refreshed_at(soup)
+
     return ApecListing(
         source="apec",
         source_job_id=normalized_source_job_id,
@@ -266,6 +285,7 @@ def normalize_apec_listing(
         contract_type=contract_type if _has_useful_text(contract_type) else None,
         description_text=description_text or None,
         published_at=published_at_value,
+        refreshed_at=refreshed_at_value,
         fetched_at=fetched_at,
         warnings=warnings,
     )

src/job_research/models.py

@@ -49,6 +49,7 @@ class ApecListing(BaseModel):
     contract_type: str | None = None
     description_text: str | None = None
     published_at: str | None = None
+    refreshed_at: str | None = None
     fetched_at: str
     warnings: list[ListingWarning] = Field(default_factory=list)
 

tests/apec/test_adapter.py

@@ -24,6 +24,9 @@ class _FakeResultButton:
                 f"?motsCles={quote_plus(self.page.current_query)}&page=0"
             )
             self.page.current_page = 0
+        elif self.name in {"ACCEPTER", "Accepter tous les cookies"}:
+            self.page.consent_button_clicks.append(self.name)
+            self.page.consent_accepted = True
 
 
 class _FakeLocator:
@@ -34,6 +37,13 @@ class _FakeLocator:
     def fill(self, value: str) -> None:
         self.page.current_query = value
 
+    def check(self, timeout: int | None = None) -> None:
+        if self.selector == 'input[name="cguAcceptees"]':
+            self.page.cgu_checkbox_checked = True
+            return None
+
+        raise PlaywrightTimeoutError(f"selector not found: {self.selector}")
+
     def evaluate_all(self, function: str):
         if self.selector == _RESULT_LINK_SELECTOR:
             return list(self.page.current_results())
@@ -49,6 +59,7 @@ class _FakeDetailPage:
         rendered_html: str = "<html>rendered</html>",
         search_ready: bool = True,
         zero_result_queries: set[str] | None = None,
+        consent_required: bool = False,
     ) -> None:
         self.result_pages = result_pages
         self.rendered_html = rendered_html
@@ -56,6 +67,10 @@ class _FakeDetailPage:
         self.waited_functions: list[tuple[str, int | None]] = []
         self.search_ready = search_ready
         self.zero_result_queries = zero_result_queries or set()
+        self.consent_required = consent_required
+        self.cgu_checkbox_checked = False
+        self.consent_button_clicks: list[str] = []
+        self.consent_accepted = not consent_required
         self.goto_urls: list[str] = []
         self.current_query = ""
         self.current_page = 0
@@ -91,13 +106,16 @@ class _FakeDetailPage:
         self.default_timeout = timeout
 
     def wait_for_function(self, function: str, polling: int | None = None, timeout: int | None = None) -> None:
+        if self.consent_required and not self.consent_accepted:
+            raise PlaywrightTimeoutError("consent not accepted")
+
         self.waited_functions.append((function, polling))
         self.rendered = True
         return None
 
     def wait_for_selector(self, selector: str, timeout: int | None = None) -> None:
         if selector == _SEARCH_INPUT_SELECTOR:
-            if self.search_ready:
+            if self.search_ready and self.consent_accepted:
                 return None
 
             raise PlaywrightTimeoutError(f"selector not found: {selector}")
@@ -363,6 +381,22 @@ def test_search_raises_when_every_query_renders_broken_search_shell(monkeypatch)
     assert [error.stage for error in adapter.search_errors] == ["search", "search"]
 
 
+def test_search_accepts_current_cgu_popin_before_waiting_for_results(monkeypatch) -> None:
+    first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"
+    page = _FakeDetailPage({"alpha": {0: [first_result]}}, consent_required=True)
+    monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
+
+    results = ApecAdapter(max_listings=10).search(
+        ["alpha"],
+        search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
+    )
+
+    assert [result.url for result in results] == [first_result]
+    assert page.cgu_checkbox_checked is True
+    assert page.consent_button_clicks == ["ACCEPTER"]
+    assert page.consent_accepted is True
+
+
 def test_search_records_pagination_navigation_failures(monkeypatch) -> None:
     first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"
     page = _FakeDetailPage(
@@ -449,6 +483,20 @@ def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> Non
     assert page.waited_functions[0][1] == 1000
 
 
+def test_fetch_listing_html_accepts_current_cgu_popin_before_waiting_for_detail_content(monkeypatch) -> None:
+    page = _FakeDetailPage({}, rendered_html="<html>rendered offer</html>", consent_required=True)
+    monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
+
+    html = ApecAdapter().fetch_listing_html(
+        "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111"
+    )
+
+    assert html == "<html>rendered offer</html>"
+    assert page.cgu_checkbox_checked is True
+    assert page.consent_button_clicks == ["ACCEPTER"]
+    assert page.consent_accepted is True
+
+
 def test_fetch_listing_html_uses_explicit_company_fallback_chain(monkeypatch) -> None:
     page = _FakeDetailPage({}, rendered_html="<html>rendered offer</html>")
     monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))

tests/apec/test_dedupe.py

@@ -87,6 +87,7 @@ def test_dedupe_apec_listings_merges_metadata_from_duplicate_rows() -> None:
         source="apec",
         source_job_id=None,
         published_at=None,
+        refreshed_at=None,
         url="url1",
         fetched_at="2026-06-01T10:00:00Z",
     )
@@ -94,6 +95,7 @@ def test_dedupe_apec_listings_merges_metadata_from_duplicate_rows() -> None:
         source="apec",
         source_job_id="job-123",
         published_at="2026-06-01",
+        refreshed_at="2026-06-02",
         url="url1",
         fetched_at="2026-06-01T10:01:00Z",
     )
@@ -104,6 +106,7 @@ def test_dedupe_apec_listings_merges_metadata_from_duplicate_rows() -> None:
     assert deduped[0].url == "url1"
     assert deduped[0].source_job_id == "job-123"
     assert deduped[0].published_at == "2026-06-01"
+    assert deduped[0].refreshed_at == "2026-06-02"
 
 
 def test_dedupe_apec_listings_merges_metadata_through_alias_chain() -> None:

tests/apec/test_normalize.py

@@ -67,6 +67,7 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None:
     assert listing.contract_type == "CDI"
     assert listing.description_text == "Build pipelines"
     assert listing.published_at == "2026-04-20"
+    assert listing.refreshed_at == "2026-06-02"
     assert listing.fetched_at == "2026-06-01T10:00:00Z"
 
 
@@ -124,6 +125,7 @@ def test_normalize_apec_listing_uses_details_offer_list_company_fallback() -> No
 
     assert listing.company == "Fallback Company"
     assert listing.description_text == "Build pipelines"
+    assert listing.refreshed_at == "2026-06-02"
 
 
 def test_normalize_apec_listing_records_warnings_for_fallback_and_missing_fields() -> None:
@@ -207,3 +209,44 @@ def test_normalize_apec_listing_records_warnings_for_placeholder_text_values() -
     assert listing.location is None
     assert listing.contract_type is None
     assert [warning.field for warning in listing.warnings] == ["title", "location", "contract_type"]
+
+
+def test_normalize_apec_listing_records_warning_for_placeholder_company() -> None:
+    html = """
+    <html>
+      <body>
+        <main class="container-details-offer">
+          <h1>Data Engineer F/H</h1>
+          <div class="card-offer">
+            <div class="ref-offre">Ref. Apec : 178554452W</div>
+            <ul class="details-offer-list mb-20">
+              <li>Example Corp</li>
+              <li>1 <span> CDI </span></li>
+              <li>Puteaux - 92</li>
+            </ul>
+            <p>Publiée le 20/04/2026 Actualisée le 02/06/2026</p>
+          </div>
+          <article class="card card-ents mb-20">
+            <div class="list-hzt mb-20">
+              <span class="ents-name">N/A</span>
+            </div>
+          </article>
+          <div class="details-post">
+            <h4>Descriptif du poste</h4>
+            <p>Build pipelines</p>
+          </div>
+        </main>
+      </body>
+    </html>
+    """
+
+    listing = normalize_apec_listing(
+        url="https://example.test/job/123",
+        html=html,
+        fetched_at="2026-06-01T10:00:00Z",
+        source_job_id="178554452W",
+    )
+
+    assert listing.company is None
+    assert listing.refreshed_at == "2026-06-02"
+    assert [warning.field for warning in listing.warnings] == ["company"]

tests/test_apec_cli.py

@@ -611,6 +611,7 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch
             "contract_type": "CDI",
             "description_text": "Build pipelines",
             "published_at": "2026-04-20",
+            "refreshed_at": "2026-06-02",
             "fetched_at": "2026-06-01T10:00:00Z",
             "warnings": [],
         }
@@ -774,6 +775,7 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings(
             "contract_type": "CDI",
             "description_text": "Build pipelines",
             "published_at": "2026-04-20",
+            "refreshed_at": "2026-06-02",
             "fetched_at": "2026-06-01T10:00:00Z",
             "warnings": [],
         }
@@ -880,6 +882,7 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_li
             "contract_type": "CDI",
             "description_text": "Build pipelines",
             "published_at": "2026-04-20",
+            "refreshed_at": "2026-06-02",
             "fetched_at": "2026-06-01T10:00:00Z",
             "warnings": [],
         },
@@ -893,6 +896,7 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_li
             "contract_type": "CDI",
             "description_text": "Build pipelines",
             "published_at": "2026-04-20",
+            "refreshed_at": "2026-06-02",
             "fetched_at": "2026-06-01T10:00:00Z",
             "warnings": [],
         },
@@ -1000,6 +1004,7 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails(
             "contract_type": "CDI",
             "description_text": "Build pipelines",
             "published_at": "2026-04-20",
+            "refreshed_at": "2026-06-02",
             "fetched_at": "2026-06-01T10:00:00Z",
             "warnings": [],
         },
@@ -1013,6 +1018,7 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails(
             "contract_type": "CDI",
             "description_text": "Build pipelines",
             "published_at": "2026-04-20",
+            "refreshed_at": "2026-06-02",
             "fetched_at": "2026-06-01T10:00:00Z",
             "warnings": [],
         },

tests/test_apec_storage.py

@@ -18,6 +18,7 @@ def test_apec_models_serialize_expected_listing_shape() -> None:
         contract_type="CDI",
         description_text="Build pipelines",
         published_at="2026-06-01",
+        refreshed_at="2026-06-02",
         fetched_at="2026-06-01T10:00:00Z",
         warnings=[
             ListingWarning(
@@ -47,6 +48,7 @@ def test_apec_models_serialize_expected_listing_shape() -> None:
 
     assert listing.model_dump()["source"] == "apec"
     assert listing.model_dump()["warnings"][0]["field"] == "location"
+    assert listing.model_dump()["refreshed_at"] == "2026-06-02"
     assert run_meta.model_dump()["run_id"] == FIXED_RUN_ID
     assert run_meta.model_dump()["run_started_at"] == "2026-06-01T10:00:00Z"
     assert run_meta.model_dump()["derived_queries"] == ["Data Engineer"]