fix: harden Apec search and canonicalization

This commit is contained in:
Antoine 2026-06-05 13:59:03 +02:00
parent ed1af201bc
commit 85c4278db3
4 changed files with 135 additions and 17 deletions

View File

@ -245,9 +245,19 @@ class ApecAdapter:
seen_page_urls.add(current_page_url) seen_page_urls.add(current_page_url)
hrefs = page.locator(_RESULT_LINK_SELECTOR).evaluate_all( try:
"nodes => nodes.map(node => node.href)" hrefs = page.locator(_RESULT_LINK_SELECTOR).evaluate_all(
) "nodes => nodes.map(node => node.href)"
)
except Exception:
self._record_search_error(
query,
search_filters,
f"page {page_number} result links could not be evaluated",
url=current_page_url,
)
break
if not hrefs: if not hrefs:
no_progress_pages += 1 no_progress_pages += 1
if no_progress_pages >= _MAX_CONSECUTIVE_NO_PROGRESS_PAGES: if no_progress_pages >= _MAX_CONSECUTIVE_NO_PROGRESS_PAGES:

View File

@ -12,6 +12,7 @@ from job_research.models import ApecListing, ListingWarning
_PUBLISHED_AT_PATTERN = re.compile(r"Publi[ée]e le (\d{2}/\d{2}/\d{4})") _PUBLISHED_AT_PATTERN = re.compile(r"Publi[ée]e le (\d{2}/\d{2}/\d{4})")
_REFRESHED_AT_PATTERN = re.compile(r"Actualis[ée]e le (\d{2}/\d{2}/\d{4})") _REFRESHED_AT_PATTERN = re.compile(r"Actualis[ée]e le (\d{2}/\d{2}/\d{4})")
_DETAIL_JOB_ID_PATTERN = re.compile(r"/detail-offre/([^/?#]+)")
_SOURCE_JOB_ID_PATTERN = re.compile(r"Ref\. Apec\s*:\s*([A-Z0-9]+)") _SOURCE_JOB_ID_PATTERN = re.compile(r"Ref\. Apec\s*:\s*([A-Z0-9]+)")
_CONTRACT_PATTERN = re.compile(r"\b(CDI|CDD|Alternance|Intérim|Stage|Freelance|Indépendant)\b") _CONTRACT_PATTERN = re.compile(r"\b(CDI|CDD|Alternance|Intérim|Stage|Freelance|Indépendant)\b")
_HEADING_TAG_NAMES = {"h1", "h2", "h3", "h4", "h5", "h6"} _HEADING_TAG_NAMES = {"h1", "h2", "h3", "h4", "h5", "h6"}
@ -103,6 +104,14 @@ def _warning(field: str, message: str) -> ListingWarning:
return ListingWarning(field=field, message=message) return ListingWarning(field=field, message=message)
def _extract_source_job_id_from_url(url: str) -> str | None:
match = _DETAIL_JOB_ID_PATTERN.search(url)
if match is None:
return None
return match.group(1)
def _extract_listing_date( def _extract_listing_date(
soup: BeautifulSoup, soup: BeautifulSoup,
pattern: re.Pattern[str], pattern: re.Pattern[str],
@ -132,10 +141,7 @@ def _extract_listing_date(
return None return None
def _extract_source_job_id(soup: BeautifulSoup, source_job_id: str | None) -> str | None: def _extract_source_job_id(soup: BeautifulSoup) -> str | None:
if source_job_id is not None:
return source_job_id
ref = soup.select_one(".ref-offre") ref = soup.select_one(".ref-offre")
if ref is None: if ref is None:
return None return None
@ -244,19 +250,29 @@ def normalize_apec_listing(
warnings.append(_warning("description_text", "Description is empty or placeholder text")) warnings.append(_warning("description_text", "Description is empty or placeholder text"))
description_text = None description_text = None
requested_source_job_id = _extract_source_job_id_from_url(url)
ref_source_job_id = _extract_source_job_id(soup)
if source_job_id is not None: if source_job_id is not None:
normalized_source_job_id = source_job_id if (
else: requested_source_job_id is not None
ref = soup.select_one(".ref-offre") and ref_source_job_id is not None
if ref is None: and requested_source_job_id != ref_source_job_id
warnings.append(_warning("source_job_id", "Source job id missing from Apec listing")) ):
normalized_source_job_id = None warnings.append(_warning("source_job_id", "Recovered source job id from ref-offre fallback"))
normalized_source_job_id = ref_source_job_id
else: else:
ref_source_job_id = _extract_source_job_id(soup, None) normalized_source_job_id = source_job_id
if ref_source_job_id is None: else:
warnings.append(_warning("source_job_id", "Source job id missing from ref-offre")) if ref_source_job_id is None:
if requested_source_job_id is None:
warnings.append(_warning("source_job_id", "Source job id missing from Apec listing"))
normalized_source_job_id = None
else: else:
warnings.append(_warning("source_job_id", "Recovered source job id from ref-offre fallback")) warnings.append(_warning("source_job_id", "Recovered source job id from detail URL fallback"))
normalized_source_job_id = requested_source_job_id
else:
warnings.append(_warning("source_job_id", "Recovered source job id from ref-offre fallback"))
normalized_source_job_id = ref_source_job_id normalized_source_job_id = ref_source_job_id
company = soup.select_one(".card-ents .ents-name") company = soup.select_one(".card-ents .ents-name")

View File

@ -466,6 +466,59 @@ def test_search_records_pagination_render_failures(monkeypatch) -> None:
assert adapter.search_errors[0].message == "page 1 results did not render" assert adapter.search_errors[0].message == "page 1 results did not render"
def test_search_records_evaluate_all_failures_and_continues_to_next_query(monkeypatch) -> None:
first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"
second_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=beta&page=0&selectedIndex=0"
page = _FakeDetailPage(
{
"alpha": {
0: [first_result],
1: [first_result],
},
"beta": {
0: [second_result],
1: [second_result],
},
}
)
original_locator = page.locator
class _FlakyLocator:
def __init__(self, locator) -> None:
self._locator = locator
def evaluate_all(self, function: str):
if (
self._locator.selector == _RESULT_LINK_SELECTOR
and page.current_query == "alpha"
and page.current_page == 1
):
raise RuntimeError("evaluate boom")
return self._locator.evaluate_all(function)
def __getattr__(self, name: str):
return getattr(self._locator, name)
def flaky_locator(selector: str):
return _FlakyLocator(original_locator(selector))
monkeypatch.setattr(page, "locator", flaky_locator)
monkeypatch.setattr(adapter_module, "_MAX_CONSECUTIVE_NO_PROGRESS_PAGES", 1)
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
adapter = ApecAdapter(max_listings=10)
results = adapter.search(
["alpha", "beta"],
search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
)
assert [result.url for result in results] == [first_result, second_result]
assert [error.stage for error in adapter.search_errors] == ["search"]
assert "page=1" in adapter.search_errors[0].url
def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> None: def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> None:
page = _FakeDetailPage({}, rendered_html="<html>rendered offer</html>") page = _FakeDetailPage({}, rendered_html="<html>rendered offer</html>")
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page)) monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))

View File

@ -71,6 +71,45 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None:
assert listing.fetched_at == "2026-06-01T10:00:00Z" assert listing.fetched_at == "2026-06-01T10:00:00Z"
def test_normalize_apec_listing_prefers_final_source_job_id_from_detail_page() -> None:
html = """
<html>
<body>
<main class="container-details-offer">
<h1>Data Engineer F/H</h1>
<div class="card-offer">
<div class="ref-offre">Ref. Apec : FINAL456</div>
<ul class="details-offer-list mb-20">
<li>CLOUD TEMPLE</li>
<li>1 <span> CDI </span></li>
<li>Puteaux - 92</li>
</ul>
<p>Publiée le 20/04/2026 Actualisée le 02/06/2026</p>
</div>
<article class="card card-ents mb-20">
<div class="list-hzt mb-20">
<span class="ents-name">CLOUD TEMPLE</span>
</div>
</article>
<div class="details-post">
<h4>Descriptif du poste</h4>
<p>Build pipelines</p>
</div>
</main>
</body>
</html>
"""
listing = normalize_apec_listing(
url="https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/REQUESTED123",
html=html,
fetched_at="2026-06-01T10:00:00Z",
source_job_id="REQUESTED123",
)
assert listing.source_job_id == "FINAL456"
def test_normalize_apec_listing_warns_and_returns_none_for_invalid_dates() -> None: def test_normalize_apec_listing_warns_and_returns_none_for_invalid_dates() -> None:
html = """ html = """
<html> <html>