fix: harden Apec search and canonicalization
This commit is contained in:
Antoine
parent
ed1af201bc
commit
85c4278db3
@ -245,9 +245,19 @@ class ApecAdapter:
|
||||
|
||||
seen_page_urls.add(current_page_url)
|
||||
|
||||
hrefs = page.locator(_RESULT_LINK_SELECTOR).evaluate_all(
|
||||
"nodes => nodes.map(node => node.href)"
|
||||
)
|
||||
try:
|
||||
hrefs = page.locator(_RESULT_LINK_SELECTOR).evaluate_all(
|
||||
"nodes => nodes.map(node => node.href)"
|
||||
)
|
||||
except Exception:
|
||||
self._record_search_error(
|
||||
query,
|
||||
search_filters,
|
||||
f"page {page_number} result links could not be evaluated",
|
||||
url=current_page_url,
|
||||
)
|
||||
break
|
||||
|
||||
if not hrefs:
|
||||
no_progress_pages += 1
|
||||
if no_progress_pages >= _MAX_CONSECUTIVE_NO_PROGRESS_PAGES:
|
||||
|
||||
@ -12,6 +12,7 @@ from job_research.models import ApecListing, ListingWarning
|
||||
|
||||
_PUBLISHED_AT_PATTERN = re.compile(r"Publi[ée]e le (\d{2}/\d{2}/\d{4})")
|
||||
_REFRESHED_AT_PATTERN = re.compile(r"Actualis[ée]e le (\d{2}/\d{2}/\d{4})")
|
||||
_DETAIL_JOB_ID_PATTERN = re.compile(r"/detail-offre/([^/?#]+)")
|
||||
_SOURCE_JOB_ID_PATTERN = re.compile(r"Ref\. Apec\s*:\s*([A-Z0-9]+)")
|
||||
_CONTRACT_PATTERN = re.compile(r"\b(CDI|CDD|Alternance|Intérim|Stage|Freelance|Indépendant)\b")
|
||||
_HEADING_TAG_NAMES = {"h1", "h2", "h3", "h4", "h5", "h6"}
|
||||
@ -103,6 +104,14 @@ def _warning(field: str, message: str) -> ListingWarning:
|
||||
return ListingWarning(field=field, message=message)
|
||||
|
||||
|
||||
def _extract_source_job_id_from_url(url: str) -> str | None:
|
||||
match = _DETAIL_JOB_ID_PATTERN.search(url)
|
||||
if match is None:
|
||||
return None
|
||||
|
||||
return match.group(1)
|
||||
|
||||
|
||||
def _extract_listing_date(
|
||||
soup: BeautifulSoup,
|
||||
pattern: re.Pattern[str],
|
||||
@ -132,10 +141,7 @@ def _extract_listing_date(
|
||||
return None
|
||||
|
||||
|
||||
def _extract_source_job_id(soup: BeautifulSoup, source_job_id: str | None) -> str | None:
|
||||
if source_job_id is not None:
|
||||
return source_job_id
|
||||
|
||||
def _extract_source_job_id(soup: BeautifulSoup) -> str | None:
|
||||
ref = soup.select_one(".ref-offre")
|
||||
if ref is None:
|
||||
return None
|
||||
@ -244,19 +250,29 @@ def normalize_apec_listing(
|
||||
warnings.append(_warning("description_text", "Description is empty or placeholder text"))
|
||||
description_text = None
|
||||
|
||||
requested_source_job_id = _extract_source_job_id_from_url(url)
|
||||
ref_source_job_id = _extract_source_job_id(soup)
|
||||
|
||||
if source_job_id is not None:
|
||||
normalized_source_job_id = source_job_id
|
||||
else:
|
||||
ref = soup.select_one(".ref-offre")
|
||||
if ref is None:
|
||||
warnings.append(_warning("source_job_id", "Source job id missing from Apec listing"))
|
||||
normalized_source_job_id = None
|
||||
if (
|
||||
requested_source_job_id is not None
|
||||
and ref_source_job_id is not None
|
||||
and requested_source_job_id != ref_source_job_id
|
||||
):
|
||||
warnings.append(_warning("source_job_id", "Recovered source job id from ref-offre fallback"))
|
||||
normalized_source_job_id = ref_source_job_id
|
||||
else:
|
||||
ref_source_job_id = _extract_source_job_id(soup, None)
|
||||
if ref_source_job_id is None:
|
||||
warnings.append(_warning("source_job_id", "Source job id missing from ref-offre"))
|
||||
normalized_source_job_id = source_job_id
|
||||
else:
|
||||
if ref_source_job_id is None:
|
||||
if requested_source_job_id is None:
|
||||
warnings.append(_warning("source_job_id", "Source job id missing from Apec listing"))
|
||||
normalized_source_job_id = None
|
||||
else:
|
||||
warnings.append(_warning("source_job_id", "Recovered source job id from ref-offre fallback"))
|
||||
warnings.append(_warning("source_job_id", "Recovered source job id from detail URL fallback"))
|
||||
normalized_source_job_id = requested_source_job_id
|
||||
else:
|
||||
warnings.append(_warning("source_job_id", "Recovered source job id from ref-offre fallback"))
|
||||
normalized_source_job_id = ref_source_job_id
|
||||
|
||||
company = soup.select_one(".card-ents .ents-name")
|
||||
|
||||
@ -466,6 +466,59 @@ def test_search_records_pagination_render_failures(monkeypatch) -> None:
|
||||
assert adapter.search_errors[0].message == "page 1 results did not render"
|
||||
|
||||
|
||||
def test_search_records_evaluate_all_failures_and_continues_to_next_query(monkeypatch) -> None:
|
||||
first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"
|
||||
second_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=beta&page=0&selectedIndex=0"
|
||||
page = _FakeDetailPage(
|
||||
{
|
||||
"alpha": {
|
||||
0: [first_result],
|
||||
1: [first_result],
|
||||
},
|
||||
"beta": {
|
||||
0: [second_result],
|
||||
1: [second_result],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
original_locator = page.locator
|
||||
|
||||
class _FlakyLocator:
|
||||
def __init__(self, locator) -> None:
|
||||
self._locator = locator
|
||||
|
||||
def evaluate_all(self, function: str):
|
||||
if (
|
||||
self._locator.selector == _RESULT_LINK_SELECTOR
|
||||
and page.current_query == "alpha"
|
||||
and page.current_page == 1
|
||||
):
|
||||
raise RuntimeError("evaluate boom")
|
||||
|
||||
return self._locator.evaluate_all(function)
|
||||
|
||||
def __getattr__(self, name: str):
|
||||
return getattr(self._locator, name)
|
||||
|
||||
def flaky_locator(selector: str):
|
||||
return _FlakyLocator(original_locator(selector))
|
||||
|
||||
monkeypatch.setattr(page, "locator", flaky_locator)
|
||||
monkeypatch.setattr(adapter_module, "_MAX_CONSECUTIVE_NO_PROGRESS_PAGES", 1)
|
||||
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
||||
|
||||
adapter = ApecAdapter(max_listings=10)
|
||||
results = adapter.search(
|
||||
["alpha", "beta"],
|
||||
search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
|
||||
)
|
||||
|
||||
assert [result.url for result in results] == [first_result, second_result]
|
||||
assert [error.stage for error in adapter.search_errors] == ["search"]
|
||||
assert "page=1" in adapter.search_errors[0].url
|
||||
|
||||
|
||||
def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> None:
|
||||
page = _FakeDetailPage({}, rendered_html="<html>rendered offer</html>")
|
||||
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
||||
|
||||
@ -71,6 +71,45 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None:
|
||||
assert listing.fetched_at == "2026-06-01T10:00:00Z"
|
||||
|
||||
|
||||
def test_normalize_apec_listing_prefers_final_source_job_id_from_detail_page() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<body>
|
||||
<main class="container-details-offer">
|
||||
<h1>Data Engineer F/H</h1>
|
||||
<div class="card-offer">
|
||||
<div class="ref-offre">Ref. Apec : FINAL456</div>
|
||||
<ul class="details-offer-list mb-20">
|
||||
<li>CLOUD TEMPLE</li>
|
||||
<li>1 <span> CDI </span></li>
|
||||
<li>Puteaux - 92</li>
|
||||
</ul>
|
||||
<p>Publiée le 20/04/2026 Actualisée le 02/06/2026</p>
|
||||
</div>
|
||||
<article class="card card-ents mb-20">
|
||||
<div class="list-hzt mb-20">
|
||||
<span class="ents-name">CLOUD TEMPLE</span>
|
||||
</div>
|
||||
</article>
|
||||
<div class="details-post">
|
||||
<h4>Descriptif du poste</h4>
|
||||
<p>Build pipelines</p>
|
||||
</div>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
listing = normalize_apec_listing(
|
||||
url="https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/REQUESTED123",
|
||||
html=html,
|
||||
fetched_at="2026-06-01T10:00:00Z",
|
||||
source_job_id="REQUESTED123",
|
||||
)
|
||||
|
||||
assert listing.source_job_id == "FINAL456"
|
||||
|
||||
|
||||
def test_normalize_apec_listing_warns_and_returns_none_for_invalid_dates() -> None:
|
||||
html = """
|
||||
<html>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user