fix: harden Apec search and canonicalization
This commit is contained in:
parent
ed1af201bc
commit
85c4278db3
@ -245,9 +245,19 @@ class ApecAdapter:
|
|||||||
|
|
||||||
seen_page_urls.add(current_page_url)
|
seen_page_urls.add(current_page_url)
|
||||||
|
|
||||||
hrefs = page.locator(_RESULT_LINK_SELECTOR).evaluate_all(
|
try:
|
||||||
"nodes => nodes.map(node => node.href)"
|
hrefs = page.locator(_RESULT_LINK_SELECTOR).evaluate_all(
|
||||||
)
|
"nodes => nodes.map(node => node.href)"
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
self._record_search_error(
|
||||||
|
query,
|
||||||
|
search_filters,
|
||||||
|
f"page {page_number} result links could not be evaluated",
|
||||||
|
url=current_page_url,
|
||||||
|
)
|
||||||
|
break
|
||||||
|
|
||||||
if not hrefs:
|
if not hrefs:
|
||||||
no_progress_pages += 1
|
no_progress_pages += 1
|
||||||
if no_progress_pages >= _MAX_CONSECUTIVE_NO_PROGRESS_PAGES:
|
if no_progress_pages >= _MAX_CONSECUTIVE_NO_PROGRESS_PAGES:
|
||||||
|
|||||||
@ -12,6 +12,7 @@ from job_research.models import ApecListing, ListingWarning
|
|||||||
|
|
||||||
_PUBLISHED_AT_PATTERN = re.compile(r"Publi[ée]e le (\d{2}/\d{2}/\d{4})")
|
_PUBLISHED_AT_PATTERN = re.compile(r"Publi[ée]e le (\d{2}/\d{2}/\d{4})")
|
||||||
_REFRESHED_AT_PATTERN = re.compile(r"Actualis[ée]e le (\d{2}/\d{2}/\d{4})")
|
_REFRESHED_AT_PATTERN = re.compile(r"Actualis[ée]e le (\d{2}/\d{2}/\d{4})")
|
||||||
|
_DETAIL_JOB_ID_PATTERN = re.compile(r"/detail-offre/([^/?#]+)")
|
||||||
_SOURCE_JOB_ID_PATTERN = re.compile(r"Ref\. Apec\s*:\s*([A-Z0-9]+)")
|
_SOURCE_JOB_ID_PATTERN = re.compile(r"Ref\. Apec\s*:\s*([A-Z0-9]+)")
|
||||||
_CONTRACT_PATTERN = re.compile(r"\b(CDI|CDD|Alternance|Intérim|Stage|Freelance|Indépendant)\b")
|
_CONTRACT_PATTERN = re.compile(r"\b(CDI|CDD|Alternance|Intérim|Stage|Freelance|Indépendant)\b")
|
||||||
_HEADING_TAG_NAMES = {"h1", "h2", "h3", "h4", "h5", "h6"}
|
_HEADING_TAG_NAMES = {"h1", "h2", "h3", "h4", "h5", "h6"}
|
||||||
@ -103,6 +104,14 @@ def _warning(field: str, message: str) -> ListingWarning:
|
|||||||
return ListingWarning(field=field, message=message)
|
return ListingWarning(field=field, message=message)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_source_job_id_from_url(url: str) -> str | None:
|
||||||
|
match = _DETAIL_JOB_ID_PATTERN.search(url)
|
||||||
|
if match is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return match.group(1)
|
||||||
|
|
||||||
|
|
||||||
def _extract_listing_date(
|
def _extract_listing_date(
|
||||||
soup: BeautifulSoup,
|
soup: BeautifulSoup,
|
||||||
pattern: re.Pattern[str],
|
pattern: re.Pattern[str],
|
||||||
@ -132,10 +141,7 @@ def _extract_listing_date(
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _extract_source_job_id(soup: BeautifulSoup, source_job_id: str | None) -> str | None:
|
def _extract_source_job_id(soup: BeautifulSoup) -> str | None:
|
||||||
if source_job_id is not None:
|
|
||||||
return source_job_id
|
|
||||||
|
|
||||||
ref = soup.select_one(".ref-offre")
|
ref = soup.select_one(".ref-offre")
|
||||||
if ref is None:
|
if ref is None:
|
||||||
return None
|
return None
|
||||||
@ -244,19 +250,29 @@ def normalize_apec_listing(
|
|||||||
warnings.append(_warning("description_text", "Description is empty or placeholder text"))
|
warnings.append(_warning("description_text", "Description is empty or placeholder text"))
|
||||||
description_text = None
|
description_text = None
|
||||||
|
|
||||||
|
requested_source_job_id = _extract_source_job_id_from_url(url)
|
||||||
|
ref_source_job_id = _extract_source_job_id(soup)
|
||||||
|
|
||||||
if source_job_id is not None:
|
if source_job_id is not None:
|
||||||
normalized_source_job_id = source_job_id
|
if (
|
||||||
else:
|
requested_source_job_id is not None
|
||||||
ref = soup.select_one(".ref-offre")
|
and ref_source_job_id is not None
|
||||||
if ref is None:
|
and requested_source_job_id != ref_source_job_id
|
||||||
warnings.append(_warning("source_job_id", "Source job id missing from Apec listing"))
|
):
|
||||||
normalized_source_job_id = None
|
warnings.append(_warning("source_job_id", "Recovered source job id from ref-offre fallback"))
|
||||||
|
normalized_source_job_id = ref_source_job_id
|
||||||
else:
|
else:
|
||||||
ref_source_job_id = _extract_source_job_id(soup, None)
|
normalized_source_job_id = source_job_id
|
||||||
if ref_source_job_id is None:
|
else:
|
||||||
warnings.append(_warning("source_job_id", "Source job id missing from ref-offre"))
|
if ref_source_job_id is None:
|
||||||
|
if requested_source_job_id is None:
|
||||||
|
warnings.append(_warning("source_job_id", "Source job id missing from Apec listing"))
|
||||||
|
normalized_source_job_id = None
|
||||||
else:
|
else:
|
||||||
warnings.append(_warning("source_job_id", "Recovered source job id from ref-offre fallback"))
|
warnings.append(_warning("source_job_id", "Recovered source job id from detail URL fallback"))
|
||||||
|
normalized_source_job_id = requested_source_job_id
|
||||||
|
else:
|
||||||
|
warnings.append(_warning("source_job_id", "Recovered source job id from ref-offre fallback"))
|
||||||
normalized_source_job_id = ref_source_job_id
|
normalized_source_job_id = ref_source_job_id
|
||||||
|
|
||||||
company = soup.select_one(".card-ents .ents-name")
|
company = soup.select_one(".card-ents .ents-name")
|
||||||
|
|||||||
@ -466,6 +466,59 @@ def test_search_records_pagination_render_failures(monkeypatch) -> None:
|
|||||||
assert adapter.search_errors[0].message == "page 1 results did not render"
|
assert adapter.search_errors[0].message == "page 1 results did not render"
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_records_evaluate_all_failures_and_continues_to_next_query(monkeypatch) -> None:
|
||||||
|
first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"
|
||||||
|
second_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=beta&page=0&selectedIndex=0"
|
||||||
|
page = _FakeDetailPage(
|
||||||
|
{
|
||||||
|
"alpha": {
|
||||||
|
0: [first_result],
|
||||||
|
1: [first_result],
|
||||||
|
},
|
||||||
|
"beta": {
|
||||||
|
0: [second_result],
|
||||||
|
1: [second_result],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
original_locator = page.locator
|
||||||
|
|
||||||
|
class _FlakyLocator:
|
||||||
|
def __init__(self, locator) -> None:
|
||||||
|
self._locator = locator
|
||||||
|
|
||||||
|
def evaluate_all(self, function: str):
|
||||||
|
if (
|
||||||
|
self._locator.selector == _RESULT_LINK_SELECTOR
|
||||||
|
and page.current_query == "alpha"
|
||||||
|
and page.current_page == 1
|
||||||
|
):
|
||||||
|
raise RuntimeError("evaluate boom")
|
||||||
|
|
||||||
|
return self._locator.evaluate_all(function)
|
||||||
|
|
||||||
|
def __getattr__(self, name: str):
|
||||||
|
return getattr(self._locator, name)
|
||||||
|
|
||||||
|
def flaky_locator(selector: str):
|
||||||
|
return _FlakyLocator(original_locator(selector))
|
||||||
|
|
||||||
|
monkeypatch.setattr(page, "locator", flaky_locator)
|
||||||
|
monkeypatch.setattr(adapter_module, "_MAX_CONSECUTIVE_NO_PROGRESS_PAGES", 1)
|
||||||
|
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
||||||
|
|
||||||
|
adapter = ApecAdapter(max_listings=10)
|
||||||
|
results = adapter.search(
|
||||||
|
["alpha", "beta"],
|
||||||
|
search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert [result.url for result in results] == [first_result, second_result]
|
||||||
|
assert [error.stage for error in adapter.search_errors] == ["search"]
|
||||||
|
assert "page=1" in adapter.search_errors[0].url
|
||||||
|
|
||||||
|
|
||||||
def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> None:
|
def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> None:
|
||||||
page = _FakeDetailPage({}, rendered_html="<html>rendered offer</html>")
|
page = _FakeDetailPage({}, rendered_html="<html>rendered offer</html>")
|
||||||
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
||||||
|
|||||||
@ -71,6 +71,45 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None:
|
|||||||
assert listing.fetched_at == "2026-06-01T10:00:00Z"
|
assert listing.fetched_at == "2026-06-01T10:00:00Z"
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_apec_listing_prefers_final_source_job_id_from_detail_page() -> None:
|
||||||
|
html = """
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<main class="container-details-offer">
|
||||||
|
<h1>Data Engineer F/H</h1>
|
||||||
|
<div class="card-offer">
|
||||||
|
<div class="ref-offre">Ref. Apec : FINAL456</div>
|
||||||
|
<ul class="details-offer-list mb-20">
|
||||||
|
<li>CLOUD TEMPLE</li>
|
||||||
|
<li>1 <span> CDI </span></li>
|
||||||
|
<li>Puteaux - 92</li>
|
||||||
|
</ul>
|
||||||
|
<p>Publiée le 20/04/2026 Actualisée le 02/06/2026</p>
|
||||||
|
</div>
|
||||||
|
<article class="card card-ents mb-20">
|
||||||
|
<div class="list-hzt mb-20">
|
||||||
|
<span class="ents-name">CLOUD TEMPLE</span>
|
||||||
|
</div>
|
||||||
|
</article>
|
||||||
|
<div class="details-post">
|
||||||
|
<h4>Descriptif du poste</h4>
|
||||||
|
<p>Build pipelines</p>
|
||||||
|
</div>
|
||||||
|
</main>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
listing = normalize_apec_listing(
|
||||||
|
url="https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/REQUESTED123",
|
||||||
|
html=html,
|
||||||
|
fetched_at="2026-06-01T10:00:00Z",
|
||||||
|
source_job_id="REQUESTED123",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert listing.source_job_id == "FINAL456"
|
||||||
|
|
||||||
|
|
||||||
def test_normalize_apec_listing_warns_and_returns_none_for_invalid_dates() -> None:
|
def test_normalize_apec_listing_warns_and_returns_none_for_invalid_dates() -> None:
|
||||||
html = """
|
html = """
|
||||||
<html>
|
<html>
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user