tonio/job-research
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: record Apec pagination failures

Browse Source
This commit is contained in:
Antoine 2026-06-05 12:35:14 +02:00
parent 53b4ac0ea3
commit 1dfaea3508
4 changed files with 169 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/job_research/apec/adapter.py
View File

@ -145,9 +145,16 @@ class ApecAdapter:
finally:
page.close()
def _record_search_error(self, query: str, search_filters: ApecSearchFilters, message: str) -> None:
def _record_search_error(
self,
query: str,
search_filters: ApecSearchFilters,
message: str,
*,
url: str | None = None,
) -> None:
self.search_errors.append(
ListingError(url=_search_url(query, search_filters), stage="search", message=message)
ListingError(url=url or _search_url(query, search_filters), stage="search", message=message)
)
@staticmethod
@ -206,11 +213,23 @@ class ApecAdapter:
break
if not _goto_and_wait(page, next_page_url):
self._record_search_error(
query,
search_filters,
f"page {page_number} navigation failed",
url=next_page_url,
)
break
try:
page.wait_for_selector(_RESULT_LINK_SELECTOR, timeout=5_000)
except PlaywrightTimeoutError:
self._record_search_error(
query,
search_filters,
f"page {page_number} results did not render",
url=next_page_url,
)
break
current_page_url = page.url

41
src/job_research/apec/normalize.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import re
from datetime import datetime
import unicodedata
from bs4 import BeautifulSoup
from bs4.element import NavigableString
@ -13,6 +14,17 @@ _PUBLISHED_AT_PATTERN = re.compile(r"Publi[ée]e le (\d{2}/\d{2}/\d{4})")
_SOURCE_JOB_ID_PATTERN = re.compile(r"Ref\. Apec\s*:\s*([A-Z0-9]+)")
_CONTRACT_PATTERN = re.compile(r"\b(CDI|CDD|Alternance|Intérim|Stage|Freelance|Indépendant)\b")
_HEADING_TAG_NAMES = {"h1", "h2", "h3", "h4", "h5", "h6"}
_PLACEHOLDER_TEXT_TOKENS = {
"na",
"nr",
"none",
"null",
"unknown",
"tbd",
"nonrenseigne",
"nonrenseignee",
"nondisponible",
}
def _clean_text(value: str | None) -> str | None:
@ -23,6 +35,20 @@ def _clean_text(value: str | None) -> str | None:
return cleaned or None
def _text_token(value: str) -> str:
normalized = unicodedata.normalize("NFKD", value)
return re.sub(r"[^a-z0-9]+", "", normalized.casefold())
def _has_useful_text(value: str | None) -> bool:
cleaned = _clean_text(value)
if cleaned is None:
return False
token = _text_token(cleaned)
return bool(token) and token not in _PLACEHOLDER_TEXT_TOKENS
def _text_before_heading(node) -> str | None:
if isinstance(node, NavigableString):
return _clean_text(str(node))
@ -155,6 +181,9 @@ def normalize_apec_listing(
warnings.append(_warning("title", "Recovered title from generic h1 fallback"))
else:
warnings.append(_warning("title", "Title missing from Apec listing"))
title_text = _clean_text(title.get_text(" ", strip=True)) if title is not None else None
if title is not None and not _has_useful_text(title_text):
warnings.append(_warning("title", "Title is empty or placeholder text"))
details_offer_list = soup.select_one(".details-offer-list")
@ -163,7 +192,9 @@ def normalize_apec_listing(
if details_offer_list is not None:
location_item = details_offer_list.select_one("li:nth-of-type(3)")
if location_item is not None:
location = location_item.get_text(" ", strip=True)
location = _clean_text(location_item.get_text(" ", strip=True))
if not _has_useful_text(location):
warnings.append(_warning("location", "Location is empty or placeholder text"))
else:
warnings.append(_warning("location", "Location missing from details-offer list"))
@ -174,6 +205,8 @@ def normalize_apec_listing(
span = contract_item.find("span")
if span is not None:
contract_type = _clean_text(span.get_text(" ", strip=True))
if not _has_useful_text(contract_type):
warnings.append(_warning("contract_type", "Contract type is empty or placeholder text"))
else:
match = _CONTRACT_PATTERN.search(contract_item.get_text(" ", strip=True))
if match is not None:
@ -227,10 +260,10 @@ def normalize_apec_listing(
source="apec",
source_job_id=normalized_source_job_id,
url=url,
title=_clean_text(title.get_text(" ", strip=True)) if title else None,
title=title_text if _has_useful_text(title_text) else None,
company=company_text,
location=_clean_text(location) if location else None,
contract_type=contract_type,
location=location if _has_useful_text(location) else None,
contract_type=contract_type if _has_useful_text(contract_type) else None,
description_text=description_text or None,
published_at=published_at_value,
fetched_at=fetched_at,

69
tests/apec/test_adapter.py
View File

@ -363,6 +363,75 @@ def test_search_raises_when_every_query_renders_broken_search_shell(monkeypatch)
assert [error.stage for error in adapter.search_errors] == ["search", "search"]
def test_search_records_pagination_navigation_failures(monkeypatch) -> None:
first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"
page = _FakeDetailPage(
{
"alpha": {
0: [first_result],
1: ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=alpha&page=1&selectedIndex=0"],
},
}
)
original_goto = page.goto
def flaky_goto(url: str, wait_until: str | None = None) -> None:
if "page=1" in url:
raise RuntimeError("navigation boom")
original_goto(url, wait_until=wait_until)
monkeypatch.setattr(page, "goto", flaky_goto)
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
adapter = ApecAdapter(max_listings=10)
results = adapter.search(
["alpha"],
search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
)
assert [result.url for result in results] == [first_result]
assert [error.stage for error in adapter.search_errors] == ["search"]
assert "page=1" in adapter.search_errors[0].url
assert adapter.search_errors[0].message == "page 1 navigation failed"
def test_search_records_pagination_render_failures(monkeypatch) -> None:
first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"
second_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=alpha&page=1&selectedIndex=0"
page = _FakeDetailPage(
{
"alpha": {
0: [first_result],
1: [second_result],
},
}
)
original_wait_for_selector = page.wait_for_selector
def flaky_wait_for_selector(selector: str, timeout: int | None = None) -> None:
if selector == _RESULT_LINK_SELECTOR and page.current_page == 1:
raise PlaywrightTimeoutError(f"selector not found: {selector}")
original_wait_for_selector(selector, timeout=timeout)
monkeypatch.setattr(page, "wait_for_selector", flaky_wait_for_selector)
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
adapter = ApecAdapter(max_listings=10)
results = adapter.search(
["alpha"],
search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
)
assert [result.url for result in results] == [first_result]
assert [error.stage for error in adapter.search_errors] == ["search"]
assert "page=1" in adapter.search_errors[0].url
assert adapter.search_errors[0].message == "page 1 results did not render"
def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> None:
page = _FakeDetailPage({}, rendered_html="<html>rendered offer</html>")
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))

42
tests/apec/test_normalize.py
View File

@ -165,3 +165,45 @@ def test_normalize_apec_listing_records_warnings_for_fallback_and_missing_fields
"company",
"published_at",
]
def test_normalize_apec_listing_records_warnings_for_placeholder_text_values() -> None:
html = """
<html>
<body>
<main class="container-details-offer">
<h1> N/A </h1>
<div class="card-offer">
<div class="ref-offre">Ref. Apec : 178554452W</div>
<ul class="details-offer-list mb-20">
<li>Example Corp</li>
<li>1 <span> N/A </span></li>
<li> - </li>
</ul>
<p>Publiée le 20/04/2026 Actualisée le 02/06/2026</p>
</div>
<article class="card card-ents mb-20">
<div class="list-hzt mb-20">
<span class="ents-name">Example Corp</span>
</div>
</article>
<div class="details-post">
<h4>Descriptif du poste</h4>
<p>Build pipelines</p>
</div>
</main>
</body>
</html>
"""
listing = normalize_apec_listing(
url="https://example.test/job/123",
html=html,
fetched_at="2026-06-01T10:00:00Z",
source_job_id="178554452W",
)
assert listing.title is None
assert listing.location is None
assert listing.contract_type is None
assert [warning.field for warning in listing.warnings] == ["title", "location", "contract_type"]