tonio/job-research
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: tighten Apec description extraction

Browse Source
This commit is contained in:
Antoine 2026-06-02 20:25:27 +02:00
parent c2ca1f457e
commit ac1e0fb269
5 changed files with 77 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/job_research/apec/adapter.py
View File

@ -136,4 +136,5 @@ class ApecAdapter:
page.wait_for_selector(".card-offer .ref-offre", timeout=15_000)
page.wait_for_selector(".details-offer-list", timeout=15_000)
page.wait_for_selector(".details-post", timeout=15_000)
page.wait_for_timeout(5_000)
return page.content()

24
src/job_research/apec/normalize.py
View File

@ -21,10 +21,20 @@ def _clean_text(value: str | None) -> str | None:
return cleaned or None
def _extract_block_value(block) -> str | None:
def _extract_section_text(block, label: str) -> str | None:
started = False
pieces: list[str] = []
for child in block.children:
if getattr(child, "name", None) == "h4":
heading = _clean_text(child.get_text(" ", strip=True))
if not started:
if heading == label:
started = True
continue
break
if not started:
continue
text = child.get_text(" ", strip=True) if hasattr(child, "get_text") else str(child)
@ -37,14 +47,12 @@ def _extract_block_value(block) -> str | None:
def _detail_block_text(soup: BeautifulSoup, label: str) -> str | None:
for block in soup.select(".details-post"):
heading = block.find("h4")
if heading is None:
if block.find("h4") is None:
continue
if _clean_text(heading.get_text(" ", strip=True)) != label:
continue
return _extract_block_value(block)
extracted = _extract_section_text(block, label)
if extracted is not None:
return extracted
return None
@ -105,7 +113,7 @@ def normalize_apec_listing(
title = soup.select_one(".container-details-offer h1") or soup.find("h1")
details_offer_list = soup.select_one(".details-offer-list")
company = soup.select_one(".card-ents-quote, .card-ents .ents-name")
company = soup.select_one(".card-ents .ents-name, .card-ents-quote")
if company is None and details_offer_list is not None:
company = details_offer_list.select_one("li:nth-of-type(1)")

15
tests/apec/test_adapter.py
View File

@ -46,6 +46,7 @@ class _FakeDetailPage:
self.rendered_html = rendered_html
self.shell_html = "<html>shell</html>"
self.waited_selectors: list[str] = []
self.waited_timeouts: list[int] = []
self.goto_urls: list[str] = []
self.current_query = ""
self.current_page = 0
@ -69,10 +70,15 @@ class _FakeDetailPage:
def wait_for_load_state(self, state: str) -> None:
return None
def wait_for_timeout(self, timeout: int) -> None:
self.waited_timeouts.append(timeout)
self.rendered = True
return None
def wait_for_selector(self, selector: str, timeout: int | None = None) -> None:
self.waited_selectors.append(selector)
if selector in {".card-offer .ref-offre", ".details-offer-list", ".details-post"}:
if selector in {".card-ents .ents-name", ".card-offer .ref-offre", ".details-offer-list", ".details-post"}:
self.rendered = True
return None
@ -127,7 +133,12 @@ def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> Non
)
assert html == "<html>rendered offer</html>"
assert page.waited_selectors == [".card-offer .ref-offre", ".details-offer-list", ".details-post"]
assert page.waited_timeouts == [5000]
assert page.waited_selectors == [
".card-offer .ref-offre",
".details-offer-list",
".details-post",
]
def test_fetch_listing_html_rejects_non_apec_hosts() -> None:

51
tests/apec/test_normalize.py
View File

@ -3,27 +3,36 @@ from job_research.apec.normalize import normalize_apec_listing
def test_normalize_apec_listing_extracts_minimal_shape() -> None:
html = """
<html>
<body>
<main class="container-details-offer">
<h1>Data Engineer F/H</h1>
<div class="card-offer">
<div class="ref-offre">Ref. Apec : 178554452W</div>
<div class="details-offer-list">
<li>CLOUD TEMPLE</li>
<li>1 <span> CDI </span></li>
<li>Puteaux - 92</li>
</div>
<p>Publiée le 20/04/2026 Actualisée le 02/06/2026</p>
</div>
<div class="card-ents-quote">CLOUD TEMPLE</div>
<div class="details-post">
<h4>Descriptif du poste</h4>
<p>Build pipelines</p>
</div>
<div class="details-post">
<h4>Profil recherché</h4>
<p>Python / SQL</p>
<html>
<body>
<main class="container-details-offer">
<h1>Data Engineer F/H</h1>
<div class="card-offer">
<div class="ref-offre">Ref. Apec : 178554452W</div>
<div class="details-offer-list">
<li>CLOUD TEMPLE</li>
<li>1 <span> CDI </span></li>
<li>Puteaux - 92</li>
</div>
<p>Publiée le 20/04/2026 Actualisée le 02/06/2026</p>
</div>
<article class="card card-ents mb-20">
<div class="list-hzt mb-20">
<span class="ents-name">CLOUD TEMPLE</span>
</div>
</article>
<div class="col-lg-8 border-L">
<div class="details-post">
<h4>Descriptif du poste</h4>
<p>Build pipelines</p>
<h4>Profil recherché</h4>
<p>Python / SQL</p>
<h4>Compétences attendues</h4>
<p>Ignored</p>
<h4>Entreprise</h4>
<p>Ignored</p>
<div class="recruiter">Ignored recruiter info</div>
</div>
</div>
</main>
</body>

25
tests/test_apec_cli.py
View File

@ -41,14 +41,23 @@ def _apec_detail_html(
<p>Publiée le {published_at} Actualisée le {updated_at}</p>
</div>
</article>
<div class="card-ents-quote">{company}</div>
<div class="details-post">
<h4>Descriptif du poste</h4>
<p>{description}</p>
</div>
<div class="details-post">
<h4>Profil recherché</h4>
<p>{profile}</p>
<article class="card card-ents mb-20">
<div class="list-hzt mb-20">
<span class="ents-name">{company}</span>
</div>
</article>
<div class="col-lg-8 border-L">
<div class="details-post">
<h4>Descriptif du poste</h4>
<p>{description}</p>
<h4>Profil recherché</h4>
<p>{profile}</p>
<h4>Compétences attendues</h4>
<p>Ignored</p>
<h4>Entreprise</h4>
<p>Ignored</p>
<div class="recruiter">Ignored recruiter info</div>
</div>
</div>
</main>
</body>