tonio/job-research
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: align Apec live detail shape

Browse Source
This commit is contained in:
Antoine 2026-06-02 20:40:19 +02:00
parent ac1e0fb269
commit 47f912ce8c
5 changed files with 108 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/job_research/apec/adapter.py
View File

@ -133,8 +133,15 @@ class ApecAdapter:
with _open_public_page() as page:
page.goto(url, wait_until="domcontentloaded")
page.wait_for_selector(".card-offer .ref-offre", timeout=15_000)
page.wait_for_selector(".details-offer-list", timeout=15_000)
page.wait_for_selector(".details-post", timeout=15_000)
page.wait_for_timeout(5_000)
page.wait_for_selector(".details-post:has-text('Descriptif du poste')", timeout=15_000)
page.wait_for_function(
"""
() => {
const company = document.querySelector('.card-ents .ents-name, .card-ents-quote');
return !!company && (company.textContent || '').trim().length > 0;
}
""",
polling=1000,
timeout=15_000,
)
return page.content()

66
src/job_research/apec/normalize.py
View File

@ -4,6 +4,7 @@ import re
from datetime import datetime
from bs4 import BeautifulSoup
from bs4.element import NavigableString
from job_research.models import ApecListing
@ -11,6 +12,7 @@ from job_research.models import ApecListing
_PUBLISHED_AT_PATTERN = re.compile(r"Publi[ée]e le (\d{2}/\d{2}/\d{4})")
_SOURCE_JOB_ID_PATTERN = re.compile(r"Ref\. Apec\s*:\s*([A-Z0-9]+)")
_CONTRACT_PATTERN = re.compile(r"\b(CDI|CDD|Alternance|Intérim|Stage|Freelance|Indépendant)\b")
_HEADING_TAG_NAMES = {"h1", "h2", "h3", "h4", "h5", "h6"}
def _clean_text(value: str | None) -> str | None:
@ -21,23 +23,19 @@ def _clean_text(value: str | None) -> str | None:
return cleaned or None
def _extract_section_text(block, label: str) -> str | None:
started = False
pieces: list[str] = []
for child in block.children:
if getattr(child, "name", None) == "h4":
heading = _clean_text(child.get_text(" ", strip=True))
if not started:
if heading == label:
started = True
continue
def _text_before_heading(node) -> str | None:
if isinstance(node, NavigableString):
return _clean_text(str(node))
if getattr(node, "name", None) in _HEADING_TAG_NAMES:
return None
pieces: list[str] = []
for child in getattr(node, "children", []):
if getattr(child, "name", None) in _HEADING_TAG_NAMES:
break
if not started:
continue
text = child.get_text(" ", strip=True) if hasattr(child, "get_text") else str(child)
text = _text_before_heading(child)
cleaned = _clean_text(text)
if cleaned:
pieces.append(cleaned)
@ -45,6 +43,23 @@ def _extract_section_text(block, label: str) -> str | None:
return _clean_text(" ".join(pieces))
def _extract_section_text(block, label: str) -> str | None:
heading = block.find(lambda tag: getattr(tag, "name", None) in _HEADING_TAG_NAMES and _clean_text(tag.get_text(" ", strip=True)) == label)
if heading is None:
return None
pieces: list[str] = []
for sibling in heading.next_siblings:
if getattr(sibling, "name", None) in _HEADING_TAG_NAMES:
break
text = _text_before_heading(sibling)
if text:
pieces.append(text)
return _clean_text(" ".join(pieces))
def _detail_block_text(soup: BeautifulSoup, label: str) -> str | None:
for block in soup.select(".details-post"):
if block.find("h4") is None:
@ -100,6 +115,23 @@ def _extract_contract_type(details_offer_list) -> str | None:
return match.group(1)
def _extract_company(soup: BeautifulSoup, details_offer_list) -> str | None:
company = soup.select_one(".card-ents .ents-name")
if company is not None:
return _clean_text(company.get_text(" ", strip=True))
company = soup.select_one(".card-ents-quote")
if company is not None:
return _clean_text(company.get_text(" ", strip=True))
if details_offer_list is not None:
company = details_offer_list.select_one("li:first-of-type")
if company is not None:
return _clean_text(company.get_text(" ", strip=True))
return None
def normalize_apec_listing(
url: str,
html: str,
@ -113,10 +145,6 @@ def normalize_apec_listing(
title = soup.select_one(".container-details-offer h1") or soup.find("h1")
details_offer_list = soup.select_one(".details-offer-list")
company = soup.select_one(".card-ents .ents-name, .card-ents-quote")
if company is None and details_offer_list is not None:
company = details_offer_list.select_one("li:nth-of-type(1)")
location = None
contract_type = None
if details_offer_list is not None:
@ -130,7 +158,7 @@ def normalize_apec_listing(
source_job_id=_extract_source_job_id(soup, source_job_id),
url=url,
title=_clean_text(title.get_text(" ", strip=True)) if title else None,
company=_clean_text(company.get_text(" ", strip=True)) if company else None,
company=_extract_company(soup, details_offer_list),
location=_clean_text(location.get_text(" ", strip=True)) if location else None,
contract_type=contract_type,
description_text=description_text or None,

23
tests/apec/test_adapter.py
View File

@ -46,7 +46,7 @@ class _FakeDetailPage:
self.rendered_html = rendered_html
self.shell_html = "<html>shell</html>"
self.waited_selectors: list[str] = []
self.waited_timeouts: list[int] = []
self.waited_functions: list[tuple[str, int | None]] = []
self.goto_urls: list[str] = []
self.current_query = ""
self.current_page = 0
@ -70,16 +70,21 @@ class _FakeDetailPage:
def wait_for_load_state(self, state: str) -> None:
return None
def wait_for_timeout(self, timeout: int) -> None:
self.waited_timeouts.append(timeout)
def wait_for_function(self, function: str, polling: int | None = None, timeout: int | None = None) -> None:
self.waited_functions.append((function, polling))
self.rendered = True
return None
def wait_for_selector(self, selector: str, timeout: int | None = None) -> None:
self.waited_selectors.append(selector)
if selector in {".card-ents .ents-name", ".card-offer .ref-offre", ".details-offer-list", ".details-post"}:
self.rendered = True
if selector in {
".card-ents .ents-name",
".card-offer .ref-offre",
".details-offer-list",
".details-post",
".details-post:has-text('Descriptif du poste')",
}:
return None
if selector == _RESULT_LINK_SELECTOR and self.current_results():
@ -133,11 +138,11 @@ def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> Non
)
assert html == "<html>rendered offer</html>"
assert page.waited_timeouts == [5000]
assert len(page.waited_functions) == 1
assert ".card-ents .ents-name" in page.waited_functions[0][0]
assert page.waited_functions[0][1] == 1000
assert page.waited_selectors == [
".card-offer .ref-offre",
".details-offer-list",
".details-post",
".details-post:has-text('Descriptif du poste')",
]

24
tests/apec/test_normalize.py
View File

@ -9,11 +9,11 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None:
<h1>Data Engineer F/H</h1>
<div class="card-offer">
<div class="ref-offre">Ref. Apec : 178554452W</div>
<div class="details-offer-list">
<ul class="details-offer-list mb-20">
<li>CLOUD TEMPLE</li>
<li>1 <span> CDI </span></li>
<li>Puteaux - 92</li>
</div>
</ul>
<p>Publiée le 20/04/2026 Actualisée le 02/06/2026</p>
</div>
<article class="card card-ents mb-20">
@ -21,10 +21,22 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None:
<span class="ents-name">CLOUD TEMPLE</span>
</div>
</article>
<div class="col-lg-8 border-L">
<div class="details-post">
<h4>Descriptif du poste</h4>
<p>Build pipelines</p>
<div class="details-post">
<h4>Salaire</h4>
<span>A partir de 70 k€ brut annuel</span>
</div>
<div class="details-post">
<h4>Prise de poste</h4>
<span>Dès que possible</span>
</div>
<div class="details-post">
<h4>Expérience</h4>
<span>Minimum 7 ans</span>
</div>
<div class="details-post">
<h4>Descriptif du poste</h4>
<p>Build pipelines</p>
<div class="nested-late-sections">
<h4>Profil recherché</h4>
<p>Python / SQL</p>
<h4>Compétences attendues</h4>

24
tests/test_apec_cli.py
View File

@ -33,11 +33,11 @@ def _apec_detail_html(
<article class="card card-offer">
<div class="ref-offre">Ref. Apec : {source_job_id}</div>
<div class="card-offer__header">
<div class="details-offer-list">
<ul class="details-offer-list mb-20">
<li>{company}</li>
<li>1 <span> {contract} </span></li>
<li>{location}</li>
</div>
</ul>
<p>Publiée le {published_at} Actualisée le {updated_at}</p>
</div>
</article>
@ -46,10 +46,22 @@ def _apec_detail_html(
<span class="ents-name">{company}</span>
</div>
</article>
<div class="col-lg-8 border-L">
<div class="details-post">
<h4>Descriptif du poste</h4>
<p>{description}</p>
<div class="details-post">
<h4>Salaire</h4>
<span>A partir de 70 k€ brut annuel</span>
</div>
<div class="details-post">
<h4>Prise de poste</h4>
<span>Dès que possible</span>
</div>
<div class="details-post">
<h4>Expérience</h4>
<span>Minimum 7 ans</span>
</div>
<div class="details-post">
<h4>Descriptif du poste</h4>
<p>{description}</p>
<div class="nested-late-sections">
<h4>Profil recherché</h4>
<p>{profile}</p>
<h4>Compétences attendues</h4>