diff --git a/src/job_research/apec/adapter.py b/src/job_research/apec/adapter.py index 639efc7..a7e7636 100644 --- a/src/job_research/apec/adapter.py +++ b/src/job_research/apec/adapter.py @@ -133,8 +133,15 @@ class ApecAdapter: with _open_public_page() as page: page.goto(url, wait_until="domcontentloaded") - page.wait_for_selector(".card-offer .ref-offre", timeout=15_000) - page.wait_for_selector(".details-offer-list", timeout=15_000) - page.wait_for_selector(".details-post", timeout=15_000) - page.wait_for_timeout(5_000) + page.wait_for_selector(".details-post:has-text('Descriptif du poste')", timeout=15_000) + page.wait_for_function( + """ + () => { + const company = document.querySelector('.card-ents .ents-name, .card-ents-quote'); + return !!company && (company.textContent || '').trim().length > 0; + } + """, + polling=1000, + timeout=15_000, + ) return page.content() diff --git a/src/job_research/apec/normalize.py b/src/job_research/apec/normalize.py index 142b6b2..622aea7 100644 --- a/src/job_research/apec/normalize.py +++ b/src/job_research/apec/normalize.py @@ -4,6 +4,7 @@ import re from datetime import datetime from bs4 import BeautifulSoup +from bs4.element import NavigableString from job_research.models import ApecListing @@ -11,6 +12,7 @@ from job_research.models import ApecListing _PUBLISHED_AT_PATTERN = re.compile(r"Publi[ée]e le (\d{2}/\d{2}/\d{4})") _SOURCE_JOB_ID_PATTERN = re.compile(r"Ref\. Apec\s*:\s*([A-Z0-9]+)") _CONTRACT_PATTERN = re.compile(r"\b(CDI|CDD|Alternance|Intérim|Stage|Freelance|Indépendant)\b") +_HEADING_TAG_NAMES = {"h1", "h2", "h3", "h4", "h5", "h6"} def _clean_text(value: str | None) -> str | None: @@ -21,23 +23,19 @@ def _clean_text(value: str | None) -> str | None: return cleaned or None -def _extract_section_text(block, label: str) -> str | None: - started = False - pieces: list[str] = [] - for child in block.children: - if getattr(child, "name", None) == "h4": - heading = _clean_text(child.get_text(" ", strip=True)) - if not started: - if heading == label: - started = True - continue +def _text_before_heading(node) -> str | None: + if isinstance(node, NavigableString): + return _clean_text(str(node)) + if getattr(node, "name", None) in _HEADING_TAG_NAMES: + return None + + pieces: list[str] = [] + for child in getattr(node, "children", []): + if getattr(child, "name", None) in _HEADING_TAG_NAMES: break - if not started: - continue - - text = child.get_text(" ", strip=True) if hasattr(child, "get_text") else str(child) + text = _text_before_heading(child) cleaned = _clean_text(text) if cleaned: pieces.append(cleaned) @@ -45,6 +43,23 @@ def _extract_section_text(block, label: str) -> str | None: return _clean_text(" ".join(pieces)) +def _extract_section_text(block, label: str) -> str | None: + heading = block.find(lambda tag: getattr(tag, "name", None) in _HEADING_TAG_NAMES and _clean_text(tag.get_text(" ", strip=True)) == label) + if heading is None: + return None + + pieces: list[str] = [] + for sibling in heading.next_siblings: + if getattr(sibling, "name", None) in _HEADING_TAG_NAMES: + break + + text = _text_before_heading(sibling) + if text: + pieces.append(text) + + return _clean_text(" ".join(pieces)) + + def _detail_block_text(soup: BeautifulSoup, label: str) -> str | None: for block in soup.select(".details-post"): if block.find("h4") is None: @@ -100,6 +115,23 @@ def _extract_contract_type(details_offer_list) -> str | None: return match.group(1) +def _extract_company(soup: BeautifulSoup, details_offer_list) -> str | None: + company = soup.select_one(".card-ents .ents-name") + if company is not None: + return _clean_text(company.get_text(" ", strip=True)) + + company = soup.select_one(".card-ents-quote") + if company is not None: + return _clean_text(company.get_text(" ", strip=True)) + + if details_offer_list is not None: + company = details_offer_list.select_one("li:first-of-type") + if company is not None: + return _clean_text(company.get_text(" ", strip=True)) + + return None + + def normalize_apec_listing( url: str, html: str, @@ -113,10 +145,6 @@ def normalize_apec_listing( title = soup.select_one(".container-details-offer h1") or soup.find("h1") details_offer_list = soup.select_one(".details-offer-list") - company = soup.select_one(".card-ents .ents-name, .card-ents-quote") - if company is None and details_offer_list is not None: - company = details_offer_list.select_one("li:nth-of-type(1)") - location = None contract_type = None if details_offer_list is not None: @@ -130,7 +158,7 @@ def normalize_apec_listing( source_job_id=_extract_source_job_id(soup, source_job_id), url=url, title=_clean_text(title.get_text(" ", strip=True)) if title else None, - company=_clean_text(company.get_text(" ", strip=True)) if company else None, + company=_extract_company(soup, details_offer_list), location=_clean_text(location.get_text(" ", strip=True)) if location else None, contract_type=contract_type, description_text=description_text or None, diff --git a/tests/apec/test_adapter.py b/tests/apec/test_adapter.py index f3ce05a..fc1f88a 100644 --- a/tests/apec/test_adapter.py +++ b/tests/apec/test_adapter.py @@ -46,7 +46,7 @@ class _FakeDetailPage: self.rendered_html = rendered_html self.shell_html = "shell" self.waited_selectors: list[str] = [] - self.waited_timeouts: list[int] = [] + self.waited_functions: list[tuple[str, int | None]] = [] self.goto_urls: list[str] = [] self.current_query = "" self.current_page = 0 @@ -70,16 +70,21 @@ class _FakeDetailPage: def wait_for_load_state(self, state: str) -> None: return None - def wait_for_timeout(self, timeout: int) -> None: - self.waited_timeouts.append(timeout) + def wait_for_function(self, function: str, polling: int | None = None, timeout: int | None = None) -> None: + self.waited_functions.append((function, polling)) self.rendered = True return None def wait_for_selector(self, selector: str, timeout: int | None = None) -> None: self.waited_selectors.append(selector) - if selector in {".card-ents .ents-name", ".card-offer .ref-offre", ".details-offer-list", ".details-post"}: - self.rendered = True + if selector in { + ".card-ents .ents-name", + ".card-offer .ref-offre", + ".details-offer-list", + ".details-post", + ".details-post:has-text('Descriptif du poste')", + }: return None if selector == _RESULT_LINK_SELECTOR and self.current_results(): @@ -133,11 +138,11 @@ def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> Non ) assert html == "rendered offer" - assert page.waited_timeouts == [5000] + assert len(page.waited_functions) == 1 + assert ".card-ents .ents-name" in page.waited_functions[0][0] + assert page.waited_functions[0][1] == 1000 assert page.waited_selectors == [ - ".card-offer .ref-offre", - ".details-offer-list", - ".details-post", + ".details-post:has-text('Descriptif du poste')", ] diff --git a/tests/apec/test_normalize.py b/tests/apec/test_normalize.py index f7184df..966414c 100644 --- a/tests/apec/test_normalize.py +++ b/tests/apec/test_normalize.py @@ -9,11 +9,11 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None:

Data Engineer F/H

Ref. Apec : 178554452W
-
+
+

Publiée le 20/04/2026 Actualisée le 02/06/2026

@@ -21,10 +21,22 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None: CLOUD TEMPLE
-
-
-

Descriptif du poste

-

Build pipelines

+
+

Salaire

+ A partir de 70 k€ brut annuel +
+
+

Prise de poste

+ Dès que possible +
+
+

Expérience

+ Minimum 7 ans +
+
+

Descriptif du poste

+

Build pipelines

+

Profil recherché

Python / SQL

Compétences attendues

diff --git a/tests/test_apec_cli.py b/tests/test_apec_cli.py index a2b2d3c..c35b3da 100644 --- a/tests/test_apec_cli.py +++ b/tests/test_apec_cli.py @@ -33,11 +33,11 @@ def _apec_detail_html(
Ref. Apec : {source_job_id}
-
+
  • {company}
  • 1 {contract}
  • {location}
  • -
+

Publiée le {published_at} Actualisée le {updated_at}

@@ -46,10 +46,22 @@ def _apec_detail_html( {company}
-
-
-

Descriptif du poste

-

{description}

+
+

Salaire

+ A partir de 70 k€ brut annuel +
+
+

Prise de poste

+ Dès que possible +
+
+

Expérience

+ Minimum 7 ans +
+
+

Descriptif du poste

+

{description}

+

Profil recherché

{profile}

Compétences attendues