fix: harden Apec live ingestion
This commit is contained in:
Antoine
parent
2da39613c7
commit
1c267520f0
@ -12,6 +12,7 @@ from playwright.sync_api import sync_playwright
|
||||
_SEARCH_URL = "https://www.apec.fr/candidat/recherche-emploi.html/emploi"
|
||||
_RESULT_LINK_SELECTOR = "a[href*='/candidat/recherche-emploi.html/emploi/detail-offre/']"
|
||||
_DETAIL_JOB_ID_PATTERN = re.compile(r"/detail-offre/([^/?#]+)")
|
||||
_APEC_HOSTS = {"apec.fr", "www.apec.fr"}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@ -55,6 +56,16 @@ def _accept_cookies_if_present(page) -> None:
|
||||
return
|
||||
|
||||
|
||||
def _is_public_apec_detail_url(url: str) -> bool:
|
||||
parsed_url = urlparse(url)
|
||||
return (
|
||||
parsed_url.scheme == "https"
|
||||
and parsed_url.hostname in _APEC_HOSTS
|
||||
and re.fullmatch(r"/candidat/recherche-emploi\.html/emploi/detail-offre/[^/?#]+", parsed_url.path)
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
class ApecAdapter:
|
||||
def __init__(self, max_listings: int = 50) -> None:
|
||||
self.max_listings = max_listings
|
||||
@ -100,7 +111,6 @@ class ApecAdapter:
|
||||
if not hrefs:
|
||||
break
|
||||
|
||||
new_results_on_page = 0
|
||||
for href in hrefs:
|
||||
source_job_id = _extract_source_job_id(href)
|
||||
dedupe_key = source_job_id or href
|
||||
@ -109,23 +119,21 @@ class ApecAdapter:
|
||||
|
||||
seen_keys.add(dedupe_key)
|
||||
results.append(ApecSearchResult(url=href, source_job_id=source_job_id))
|
||||
new_results_on_page += 1
|
||||
|
||||
if len(results) >= self.max_listings:
|
||||
break
|
||||
|
||||
if new_results_on_page == 0:
|
||||
break
|
||||
|
||||
page_number += 1
|
||||
|
||||
return results
|
||||
|
||||
def fetch_listing_html(self, url: str) -> str:
|
||||
parsed_url = urlparse(url)
|
||||
if not parsed_url.netloc.endswith("apec.fr"):
|
||||
if not _is_public_apec_detail_url(url):
|
||||
raise ValueError("ApecAdapter only fetches public Apec URLs")
|
||||
|
||||
with _open_public_page() as page:
|
||||
page.goto(url, wait_until="domcontentloaded")
|
||||
page.wait_for_selector(".card-offer .ref-offre", timeout=15_000)
|
||||
page.wait_for_selector(".details-offer-list", timeout=15_000)
|
||||
page.wait_for_selector(".details-post", timeout=15_000)
|
||||
return page.content()
|
||||
|
||||
@ -1,8 +1,97 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from job_research.models import ApecListing
|
||||
|
||||
|
||||
_PUBLISHED_AT_PATTERN = re.compile(r"Publi[ée]e le (\d{2}/\d{2}/\d{4})")
|
||||
_SOURCE_JOB_ID_PATTERN = re.compile(r"Ref\. Apec\s*:\s*([A-Z0-9]+)")
|
||||
_CONTRACT_PATTERN = re.compile(r"\b(CDI|CDD|Alternance|Intérim|Stage|Freelance|Indépendant)\b")
|
||||
|
||||
|
||||
def _clean_text(value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
cleaned = " ".join(value.split())
|
||||
return cleaned or None
|
||||
|
||||
|
||||
def _extract_block_value(block) -> str | None:
|
||||
pieces: list[str] = []
|
||||
for child in block.children:
|
||||
if getattr(child, "name", None) == "h4":
|
||||
continue
|
||||
|
||||
text = child.get_text(" ", strip=True) if hasattr(child, "get_text") else str(child)
|
||||
cleaned = _clean_text(text)
|
||||
if cleaned:
|
||||
pieces.append(cleaned)
|
||||
|
||||
return _clean_text(" ".join(pieces))
|
||||
|
||||
|
||||
def _detail_block_text(soup: BeautifulSoup, label: str) -> str | None:
|
||||
for block in soup.select(".details-post"):
|
||||
heading = block.find("h4")
|
||||
if heading is None:
|
||||
continue
|
||||
|
||||
if _clean_text(heading.get_text(" ", strip=True)) != label:
|
||||
continue
|
||||
|
||||
return _extract_block_value(block)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _extract_published_at(soup: BeautifulSoup) -> str | None:
|
||||
card_offer = soup.select_one(".card-offer")
|
||||
if card_offer is None:
|
||||
return None
|
||||
|
||||
match = _PUBLISHED_AT_PATTERN.search(card_offer.get_text(" ", strip=True))
|
||||
if match is None:
|
||||
return None
|
||||
|
||||
return datetime.strptime(match.group(1), "%d/%m/%Y").date().isoformat()
|
||||
|
||||
|
||||
def _extract_source_job_id(soup: BeautifulSoup, source_job_id: str | None) -> str | None:
|
||||
if source_job_id is not None:
|
||||
return source_job_id
|
||||
|
||||
ref = soup.select_one(".ref-offre")
|
||||
if ref is None:
|
||||
return None
|
||||
|
||||
match = _SOURCE_JOB_ID_PATTERN.search(ref.get_text(" ", strip=True))
|
||||
if match is None:
|
||||
return None
|
||||
|
||||
return match.group(1)
|
||||
|
||||
|
||||
def _extract_contract_type(details_offer_list) -> str | None:
|
||||
contract_item = details_offer_list.select_one("li:nth-of-type(2)")
|
||||
if contract_item is None:
|
||||
return None
|
||||
|
||||
span = contract_item.find("span")
|
||||
if span is not None:
|
||||
return _clean_text(span.get_text(" ", strip=True))
|
||||
|
||||
match = _CONTRACT_PATTERN.search(contract_item.get_text(" ", strip=True))
|
||||
if match is None:
|
||||
return None
|
||||
|
||||
return match.group(1)
|
||||
|
||||
|
||||
def normalize_apec_listing(
|
||||
url: str,
|
||||
html: str,
|
||||
@ -13,21 +102,34 @@ def normalize_apec_listing(
|
||||
) -> ApecListing:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
title = soup.find("h1")
|
||||
company = soup.select_one(".company")
|
||||
location = soup.select_one(".location")
|
||||
contract = soup.select_one(".contract")
|
||||
description = soup.select_one(".description")
|
||||
title = soup.select_one(".container-details-offer h1") or soup.find("h1")
|
||||
details_offer_list = soup.select_one(".details-offer-list")
|
||||
|
||||
company = soup.select_one(".card-ents .ents-name")
|
||||
if company is None and details_offer_list is not None:
|
||||
company = details_offer_list.select_one("li:nth-of-type(1)")
|
||||
|
||||
location = None
|
||||
contract_type = None
|
||||
if details_offer_list is not None:
|
||||
location = details_offer_list.select_one("li:nth-of-type(3)")
|
||||
contract_type = _extract_contract_type(details_offer_list)
|
||||
|
||||
description_sections = [
|
||||
_detail_block_text(soup, "Descriptif du poste"),
|
||||
_detail_block_text(soup, "Profil recherché"),
|
||||
]
|
||||
description_text = "\n\n".join(section for section in description_sections if section)
|
||||
|
||||
return ApecListing(
|
||||
source="apec",
|
||||
source_job_id=source_job_id,
|
||||
source_job_id=_extract_source_job_id(soup, source_job_id),
|
||||
url=url,
|
||||
title=title.get_text(" ", strip=True) if title else None,
|
||||
company=company.get_text(" ", strip=True) if company else None,
|
||||
location=location.get_text(" ", strip=True) if location else None,
|
||||
contract_type=contract.get_text(" ", strip=True) if contract else None,
|
||||
description_text=description.get_text(" ", strip=True) if description else None,
|
||||
published_at=published_at,
|
||||
title=_clean_text(title.get_text(" ", strip=True)) if title else None,
|
||||
company=_clean_text(company.get_text(" ", strip=True)) if company else None,
|
||||
location=_clean_text(location.get_text(" ", strip=True)) if location else None,
|
||||
contract_type=contract_type,
|
||||
description_text=description_text or None,
|
||||
published_at=published_at or _extract_published_at(soup),
|
||||
fetched_at=fetched_at,
|
||||
)
|
||||
|
||||
137
tests/apec/test_adapter.py
Normal file
137
tests/apec/test_adapter.py
Normal file
@ -0,0 +1,137 @@
|
||||
from contextlib import contextmanager
|
||||
from urllib.parse import parse_qs, quote_plus, urlparse
|
||||
|
||||
import pytest
|
||||
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
|
||||
|
||||
from job_research.apec import adapter as adapter_module
|
||||
from job_research.apec.adapter import ApecAdapter
|
||||
|
||||
|
||||
_RESULT_LINK_SELECTOR = "a[href*='/candidat/recherche-emploi.html/emploi/detail-offre/']"
|
||||
|
||||
|
||||
class _FakeResultButton:
|
||||
def __init__(self, page, name: str) -> None:
|
||||
self.page = page
|
||||
self.name = name
|
||||
|
||||
def click(self, timeout: int | None = None) -> None:
|
||||
if self.name == "Rechercher":
|
||||
self.page.url = (
|
||||
"https://www.apec.fr/candidat/recherche-emploi.html/emploi"
|
||||
f"?motsCles={quote_plus(self.page.current_query)}&page=0"
|
||||
)
|
||||
self.page.current_page = 0
|
||||
|
||||
|
||||
class _FakeLocator:
|
||||
def __init__(self, page, selector: str) -> None:
|
||||
self.page = page
|
||||
self.selector = selector
|
||||
|
||||
def fill(self, value: str) -> None:
|
||||
self.page.current_query = value
|
||||
|
||||
def evaluate_all(self, function: str):
|
||||
if self.selector == _RESULT_LINK_SELECTOR:
|
||||
return list(self.page.current_results())
|
||||
|
||||
return []
|
||||
|
||||
|
||||
class _FakeDetailPage:
|
||||
def __init__(self, result_pages: dict[str, dict[int, list[str]]], *, rendered_html: str = "<html>rendered</html>") -> None:
|
||||
self.result_pages = result_pages
|
||||
self.rendered_html = rendered_html
|
||||
self.shell_html = "<html>shell</html>"
|
||||
self.waited_selectors: list[str] = []
|
||||
self.goto_urls: list[str] = []
|
||||
self.current_query = ""
|
||||
self.current_page = 0
|
||||
self.url = ""
|
||||
self.rendered = False
|
||||
|
||||
def goto(self, url: str, wait_until: str | None = None) -> None:
|
||||
self.goto_urls.append(url)
|
||||
self.url = url
|
||||
|
||||
parsed_url = urlparse(url)
|
||||
params = parse_qs(parsed_url.query)
|
||||
if "motsCles" in params:
|
||||
self.current_query = params["motsCles"][0]
|
||||
if "page" in params:
|
||||
self.current_page = int(params["page"][0])
|
||||
|
||||
if "/detail-offre/" in parsed_url.path:
|
||||
self.rendered = False
|
||||
|
||||
def wait_for_load_state(self, state: str) -> None:
|
||||
return None
|
||||
|
||||
def wait_for_selector(self, selector: str, timeout: int | None = None) -> None:
|
||||
self.waited_selectors.append(selector)
|
||||
|
||||
if selector in {".card-offer .ref-offre", ".details-offer-list", ".details-post"}:
|
||||
self.rendered = True
|
||||
return None
|
||||
|
||||
if selector == _RESULT_LINK_SELECTOR and self.current_results():
|
||||
return None
|
||||
|
||||
raise PlaywrightTimeoutError(f"selector not found: {selector}")
|
||||
|
||||
def get_by_role(self, role: str, name: str):
|
||||
return _FakeResultButton(self, name)
|
||||
|
||||
def locator(self, selector: str):
|
||||
return _FakeLocator(self, selector)
|
||||
|
||||
def content(self) -> str:
|
||||
return self.rendered_html if self.rendered else self.shell_html
|
||||
|
||||
def current_results(self) -> list[str]:
|
||||
return self.result_pages.get(self.current_query, {}).get(self.current_page, [])
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _fake_open_public_page(page: _FakeDetailPage):
|
||||
yield page
|
||||
|
||||
|
||||
def test_search_continues_past_duplicate_only_pages(monkeypatch) -> None:
|
||||
first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"
|
||||
second_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=beta&page=1&selectedIndex=0"
|
||||
page = _FakeDetailPage(
|
||||
{
|
||||
"alpha": {0: [first_result], 1: []},
|
||||
"beta": {0: [first_result], 1: [second_result], 2: []},
|
||||
}
|
||||
)
|
||||
|
||||
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
||||
|
||||
results = ApecAdapter(max_listings=10).search(["alpha", "beta"])
|
||||
|
||||
assert [result.url for result in results] == [first_result, second_result]
|
||||
assert [result.source_job_id for result in results] == ["111", "222"]
|
||||
assert any("page=1" in url for url in page.goto_urls)
|
||||
|
||||
|
||||
def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> None:
|
||||
page = _FakeDetailPage({}, rendered_html="<html>rendered offer</html>")
|
||||
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
||||
|
||||
html = ApecAdapter().fetch_listing_html(
|
||||
"https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111"
|
||||
)
|
||||
|
||||
assert html == "<html>rendered offer</html>"
|
||||
assert page.waited_selectors == [".card-offer .ref-offre", ".details-offer-list", ".details-post"]
|
||||
|
||||
|
||||
def test_fetch_listing_html_rejects_non_apec_hosts() -> None:
|
||||
adapter = ApecAdapter()
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
adapter.fetch_listing_html("https://evilapec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111")
|
||||
@ -5,11 +5,29 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<body>
|
||||
<h1><span>Data</span> <strong>Engineer</strong></h1>
|
||||
<div class="company"><span>Example</span> <em>Corp</em></div>
|
||||
<div class="location">Paris</div>
|
||||
<div class="contract">CDI</div>
|
||||
<div class="description">Build pipelines</div>
|
||||
<main class="container-details-offer">
|
||||
<h1>Data Engineer F/H</h1>
|
||||
<div class="card-offer">
|
||||
<div class="ref-offre">Ref. Apec : 178554452W</div>
|
||||
<div class="details-offer-list">
|
||||
<li>CLOUD TEMPLE</li>
|
||||
<li>1 <span> CDI </span></li>
|
||||
<li>Puteaux - 92</li>
|
||||
</div>
|
||||
<p>Publiée le 20/04/2026 Actualisée le 02/06/2026</p>
|
||||
</div>
|
||||
<div class="card-ents">
|
||||
<span class="ents-name">CLOUD TEMPLE</span>
|
||||
</div>
|
||||
<div class="details-post">
|
||||
<h4>Descriptif du poste</h4>
|
||||
<p>Build pipelines</p>
|
||||
</div>
|
||||
<div class="details-post">
|
||||
<h4>Profil recherché</h4>
|
||||
<p>Python / SQL</p>
|
||||
</div>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
@ -19,16 +37,15 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None:
|
||||
html=html,
|
||||
fetched_at="2026-06-01T10:00:00Z",
|
||||
source_job_id="job-123",
|
||||
published_at="2026-05-31",
|
||||
)
|
||||
|
||||
assert listing.source == "apec"
|
||||
assert listing.source_job_id == "job-123"
|
||||
assert listing.url == "https://example.test/job/123"
|
||||
assert listing.title == "Data Engineer"
|
||||
assert listing.company == "Example Corp"
|
||||
assert listing.location == "Paris"
|
||||
assert listing.title == "Data Engineer F/H"
|
||||
assert listing.company == "CLOUD TEMPLE"
|
||||
assert listing.location == "Puteaux - 92"
|
||||
assert listing.contract_type == "CDI"
|
||||
assert listing.description_text == "Build pipelines"
|
||||
assert listing.published_at == "2026-05-31"
|
||||
assert listing.description_text == "Build pipelines\n\nPython / SQL"
|
||||
assert listing.published_at == "2026-04-20"
|
||||
assert listing.fetched_at == "2026-06-01T10:00:00Z"
|
||||
|
||||
@ -12,6 +12,53 @@ def _fixed_now() -> datetime:
|
||||
return datetime(2026, 6, 1, 10, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _apec_detail_html(
|
||||
*,
|
||||
title: str = "Role From YAML",
|
||||
company: str = "Example Corp",
|
||||
location: str = "Paris - 75",
|
||||
contract: str = "CDI",
|
||||
description: str = "Build pipelines",
|
||||
profile: str = "Python / SQL",
|
||||
source_job_id: str = "178554452W",
|
||||
published_at: str = "20/04/2026",
|
||||
updated_at: str = "02/06/2026",
|
||||
) -> str:
|
||||
return dedent(
|
||||
f"""
|
||||
<html>
|
||||
<body>
|
||||
<main class="container-details-offer">
|
||||
<h1>{title}</h1>
|
||||
<article class="card card-offer">
|
||||
<div class="ref-offre">Ref. Apec : {source_job_id}</div>
|
||||
<div class="card-offer__header">
|
||||
<div class="details-offer-list">
|
||||
<li>{company}</li>
|
||||
<li>1 <span> {contract} </span></li>
|
||||
<li>{location}</li>
|
||||
</div>
|
||||
<p>Publiée le {published_at} Actualisée le {updated_at}</p>
|
||||
</div>
|
||||
</article>
|
||||
<div class="card-ents">
|
||||
<span class="ents-name">{company}</span>
|
||||
</div>
|
||||
<div class="details-post">
|
||||
<h4>Descriptif du poste</h4>
|
||||
<p>{description}</p>
|
||||
</div>
|
||||
<div class="details-post">
|
||||
<h4>Profil recherché</h4>
|
||||
<p>{profile}</p>
|
||||
</div>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
).strip()
|
||||
|
||||
|
||||
def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch) -> None:
|
||||
data_root = tmp_path / "data"
|
||||
data_root.mkdir()
|
||||
@ -25,19 +72,7 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
html = dedent(
|
||||
"""
|
||||
<html>
|
||||
<body>
|
||||
<h1>Role From YAML</h1>
|
||||
<div class="company">Example Corp</div>
|
||||
<div class="location">Paris</div>
|
||||
<div class="contract">CDI</div>
|
||||
<div class="description">Build pipelines</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
).strip()
|
||||
html = _apec_detail_html()
|
||||
|
||||
class FakeApecAdapter:
|
||||
instances: list["FakeApecAdapter"] = []
|
||||
@ -94,10 +129,10 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch
|
||||
"url": "https://example.test/job/123",
|
||||
"title": "Role From YAML",
|
||||
"company": "Example Corp",
|
||||
"location": "Paris",
|
||||
"location": "Paris - 75",
|
||||
"contract_type": "CDI",
|
||||
"description_text": "Build pipelines",
|
||||
"published_at": None,
|
||||
"description_text": "Build pipelines\n\nPython / SQL",
|
||||
"published_at": "2026-04-20",
|
||||
"fetched_at": "2026-06-01T10:00:00Z",
|
||||
"warnings": [],
|
||||
}
|
||||
@ -130,19 +165,7 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings(
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
html = dedent(
|
||||
"""
|
||||
<html>
|
||||
<body>
|
||||
<h1>Role From YAML</h1>
|
||||
<div class="company">Example Corp</div>
|
||||
<div class="location">Paris</div>
|
||||
<div class="contract">CDI</div>
|
||||
<div class="description">Build pipelines</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
).strip()
|
||||
html = _apec_detail_html()
|
||||
good_result = ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123")
|
||||
bad_result = ApecSearchResult(url="https://example.test/job/456", source_job_id="job-456")
|
||||
|
||||
@ -194,10 +217,10 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings(
|
||||
"url": "https://example.test/job/123",
|
||||
"title": "Role From YAML",
|
||||
"company": "Example Corp",
|
||||
"location": "Paris",
|
||||
"location": "Paris - 75",
|
||||
"contract_type": "CDI",
|
||||
"description_text": "Build pipelines",
|
||||
"published_at": None,
|
||||
"description_text": "Build pipelines\n\nPython / SQL",
|
||||
"published_at": "2026-04-20",
|
||||
"fetched_at": "2026-06-01T10:00:00Z",
|
||||
"warnings": [],
|
||||
}
|
||||
@ -262,19 +285,7 @@ def test_fetch_apec_processes_only_the_first_fifty_search_results(tmp_path, monk
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
html = dedent(
|
||||
"""
|
||||
<html>
|
||||
<body>
|
||||
<h1>Role From YAML</h1>
|
||||
<div class="company">Example Corp</div>
|
||||
<div class="location">Paris</div>
|
||||
<div class="contract">CDI</div>
|
||||
<div class="description">Build pipelines</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
).strip()
|
||||
html = _apec_detail_html()
|
||||
results = [
|
||||
ApecSearchResult(url=f"https://example.test/job/{index}", source_job_id=f"job-{index}")
|
||||
for index in range(51)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user