fix: harden Apec readiness and snapshot writes
This commit is contained in:
Antoine
parent
47f912ce8c
commit
c28f804e23
@ -133,12 +133,16 @@ class ApecAdapter:
|
||||
|
||||
with _open_public_page() as page:
|
||||
page.goto(url, wait_until="domcontentloaded")
|
||||
page.wait_for_selector(".details-post:has-text('Descriptif du poste')", timeout=15_000)
|
||||
page.wait_for_function(
|
||||
"""
|
||||
() => {
|
||||
const company = document.querySelector('.card-ents .ents-name, .card-ents-quote');
|
||||
return !!company && (company.textContent || '').trim().length > 0;
|
||||
const title = document.querySelector('.container-details-offer h1, h1');
|
||||
const company = document.querySelector('.card-ents .ents-name, .card-ents-quote, .details-offer-list li:first-of-type');
|
||||
const offerList = document.querySelector('.details-offer-list');
|
||||
const descriptionReady = Array.from(document.querySelectorAll('.details-post h4')).some((heading) => {
|
||||
return (heading.textContent || '').trim() === 'Descriptif du poste';
|
||||
});
|
||||
return !!title && !!offerList && !!company && (company.textContent || '').trim().length > 0 && descriptionReady;
|
||||
}
|
||||
""",
|
||||
polling=1000,
|
||||
|
||||
@ -116,18 +116,19 @@ def _extract_contract_type(details_offer_list) -> str | None:
|
||||
|
||||
|
||||
def _extract_company(soup: BeautifulSoup, details_offer_list) -> str | None:
|
||||
company = soup.select_one(".card-ents .ents-name")
|
||||
if company is not None:
|
||||
return _clean_text(company.get_text(" ", strip=True))
|
||||
|
||||
company = soup.select_one(".card-ents-quote")
|
||||
if company is not None:
|
||||
return _clean_text(company.get_text(" ", strip=True))
|
||||
for selector in (".card-ents .ents-name", ".card-ents-quote"):
|
||||
company = soup.select_one(selector)
|
||||
if company is not None:
|
||||
text = _clean_text(company.get_text(" ", strip=True))
|
||||
if text is not None:
|
||||
return text
|
||||
|
||||
if details_offer_list is not None:
|
||||
company = details_offer_list.select_one("li:first-of-type")
|
||||
if company is not None:
|
||||
return _clean_text(company.get_text(" ", strip=True))
|
||||
text = _clean_text(company.get_text(" ", strip=True))
|
||||
if text is not None:
|
||||
return text
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@ -135,8 +135,12 @@ def fetch_apec(
|
||||
listing_errors.append(ListingError(url=result.url, stage="fetch_html", message=str(exc)))
|
||||
continue
|
||||
|
||||
snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html"
|
||||
snapshot_path.write_text(html, encoding="utf-8")
|
||||
try:
|
||||
snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html"
|
||||
snapshot_path.write_text(html, encoding="utf-8")
|
||||
except Exception as exc: # pragma: no cover - defensive boundary
|
||||
listing_errors.append(ListingError(url=result.url, stage="snapshot_write", message=str(exc)))
|
||||
continue
|
||||
|
||||
try:
|
||||
listing = normalize_apec_listing(
|
||||
|
||||
@ -45,7 +45,6 @@ class _FakeDetailPage:
|
||||
self.result_pages = result_pages
|
||||
self.rendered_html = rendered_html
|
||||
self.shell_html = "<html>shell</html>"
|
||||
self.waited_selectors: list[str] = []
|
||||
self.waited_functions: list[tuple[str, int | None]] = []
|
||||
self.goto_urls: list[str] = []
|
||||
self.current_query = ""
|
||||
@ -76,17 +75,6 @@ class _FakeDetailPage:
|
||||
return None
|
||||
|
||||
def wait_for_selector(self, selector: str, timeout: int | None = None) -> None:
|
||||
self.waited_selectors.append(selector)
|
||||
|
||||
if selector in {
|
||||
".card-ents .ents-name",
|
||||
".card-offer .ref-offre",
|
||||
".details-offer-list",
|
||||
".details-post",
|
||||
".details-post:has-text('Descriptif du poste')",
|
||||
}:
|
||||
return None
|
||||
|
||||
if selector == _RESULT_LINK_SELECTOR and self.current_results():
|
||||
return None
|
||||
|
||||
@ -139,11 +127,9 @@ def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> Non
|
||||
|
||||
assert html == "<html>rendered offer</html>"
|
||||
assert len(page.waited_functions) == 1
|
||||
assert ".card-ents .ents-name" in page.waited_functions[0][0]
|
||||
assert ".details-offer-list" in page.waited_functions[0][0]
|
||||
assert "Descriptif du poste" in page.waited_functions[0][0]
|
||||
assert page.waited_functions[0][1] == 1000
|
||||
assert page.waited_selectors == [
|
||||
".details-post:has-text('Descriptif du poste')",
|
||||
]
|
||||
|
||||
|
||||
def test_fetch_listing_html_rejects_non_apec_hosts() -> None:
|
||||
|
||||
@ -68,3 +68,59 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None:
|
||||
assert listing.description_text == "Build pipelines"
|
||||
assert listing.published_at == "2026-04-20"
|
||||
assert listing.fetched_at == "2026-06-01T10:00:00Z"
|
||||
|
||||
|
||||
def test_normalize_apec_listing_uses_details_offer_list_company_fallback() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<body>
|
||||
<main class="container-details-offer">
|
||||
<h1>Data Engineer F/H</h1>
|
||||
<div class="card-offer">
|
||||
<div class="ref-offre">Ref. Apec : 178554452W</div>
|
||||
<ul class="details-offer-list mb-20">
|
||||
<li>Fallback Company</li>
|
||||
<li>1 <span> CDI </span></li>
|
||||
<li>Puteaux - 92</li>
|
||||
</ul>
|
||||
<p>Publiée le 20/04/2026 Actualisée le 02/06/2026</p>
|
||||
</div>
|
||||
<div class="details-post">
|
||||
<h4>Salaire</h4>
|
||||
<span>A partir de 70 k€ brut annuel</span>
|
||||
</div>
|
||||
<div class="details-post">
|
||||
<h4>Prise de poste</h4>
|
||||
<span>Dès que possible</span>
|
||||
</div>
|
||||
<div class="details-post">
|
||||
<h4>Expérience</h4>
|
||||
<span>Minimum 7 ans</span>
|
||||
</div>
|
||||
<div class="details-post">
|
||||
<h4>Descriptif du poste</h4>
|
||||
<p>Build pipelines</p>
|
||||
<div class="nested-late-sections">
|
||||
<h4>Profil recherché</h4>
|
||||
<p>Python / SQL</p>
|
||||
<h4>Compétences attendues</h4>
|
||||
<p>Ignored</p>
|
||||
<h4>Entreprise</h4>
|
||||
<p>Ignored</p>
|
||||
<div class="recruiter">Ignored recruiter info</div>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
listing = normalize_apec_listing(
|
||||
url="https://example.test/job/123",
|
||||
html=html,
|
||||
fetched_at="2026-06-01T10:00:00Z",
|
||||
source_job_id=None,
|
||||
)
|
||||
|
||||
assert listing.company == "Fallback Company"
|
||||
assert listing.description_text == "Build pipelines"
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from textwrap import dedent
|
||||
|
||||
import yaml
|
||||
@ -262,6 +263,101 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings(
|
||||
}
|
||||
|
||||
|
||||
def test_fetch_apec_records_snapshot_write_failures_without_losing_successful_listings(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
data_root = tmp_path / "data"
|
||||
data_root.mkdir()
|
||||
(data_root / "candidate-profile.yaml").write_text(
|
||||
dedent(
|
||||
"""
|
||||
target_roles:
|
||||
- Role From YAML
|
||||
"""
|
||||
).strip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
html = _apec_detail_html()
|
||||
first_result = ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123")
|
||||
second_result = ApecSearchResult(url="https://example.test/job/456", source_job_id="job-456")
|
||||
|
||||
class SnapshotWriteFailingAdapter:
|
||||
instances: list["SnapshotWriteFailingAdapter"] = []
|
||||
|
||||
def __init__(self, max_listings: int = 50) -> None:
|
||||
self.max_listings = max_listings
|
||||
self.search_calls: list[list[str]] = []
|
||||
self.fetch_calls: list[str] = []
|
||||
SnapshotWriteFailingAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
return [first_result, second_result]
|
||||
|
||||
def fetch_listing_html(self, url: str) -> str:
|
||||
self.fetch_calls.append(url)
|
||||
return html
|
||||
|
||||
original_write_text = Path.write_text
|
||||
|
||||
def write_text_with_snapshot_failure(
|
||||
self: Path,
|
||||
data: str,
|
||||
encoding: str | None = None,
|
||||
errors: str | None = None,
|
||||
newline: str | None = None,
|
||||
) -> int:
|
||||
if self.parent.name == "snapshots" and self.name == "job-123.html":
|
||||
raise OSError("disk full")
|
||||
|
||||
return original_write_text(self, data, encoding=encoding, errors=errors, newline=newline)
|
||||
|
||||
monkeypatch.setattr("job_research.cli.ApecAdapter", SnapshotWriteFailingAdapter)
|
||||
monkeypatch.setattr("job_research.cli._utc_now", _fixed_now)
|
||||
monkeypatch.setattr(Path, "write_text", write_text_with_snapshot_failure)
|
||||
|
||||
result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "fetched=2" in result.stdout
|
||||
assert "normalized=1" in result.stdout
|
||||
assert "deduplicated=1" in result.stdout
|
||||
assert "failed=1" in result.stdout
|
||||
|
||||
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
||||
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
|
||||
assert [snapshot.name for snapshot in snapshot_files] == ["job-456.html"]
|
||||
|
||||
listings_payload = yaml.safe_load((run_dir / "listings.yaml").read_text(encoding="utf-8"))
|
||||
assert listings_payload == [
|
||||
{
|
||||
"source": "apec",
|
||||
"source_job_id": "job-456",
|
||||
"url": "https://example.test/job/456",
|
||||
"title": "Role From YAML",
|
||||
"company": "Example Corp",
|
||||
"location": "Paris - 75",
|
||||
"contract_type": "CDI",
|
||||
"description_text": "Build pipelines",
|
||||
"published_at": "2026-04-20",
|
||||
"fetched_at": "2026-06-01T10:00:00Z",
|
||||
"warnings": [],
|
||||
}
|
||||
]
|
||||
|
||||
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
||||
assert run_meta_payload["failed_count"] == 1
|
||||
assert run_meta_payload["listing_errors"] == [
|
||||
{
|
||||
"url": first_result.url,
|
||||
"stage": "snapshot_write",
|
||||
"message": "disk full",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_fetch_apec_fails_when_no_queries_are_derived(tmp_path, monkeypatch) -> None:
|
||||
data_root = tmp_path / "data"
|
||||
data_root.mkdir()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user