fix: harden Apec ingestion boundaries
This commit is contained in:
parent
c218a9040e
commit
fa2000abf0
@ -300,4 +300,7 @@ class ApecAdapter:
|
|||||||
polling=1000,
|
polling=1000,
|
||||||
timeout=15_000,
|
timeout=15_000,
|
||||||
)
|
)
|
||||||
|
final_url = page.url
|
||||||
|
if not _is_public_apec_detail_url(final_url):
|
||||||
|
raise ValueError(f"ApecAdapter landed on an unexpected URL after redirects: {final_url}")
|
||||||
return page.content()
|
return page.content()
|
||||||
|
|||||||
@ -235,6 +235,9 @@ def normalize_apec_listing(
|
|||||||
description_text = _detail_block_text(soup, "Descriptif du poste")
|
description_text = _detail_block_text(soup, "Descriptif du poste")
|
||||||
if description_text is None:
|
if description_text is None:
|
||||||
warnings.append(_warning("description_text", "Description missing from Apec listing"))
|
warnings.append(_warning("description_text", "Description missing from Apec listing"))
|
||||||
|
elif not _has_useful_text(description_text):
|
||||||
|
warnings.append(_warning("description_text", "Description is empty or placeholder text"))
|
||||||
|
description_text = None
|
||||||
|
|
||||||
if source_job_id is not None:
|
if source_job_id is not None:
|
||||||
normalized_source_job_id = source_job_id
|
normalized_source_job_id = source_job_id
|
||||||
@ -283,7 +286,7 @@ def normalize_apec_listing(
|
|||||||
company=company_text,
|
company=company_text,
|
||||||
location=location if _has_useful_text(location) else None,
|
location=location if _has_useful_text(location) else None,
|
||||||
contract_type=contract_type if _has_useful_text(contract_type) else None,
|
contract_type=contract_type if _has_useful_text(contract_type) else None,
|
||||||
description_text=description_text or None,
|
description_text=description_text,
|
||||||
published_at=published_at_value,
|
published_at=published_at_value,
|
||||||
refreshed_at=refreshed_at_value,
|
refreshed_at=refreshed_at_value,
|
||||||
fetched_at=fetched_at,
|
fetched_at=fetched_at,
|
||||||
|
|||||||
@ -115,7 +115,11 @@ def build_profile(
|
|||||||
cv_signals = extract_cv_signals(cv_text)
|
cv_signals = extract_cv_signals(cv_text)
|
||||||
candidate_profile = build_candidate_profile_output(cv_signals, authored_profile)
|
candidate_profile = build_candidate_profile_output(cv_signals, authored_profile)
|
||||||
|
|
||||||
|
try:
|
||||||
save_candidate_profile_yaml(out, candidate_profile)
|
save_candidate_profile_yaml(out, candidate_profile)
|
||||||
|
except OSError as exc:
|
||||||
|
typer.echo(f"Unable to write candidate profile to {out}: {exc}", err=True)
|
||||||
|
raise typer.Exit(code=1)
|
||||||
|
|
||||||
typer.echo(f"candidate profile written to {out}")
|
typer.echo(f"candidate profile written to {out}")
|
||||||
warning_count = len(candidate_profile.warnings)
|
warning_count = len(candidate_profile.warnings)
|
||||||
|
|||||||
@ -512,6 +512,27 @@ def test_fetch_listing_html_uses_explicit_company_fallback_chain(monkeypatch) ->
|
|||||||
assert ".details-offer-list" in wait_script
|
assert ".details-offer-list" in wait_script
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_listing_html_rejects_redirected_non_apec_urls(monkeypatch) -> None:
|
||||||
|
page = _FakeDetailPage({}, rendered_html="<html>redirected</html>")
|
||||||
|
|
||||||
|
original_goto = page.goto
|
||||||
|
|
||||||
|
def redirecting_goto(url: str, wait_until: str | None = None) -> None:
|
||||||
|
original_goto(url, wait_until=wait_until)
|
||||||
|
page.url = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/recherche-avancee?error=true"
|
||||||
|
|
||||||
|
monkeypatch.setattr(page, "goto", redirecting_goto)
|
||||||
|
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="unexpected URL after redirects"):
|
||||||
|
ApecAdapter().fetch_listing_html(
|
||||||
|
"https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(page.waited_functions) == 1
|
||||||
|
assert page.goto_urls == ["https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111"]
|
||||||
|
|
||||||
|
|
||||||
def test_fetch_listing_html_rejects_non_apec_hosts() -> None:
|
def test_fetch_listing_html_rejects_non_apec_hosts() -> None:
|
||||||
adapter = ApecAdapter()
|
adapter = ApecAdapter()
|
||||||
|
|
||||||
@ -524,12 +545,14 @@ def test_fetch_listing_html_reuses_browser_context_across_calls(monkeypatch) ->
|
|||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self.goto_urls: list[str] = []
|
self.goto_urls: list[str] = []
|
||||||
self.default_timeout: int | None = None
|
self.default_timeout: int | None = None
|
||||||
|
self.url = ""
|
||||||
|
|
||||||
def set_default_timeout(self, timeout: int) -> None:
|
def set_default_timeout(self, timeout: int) -> None:
|
||||||
self.default_timeout = timeout
|
self.default_timeout = timeout
|
||||||
|
|
||||||
def goto(self, url: str, wait_until: str | None = None) -> None:
|
def goto(self, url: str, wait_until: str | None = None) -> None:
|
||||||
self.goto_urls.append(url)
|
self.goto_urls.append(url)
|
||||||
|
self.url = url
|
||||||
|
|
||||||
def wait_for_function(self, function: str, polling: int | None = None, timeout: int | None = None) -> None:
|
def wait_for_function(self, function: str, polling: int | None = None, timeout: int | None = None) -> None:
|
||||||
return None
|
return None
|
||||||
|
|||||||
@ -250,3 +250,43 @@ def test_normalize_apec_listing_records_warning_for_placeholder_company() -> Non
|
|||||||
assert listing.company is None
|
assert listing.company is None
|
||||||
assert listing.refreshed_at == "2026-06-02"
|
assert listing.refreshed_at == "2026-06-02"
|
||||||
assert [warning.field for warning in listing.warnings] == ["company"]
|
assert [warning.field for warning in listing.warnings] == ["company"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_apec_listing_records_warning_for_placeholder_description_text() -> None:
|
||||||
|
html = """
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<main class="container-details-offer">
|
||||||
|
<h1>Data Engineer F/H</h1>
|
||||||
|
<div class="card-offer">
|
||||||
|
<div class="ref-offre">Ref. Apec : 178554452W</div>
|
||||||
|
<ul class="details-offer-list mb-20">
|
||||||
|
<li>CLOUD TEMPLE</li>
|
||||||
|
<li>1 <span> CDI </span></li>
|
||||||
|
<li>Puteaux - 92</li>
|
||||||
|
</ul>
|
||||||
|
<p>Publiée le 20/04/2026 Actualisée le 02/06/2026</p>
|
||||||
|
</div>
|
||||||
|
<article class="card card-ents mb-20">
|
||||||
|
<div class="list-hzt mb-20">
|
||||||
|
<span class="ents-name">CLOUD TEMPLE</span>
|
||||||
|
</div>
|
||||||
|
</article>
|
||||||
|
<div class="details-post">
|
||||||
|
<h4>Descriptif du poste</h4>
|
||||||
|
<p>N/A</p>
|
||||||
|
</div>
|
||||||
|
</main>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
listing = normalize_apec_listing(
|
||||||
|
url="https://example.test/job/123",
|
||||||
|
html=html,
|
||||||
|
fetched_at="2026-06-01T10:00:00Z",
|
||||||
|
source_job_id="job-123",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert listing.description_text is None
|
||||||
|
assert [warning.field for warning in listing.warnings] == ["description_text"]
|
||||||
|
|||||||
@ -1,5 +1,6 @@
|
|||||||
from subprocess import run
|
from subprocess import run
|
||||||
from textwrap import dedent
|
from textwrap import dedent
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from typer.testing import CliRunner
|
from typer.testing import CliRunner
|
||||||
|
|
||||||
@ -150,6 +151,78 @@ def test_build_profile_reports_when_no_warnings_are_included(tmp_path) -> None:
|
|||||||
assert "No warnings included." in result.stdout
|
assert "No warnings included." in result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_profile_reports_output_write_failures_cleanly(tmp_path, monkeypatch) -> None:
|
||||||
|
cv = tmp_path / "cv.txt"
|
||||||
|
cv.write_text(
|
||||||
|
dedent(
|
||||||
|
"""
|
||||||
|
Tonio Example
|
||||||
|
Location: France
|
||||||
|
Languages: French, English
|
||||||
|
Skills: Python, SQL
|
||||||
|
Data Engineer at Acme
|
||||||
|
Education: Master of Science at Example University
|
||||||
|
"""
|
||||||
|
).strip(),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
profile = tmp_path / "profile.md"
|
||||||
|
profile.write_text(
|
||||||
|
dedent(
|
||||||
|
"""
|
||||||
|
# Candidate Profile
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
Junior data engineer focused on Python and GCP.
|
||||||
|
|
||||||
|
## Target Roles
|
||||||
|
- Data Engineer
|
||||||
|
|
||||||
|
## Strengths
|
||||||
|
- Python
|
||||||
|
- SQL
|
||||||
|
|
||||||
|
## Skills To Emphasize
|
||||||
|
- GCP
|
||||||
|
- BigQuery
|
||||||
|
|
||||||
|
## Constraints
|
||||||
|
- CDI only
|
||||||
|
- France only
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
- Slight preference for French listings.
|
||||||
|
"""
|
||||||
|
).strip(),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
out = tmp_path / "candidate-profile.yaml"
|
||||||
|
|
||||||
|
original_write_text = Path.write_text
|
||||||
|
|
||||||
|
def flaky_write_text(
|
||||||
|
self: Path,
|
||||||
|
data: str,
|
||||||
|
encoding: str | None = None,
|
||||||
|
errors: str | None = None,
|
||||||
|
newline: str | None = None,
|
||||||
|
) -> int:
|
||||||
|
if self == out:
|
||||||
|
raise OSError("disk full")
|
||||||
|
|
||||||
|
return original_write_text(self, data, encoding=encoding, errors=errors, newline=newline)
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "write_text", flaky_write_text)
|
||||||
|
|
||||||
|
result = CliRunner().invoke(app, ["build-profile", "--cv", str(cv), "--profile", str(profile), "--out", str(out)])
|
||||||
|
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert not out.exists()
|
||||||
|
assert "Unable to write candidate profile to" in result.stderr
|
||||||
|
assert "disk full" in result.stderr
|
||||||
|
assert "Traceback" not in result.stderr
|
||||||
|
|
||||||
|
|
||||||
def test_build_profile_rejects_empty_cv_text_before_writing(tmp_path) -> None:
|
def test_build_profile_rejects_empty_cv_text_before_writing(tmp_path) -> None:
|
||||||
cv = tmp_path / "cv.txt"
|
cv = tmp_path / "cv.txt"
|
||||||
cv.write_text(" \n", encoding="utf-8")
|
cv.write_text(" \n", encoding="utf-8")
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user