fix: derive Apec filters from profile constraints
This commit is contained in:
Antoine
parent
207d5c51c3
commit
3768bf9b3c
@ -23,6 +23,12 @@ class ApecSearchResult:
|
||||
source_job_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ApecSearchFilters:
|
||||
location: str | None = None
|
||||
contract_type: str | None = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _open_public_page():
|
||||
with sync_playwright() as playwright:
|
||||
@ -51,15 +57,18 @@ def _search_results_url(base_url: str, page_number: int) -> str:
|
||||
return urlunsplit(parsed_url._replace(query=urlencode(filtered_params, doseq=True)))
|
||||
|
||||
|
||||
def _search_url(query: str, page_number: int = 0) -> str:
|
||||
# Apec exposes stable public filter ids for France and CDI in the search URL.
|
||||
# Using the URL directly is less brittle than driving the collapsed filter UI.
|
||||
def _search_url(query: str, search_filters: ApecSearchFilters, page_number: int = 0) -> str:
|
||||
params = [
|
||||
("motsCles", query),
|
||||
("lieux", _FRANCE_LOCATION_ID),
|
||||
("typesContrat", _CDI_CONTRACT_ID),
|
||||
("page", str(page_number)),
|
||||
]
|
||||
|
||||
if search_filters.location == "France":
|
||||
params.insert(1, ("lieux", _FRANCE_LOCATION_ID))
|
||||
|
||||
if search_filters.contract_type == "CDI":
|
||||
params.insert(2 if search_filters.location == "France" else 1, ("typesContrat", _CDI_CONTRACT_ID))
|
||||
|
||||
return f"{_SEARCH_URL}?{urlencode(params)}"
|
||||
|
||||
|
||||
@ -84,7 +93,7 @@ class ApecAdapter:
|
||||
def __init__(self, max_listings: int = 50) -> None:
|
||||
self.max_listings = max_listings
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters: ApecSearchFilters) -> list[ApecSearchResult]:
|
||||
results: list[ApecSearchResult] = []
|
||||
seen_keys: set[str] = set()
|
||||
|
||||
@ -96,7 +105,7 @@ class ApecAdapter:
|
||||
if len(results) >= self.max_listings:
|
||||
break
|
||||
|
||||
page.goto(_search_url(query), wait_until="domcontentloaded")
|
||||
page.goto(_search_url(query, search_filters), wait_until="domcontentloaded")
|
||||
_accept_cookies_if_present(page)
|
||||
page.wait_for_load_state("domcontentloaded")
|
||||
|
||||
@ -107,6 +116,7 @@ class ApecAdapter:
|
||||
|
||||
result_page_url = page.url
|
||||
page_number = 0
|
||||
no_progress_pages = 0
|
||||
|
||||
while len(results) < self.max_listings:
|
||||
if page_number > 0:
|
||||
@ -123,6 +133,7 @@ class ApecAdapter:
|
||||
if not hrefs:
|
||||
break
|
||||
|
||||
added_any_result = False
|
||||
for href in hrefs:
|
||||
source_job_id = _extract_source_job_id(href)
|
||||
dedupe_key = source_job_id or href
|
||||
@ -131,10 +142,18 @@ class ApecAdapter:
|
||||
|
||||
seen_keys.add(dedupe_key)
|
||||
results.append(ApecSearchResult(url=href, source_job_id=source_job_id))
|
||||
added_any_result = True
|
||||
|
||||
if len(results) >= self.max_listings:
|
||||
break
|
||||
|
||||
if added_any_result:
|
||||
no_progress_pages = 0
|
||||
else:
|
||||
no_progress_pages += 1
|
||||
if no_progress_pages >= 2:
|
||||
break
|
||||
|
||||
page_number += 1
|
||||
|
||||
return results
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
from job_research.apec.adapter import ApecSearchFilters
|
||||
from job_research.models import CandidateProfileOutput
|
||||
|
||||
|
||||
@ -5,6 +6,19 @@ def _normalize_term(raw_term: str) -> str:
|
||||
return " ".join(raw_term.split())
|
||||
|
||||
|
||||
def _normalize_constraint(raw_term: str) -> str:
|
||||
return _normalize_term(raw_term).casefold()
|
||||
|
||||
|
||||
def derive_apec_search_filters(profile: CandidateProfileOutput) -> ApecSearchFilters:
|
||||
normalized_constraints = {_normalize_constraint(constraint) for constraint in profile.constraints}
|
||||
|
||||
return ApecSearchFilters(
|
||||
location="France" if "france only" in normalized_constraints else None,
|
||||
contract_type="CDI" if "cdi only" in normalized_constraints else None,
|
||||
)
|
||||
|
||||
|
||||
def derive_apec_queries(profile: CandidateProfileOutput) -> list[str]:
|
||||
queries: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
@ -8,10 +8,10 @@ import typer
|
||||
import yaml
|
||||
from pydantic import ValidationError
|
||||
|
||||
from job_research.apec.adapter import ApecAdapter
|
||||
from job_research.apec.adapter import ApecAdapter, ApecSearchFilters
|
||||
from job_research.apec.dedupe import dedupe_apec_listings
|
||||
from job_research.apec.normalize import normalize_apec_listing
|
||||
from job_research.apec.query_derivation import derive_apec_queries
|
||||
from job_research.apec.query_derivation import derive_apec_queries, derive_apec_search_filters
|
||||
from job_research.models import ApecRunMeta, ApecSnapshotMeta, CandidateProfileOutput, ListingError
|
||||
from job_research.profile.cv_extractor import extract_cv_signals, extract_pdf_text
|
||||
from job_research.profile.merge import build_candidate_profile_output
|
||||
@ -128,6 +128,12 @@ def fetch_apec(
|
||||
typer.echo("No usable Apec queries derived from candidate profile", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
derived_search_filters = derive_apec_search_filters(profile)
|
||||
search_filters = ApecSearchFilters(
|
||||
location=derived_search_filters.location or "France",
|
||||
contract_type=derived_search_filters.contract_type or "CDI",
|
||||
)
|
||||
|
||||
current = _utc_now().astimezone(timezone.utc).replace(microsecond=0)
|
||||
run_id = current.strftime("%Y-%m-%dT%H-%M-%SZ")
|
||||
run_started_at = current.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
@ -135,13 +141,17 @@ def fetch_apec(
|
||||
|
||||
adapter = ApecAdapter(max_listings=50)
|
||||
try:
|
||||
search_results = adapter.search(derived_queries)[:50]
|
||||
search_results = adapter.search(derived_queries, search_filters=search_filters)[:50]
|
||||
except Exception as exc: # pragma: no cover - defensive boundary
|
||||
typer.echo(f"Unable to fetch Apec search results: {exc}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
paths = apec_run_paths(data_root, run_id)
|
||||
paths["snapshots"].mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
paths["snapshots"].mkdir(parents=True, exist_ok=True)
|
||||
except OSError as exc: # pragma: no cover - defensive boundary
|
||||
typer.echo(f"Unable to create Apec snapshot directory: {exc}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
normalized_listings = []
|
||||
listing_errors: list[ListingError] = []
|
||||
|
||||
@ -5,7 +5,7 @@ import pytest
|
||||
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
|
||||
|
||||
from job_research.apec import adapter as adapter_module
|
||||
from job_research.apec.adapter import ApecAdapter
|
||||
from job_research.apec.adapter import ApecAdapter, ApecSearchFilters
|
||||
|
||||
|
||||
_RESULT_LINK_SELECTOR = "a[href*='/candidat/recherche-emploi.html/emploi/detail-offre/']"
|
||||
@ -110,7 +110,10 @@ def test_search_continues_past_duplicate_only_pages(monkeypatch) -> None:
|
||||
|
||||
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
||||
|
||||
results = ApecAdapter(max_listings=10).search(["alpha", "beta"])
|
||||
results = ApecAdapter(max_listings=10).search(
|
||||
["alpha", "beta"],
|
||||
search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
|
||||
)
|
||||
|
||||
assert [result.url for result in results] == [first_result, second_result]
|
||||
assert [result.source_job_id for result in results] == ["111", "222"]
|
||||
@ -124,6 +127,28 @@ def test_search_continues_past_duplicate_only_pages(monkeypatch) -> None:
|
||||
assert any("page=1" in url for url in page.goto_urls)
|
||||
|
||||
|
||||
def test_search_stops_after_repeated_duplicate_only_pages(monkeypatch) -> None:
|
||||
first_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/111?motsCles=alpha&page=0&selectedIndex=0"
|
||||
second_result = "https://www.apec.fr/candidat/recherche-emploi.html/emploi/detail-offre/222?motsCles=beta&page=2&selectedIndex=0"
|
||||
page = _FakeDetailPage(
|
||||
{
|
||||
"alpha": {0: [first_result], 1: []},
|
||||
"beta": {0: [first_result], 1: [first_result], 2: [second_result], 3: []},
|
||||
}
|
||||
)
|
||||
|
||||
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
||||
|
||||
results = ApecAdapter(max_listings=10).search(
|
||||
["alpha", "beta"],
|
||||
search_filters=ApecSearchFilters(location="France", contract_type="CDI"),
|
||||
)
|
||||
|
||||
assert [result.url for result in results] == [first_result]
|
||||
assert any("page=1" in url for url in page.goto_urls)
|
||||
assert not any("page=2" in url for url in page.goto_urls)
|
||||
|
||||
|
||||
def test_fetch_listing_html_waits_for_rendered_offer_content(monkeypatch) -> None:
|
||||
page = _FakeDetailPage({}, rendered_html="<html>rendered offer</html>")
|
||||
monkeypatch.setattr(adapter_module, "_open_public_page", lambda: _fake_open_public_page(page))
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
from job_research.apec.query_derivation import derive_apec_queries
|
||||
from job_research.apec.adapter import ApecSearchFilters
|
||||
from job_research.apec.query_derivation import derive_apec_queries, derive_apec_search_filters
|
||||
from job_research.models import CandidateProfileOutput
|
||||
|
||||
|
||||
@ -66,3 +67,9 @@ def test_derive_apec_queries_uses_support_terms_without_constraints() -> None:
|
||||
"Data Engineer Python",
|
||||
"Data Engineer BigQuery",
|
||||
]
|
||||
|
||||
|
||||
def test_derive_apec_search_filters_from_constraints() -> None:
|
||||
profile = CandidateProfileOutput(constraints=["CDI only", "France only"])
|
||||
|
||||
assert derive_apec_search_filters(profile) == ApecSearchFilters(location="France", contract_type="CDI")
|
||||
|
||||
@ -6,7 +6,7 @@ import pytest
|
||||
import yaml
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from job_research.apec.adapter import ApecSearchResult
|
||||
from job_research.apec.adapter import ApecSearchFilters, ApecSearchResult
|
||||
from job_research.apec.normalize import normalize_apec_listing as real_normalize_apec_listing
|
||||
from job_research.cli import app
|
||||
|
||||
@ -143,6 +143,9 @@ def test_fetch_apec_fails_cleanly_when_candidate_profile_file_is_unreadable(
|
||||
"""
|
||||
target_roles:
|
||||
- Role From YAML
|
||||
constraints:
|
||||
- CDI only
|
||||
- France only
|
||||
"""
|
||||
).strip(),
|
||||
encoding="utf-8",
|
||||
@ -190,7 +193,7 @@ def test_fetch_apec_fails_cleanly_when_adapter_search_raises(
|
||||
self.search_calls: list[list[str]] = []
|
||||
FailingSearchAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
raise RuntimeError("search boom")
|
||||
|
||||
@ -208,6 +211,70 @@ def test_fetch_apec_fails_cleanly_when_adapter_search_raises(
|
||||
assert FailingSearchAdapter.instances[0].search_calls == [["Role From YAML"]]
|
||||
|
||||
|
||||
def test_fetch_apec_reports_snapshot_directory_creation_failures_cleanly(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
data_root = tmp_path / "data"
|
||||
data_root.mkdir()
|
||||
(data_root / "candidate-profile.yaml").write_text(
|
||||
dedent(
|
||||
"""
|
||||
target_roles:
|
||||
- Role From YAML
|
||||
"""
|
||||
).strip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
class SnapshotDirFailingAdapter:
|
||||
instances: list["SnapshotDirFailingAdapter"] = []
|
||||
|
||||
def __init__(self, max_listings: int = 50) -> None:
|
||||
self.max_listings = max_listings
|
||||
self.search_calls: list[list[str]] = []
|
||||
self.fetch_calls: list[str] = []
|
||||
SnapshotDirFailingAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
return [ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123")]
|
||||
|
||||
def fetch_listing_html(self, url: str) -> str:
|
||||
self.fetch_calls.append(url)
|
||||
raise AssertionError("fetch should not be called when snapshot directory creation fails")
|
||||
|
||||
original_mkdir = Path.mkdir
|
||||
|
||||
def mkdir_with_failure(
|
||||
self: Path,
|
||||
mode: int = 0o777,
|
||||
parents: bool = False,
|
||||
exist_ok: bool = False,
|
||||
) -> None:
|
||||
if self.name == "snapshots" and parents:
|
||||
raise OSError("disk full")
|
||||
|
||||
return original_mkdir(self, mode=mode, parents=parents, exist_ok=exist_ok)
|
||||
|
||||
monkeypatch.setattr("job_research.cli.ApecAdapter", SnapshotDirFailingAdapter)
|
||||
monkeypatch.setattr("job_research.cli._utc_now", _fixed_now)
|
||||
monkeypatch.setattr(Path, "mkdir", mkdir_with_failure)
|
||||
|
||||
result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)])
|
||||
|
||||
assert result.exit_code == 1
|
||||
assert "Unable to create Apec snapshot directory" in result.stderr
|
||||
assert "Traceback" not in result.stderr
|
||||
|
||||
adapter = SnapshotDirFailingAdapter.instances[0]
|
||||
assert adapter.search_calls == [["Role From YAML"]]
|
||||
assert adapter.fetch_calls == []
|
||||
|
||||
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
||||
assert not (run_dir / "snapshots").exists()
|
||||
|
||||
|
||||
def test_fetch_apec_succeeds_with_empty_search_results(tmp_path, monkeypatch) -> None:
|
||||
data_root = tmp_path / "data"
|
||||
data_root.mkdir()
|
||||
@ -230,7 +297,7 @@ def test_fetch_apec_succeeds_with_empty_search_results(tmp_path, monkeypatch) ->
|
||||
self.fetch_calls: list[str] = []
|
||||
EmptyResultAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
return []
|
||||
|
||||
@ -303,7 +370,7 @@ def test_fetch_apec_reports_final_artifact_write_failures_cleanly(
|
||||
self.fetch_calls: list[str] = []
|
||||
FinalWriteFailingAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
return [ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123")]
|
||||
|
||||
@ -369,11 +436,13 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch
|
||||
def __init__(self, max_listings: int = 50) -> None:
|
||||
self.max_listings = max_listings
|
||||
self.search_calls: list[list[str]] = []
|
||||
self.search_filters: list[ApecSearchFilters | None] = []
|
||||
self.fetch_calls: list[str] = []
|
||||
FakeApecAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
self.search_filters.append(search_filters)
|
||||
return [ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123")]
|
||||
|
||||
def fetch_listing_html(self, url: str) -> str:
|
||||
@ -398,6 +467,7 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch
|
||||
adapter = FakeApecAdapter.instances[0]
|
||||
assert adapter.max_listings == 50
|
||||
assert adapter.search_calls == [["Role From YAML"]]
|
||||
assert adapter.search_filters == [ApecSearchFilters(location="France", contract_type="CDI")]
|
||||
assert adapter.fetch_calls == ["https://example.test/job/123"]
|
||||
|
||||
runs_root = data_root / "apec" / "runs"
|
||||
@ -476,7 +546,7 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings(
|
||||
self.fetch_calls: list[str] = []
|
||||
MixedApecAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
return [good_result, bad_result]
|
||||
|
||||
@ -575,7 +645,7 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_li
|
||||
self.fetch_calls: list[str] = []
|
||||
SnapshotWriteFailingAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
return [first_result, second_result]
|
||||
|
||||
@ -695,7 +765,7 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails(
|
||||
self.fetch_calls: list[str] = []
|
||||
SnapshotWriteFailingAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
return [first_result, second_result]
|
||||
|
||||
@ -820,7 +890,7 @@ def test_fetch_apec_counts_each_failed_url_once_even_when_multiple_errors_are_re
|
||||
self.fetch_calls: list[str] = []
|
||||
MixedErrorAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
return [first_result, second_result]
|
||||
|
||||
@ -922,7 +992,7 @@ def test_fetch_apec_preserves_snapshot_and_run_meta_when_normalization_fails(
|
||||
self.fetch_calls: list[str] = []
|
||||
NormalizeFailingAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
return [result_item]
|
||||
|
||||
@ -1004,7 +1074,7 @@ def test_fetch_apec_writes_artifacts_when_every_normalization_attempt_fails(
|
||||
self.fetch_calls: list[str] = []
|
||||
NormalizeFailingAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
return [first_result, second_result]
|
||||
|
||||
@ -1075,7 +1145,7 @@ def test_fetch_apec_fails_when_no_queries_are_derived(tmp_path, monkeypatch) ->
|
||||
self.max_listings = max_listings
|
||||
SpyApecAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
raise AssertionError("search should not be called when there are no queries")
|
||||
|
||||
def fetch_listing_html(self, url: str) -> str:
|
||||
@ -1120,7 +1190,7 @@ def test_fetch_apec_processes_only_the_first_fifty_search_results(tmp_path, monk
|
||||
self.fetch_calls: list[str] = []
|
||||
FakeApecAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
return list(results)
|
||||
|
||||
@ -1181,7 +1251,7 @@ def test_fetch_apec_fails_when_every_listing_attempt_fails(tmp_path, monkeypatch
|
||||
self.fetch_calls: list[str] = []
|
||||
FailingApecAdapter.instances.append(self)
|
||||
|
||||
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||
def search(self, queries: list[str], search_filters=None) -> list[ApecSearchResult]:
|
||||
self.search_calls.append(list(queries))
|
||||
return list(results)
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user