From 2da39613c779958d110742ed4c85ae0bdad7593c Mon Sep 17 00:00:00 2001 From: Antoine Date: Tue, 2 Jun 2026 19:23:14 +0200 Subject: [PATCH] fix: make Apec adapter real --- src/job_research/apec/adapter.py | 115 ++++++++++++++++++++++++++++++- src/job_research/cli.py | 19 ++++- tests/test_apec_cli.py | 107 ++++++++++++++++++++++++++++ 3 files changed, 237 insertions(+), 4 deletions(-) diff --git a/src/job_research/apec/adapter.py b/src/job_research/apec/adapter.py index 4f35dc0..341f0f3 100644 --- a/src/job_research/apec/adapter.py +++ b/src/job_research/apec/adapter.py @@ -1,6 +1,17 @@ from __future__ import annotations +import re +from contextlib import contextmanager from dataclasses import dataclass +from urllib.parse import parse_qsl, urlencode, urlparse, urlsplit, urlunsplit + +from playwright.sync_api import TimeoutError as PlaywrightTimeoutError +from playwright.sync_api import sync_playwright + + +_SEARCH_URL = "https://www.apec.fr/candidat/recherche-emploi.html/emploi" +_RESULT_LINK_SELECTOR = "a[href*='/candidat/recherche-emploi.html/emploi/detail-offre/']" +_DETAIL_JOB_ID_PATTERN = re.compile(r"/detail-offre/([^/?#]+)") @dataclass(slots=True) @@ -9,12 +20,112 @@ class ApecSearchResult: source_job_id: str | None = None +@contextmanager +def _open_public_page(): + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=True) + try: + page = browser.new_page() + page.set_default_timeout(15_000) + yield page + finally: + browser.close() + + +def _extract_source_job_id(url: str) -> str | None: + match = _DETAIL_JOB_ID_PATTERN.search(url) + if match is None: + return None + + return match.group(1) + + +def _search_results_url(base_url: str, page_number: int) -> str: + parsed_url = urlsplit(base_url) + params = parse_qsl(parsed_url.query, keep_blank_values=True) + filtered_params = [(key, value) for key, value in params if key != "page"] + filtered_params.append(("page", str(page_number))) + return urlunsplit(parsed_url._replace(query=urlencode(filtered_params, doseq=True))) + + +def _accept_cookies_if_present(page) -> None: + try: + page.get_by_role("button", name="Accepter tous les cookies").click(timeout=2_000) + except PlaywrightTimeoutError: + return + + class ApecAdapter: def __init__(self, max_listings: int = 50) -> None: self.max_listings = max_listings def search(self, queries: list[str]) -> list[ApecSearchResult]: - return [] + results: list[ApecSearchResult] = [] + seen_keys: set[str] = set() + + with _open_public_page() as page: + for query in queries: + if not query.strip(): + continue + + if len(results) >= self.max_listings: + break + + page.goto(_SEARCH_URL, wait_until="domcontentloaded") + _accept_cookies_if_present(page) + page.locator('input[name="keywords"]').fill(query) + page.get_by_role("button", name="Rechercher").click() + page.wait_for_load_state("domcontentloaded") + + try: + page.wait_for_selector(_RESULT_LINK_SELECTOR, timeout=5_000) + except PlaywrightTimeoutError: + continue + + result_page_url = page.url + page_number = 0 + + while len(results) < self.max_listings: + if page_number > 0: + page.goto(_search_results_url(result_page_url, page_number), wait_until="domcontentloaded") + + try: + page.wait_for_selector(_RESULT_LINK_SELECTOR, timeout=5_000) + except PlaywrightTimeoutError: + break + + hrefs = page.locator(_RESULT_LINK_SELECTOR).evaluate_all( + "nodes => nodes.map(node => node.href)" + ) + if not hrefs: + break + + new_results_on_page = 0 + for href in hrefs: + source_job_id = _extract_source_job_id(href) + dedupe_key = source_job_id or href + if dedupe_key in seen_keys: + continue + + seen_keys.add(dedupe_key) + results.append(ApecSearchResult(url=href, source_job_id=source_job_id)) + new_results_on_page += 1 + + if len(results) >= self.max_listings: + break + + if new_results_on_page == 0: + break + + page_number += 1 + + return results def fetch_listing_html(self, url: str) -> str: - return "" + parsed_url = urlparse(url) + if not parsed_url.netloc.endswith("apec.fr"): + raise ValueError("ApecAdapter only fetches public Apec URLs") + + with _open_public_page() as page: + page.goto(url, wait_until="domcontentloaded") + return page.content() diff --git a/src/job_research/cli.py b/src/job_research/cli.py index 6246946..aff39f9 100644 --- a/src/job_research/cli.py +++ b/src/job_research/cli.py @@ -1,6 +1,8 @@ from datetime import datetime, timezone +import re from pathlib import Path from typing import Any +from urllib.parse import unquote, urlparse import typer import yaml @@ -22,6 +24,19 @@ def _utc_now() -> datetime: return datetime.now(timezone.utc) +def _snapshot_stem(url: str, source_job_id: str | None) -> str: + if source_job_id: + return source_job_id + + parsed_url = urlparse(url) + fallback = parsed_url.path.rstrip("/").rsplit("/", 1)[-1] or parsed_url.netloc or "listing" + if parsed_url.query: + fallback = f"{fallback}-{parsed_url.query}" + + stem = re.sub(r"[^A-Za-z0-9]+", "-", unquote(fallback)).strip("-") + return stem or "listing" + + def _write_yaml(path: Path, payload: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8") @@ -111,7 +126,7 @@ def fetch_apec( listing_errors: list[ListingError] = [] fetched_count = 0 - for index, result in enumerate(search_results, start=1): + for result in search_results: fetched_count += 1 try: @@ -120,7 +135,7 @@ def fetch_apec( listing_errors.append(ListingError(url=result.url, stage="fetch_html", message=str(exc))) continue - snapshot_path = paths["snapshots"] / f"{index:04d}.html" + snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html" snapshot_path.write_text(html, encoding="utf-8") try: diff --git a/tests/test_apec_cli.py b/tests/test_apec_cli.py index d4b6c91..9d5a484 100644 --- a/tests/test_apec_cli.py +++ b/tests/test_apec_cli.py @@ -83,6 +83,7 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch run_dir = run_dirs[0] snapshot_files = list((run_dir / "snapshots").glob("*.html")) assert len(snapshot_files) == 1 + assert snapshot_files[0].name == "job-123.html" assert snapshot_files[0].read_text(encoding="utf-8") == html listings_payload = yaml.safe_load((run_dir / "listings.yaml").read_text(encoding="utf-8")) @@ -113,6 +114,112 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch } +def test_fetch_apec_records_partial_failures_without_losing_successful_listings( + tmp_path, + monkeypatch, +) -> None: + data_root = tmp_path / "data" + data_root.mkdir() + (data_root / "candidate-profile.yaml").write_text( + dedent( + """ + target_roles: + - Role From YAML + """ + ).strip(), + encoding="utf-8", + ) + + html = dedent( + """ + + +

Role From YAML

+
Example Corp
+
Paris
+
CDI
+
Build pipelines
+ + + """ + ).strip() + good_result = ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123") + bad_result = ApecSearchResult(url="https://example.test/job/456", source_job_id="job-456") + + class MixedApecAdapter: + instances: list["MixedApecAdapter"] = [] + + def __init__(self, max_listings: int = 50) -> None: + self.max_listings = max_listings + self.search_calls: list[list[str]] = [] + self.fetch_calls: list[str] = [] + MixedApecAdapter.instances.append(self) + + def search(self, queries: list[str]) -> list[ApecSearchResult]: + self.search_calls.append(list(queries)) + return [good_result, bad_result] + + def fetch_listing_html(self, url: str) -> str: + self.fetch_calls.append(url) + if url == good_result.url: + return html + + raise RuntimeError(f"boom: {url}") + + monkeypatch.setattr("job_research.cli.ApecAdapter", MixedApecAdapter) + monkeypatch.setattr("job_research.cli._utc_now", _fixed_now) + + result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)]) + + assert result.exit_code == 0 + assert "fetched=2" in result.stdout + assert "normalized=1" in result.stdout + assert "deduplicated=1" in result.stdout + assert "failed=1" in result.stdout + + adapter = MixedApecAdapter.instances[0] + assert adapter.search_calls == [["Role From YAML"]] + assert adapter.fetch_calls == [good_result.url, bad_result.url] + + run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + snapshot_files = list((run_dir / "snapshots").glob("*.html")) + assert [snapshot.name for snapshot in snapshot_files] == ["job-123.html"] + assert snapshot_files[0].read_text(encoding="utf-8") == html + + listings_payload = yaml.safe_load((run_dir / "listings.yaml").read_text(encoding="utf-8")) + assert listings_payload == [ + { + "source": "apec", + "source_job_id": "job-123", + "url": "https://example.test/job/123", + "title": "Role From YAML", + "company": "Example Corp", + "location": "Paris", + "contract_type": "CDI", + "description_text": "Build pipelines", + "published_at": None, + "fetched_at": "2026-06-01T10:00:00Z", + "warnings": [], + } + ] + + run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) + assert run_meta_payload == { + "derived_queries": ["Role From YAML"], + "fetched_count": 2, + "normalized_count": 1, + "deduplicated_count": 1, + "failed_count": 1, + "listing_errors": [ + { + "url": bad_result.url, + "stage": "fetch_html", + "message": f"boom: {bad_result.url}", + } + ], + } + + def test_fetch_apec_fails_when_no_queries_are_derived(tmp_path, monkeypatch) -> None: data_root = tmp_path / "data" data_root.mkdir()