feat: add fetch-apec command

src/job_research/cli.py

@@ -1,15 +1,32 @@
+from datetime import datetime, timezone
 from pathlib import Path
+from typing import Any
 
 import typer
+import yaml
 
+from job_research.apec.adapter import ApecAdapter
+from job_research.apec.dedupe import dedupe_apec_listings
+from job_research.apec.normalize import normalize_apec_listing
+from job_research.apec.query_derivation import derive_apec_queries
+from job_research.models import ApecRunMeta, CandidateProfileOutput, ListingError
 from job_research.profile.cv_extractor import extract_cv_signals, extract_pdf_text
 from job_research.profile.merge import build_candidate_profile_output
 from job_research.profile.profile_parser import parse_profile_markdown
-from job_research.storage import save_candidate_profile_yaml
+from job_research.storage import apec_run_paths, load_yaml, save_candidate_profile_yaml
 
 app = typer.Typer(help="Build one canonical candidate profile YAML")
 
 
+def _utc_now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _write_yaml(path: Path, payload: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8")
+
+
 @app.callback()
 def main_command() -> None:
     pass
@@ -59,6 +76,80 @@ def build_profile(
         typer.echo("No warnings included.")
 
 
+@app.command("fetch-apec")
+def fetch_apec(
+    data_root: Path = typer.Option(
+        Path("data"),
+        "--data-root",
+        exists=True,
+        file_okay=False,
+        dir_okay=True,
+        readable=True,
+        help="Directory containing candidate-profile.yaml and Apec run artifacts.",
+    ),
+) -> None:
+    """Fetch, normalize, dedupe, and persist Apec listings."""
+
+    profile_path = data_root / "candidate-profile.yaml"
+    profile = CandidateProfileOutput.model_validate(load_yaml(profile_path))
+    derived_queries = derive_apec_queries(profile)
+
+    current = _utc_now().astimezone(timezone.utc).replace(microsecond=0)
+    run_id = current.strftime("%Y-%m-%dT%H-%M-%SZ")
+    fetched_at = current.strftime("%Y-%m-%dT%H:%M:%SZ")
+    paths = apec_run_paths(data_root, run_id)
+    paths["snapshots"].mkdir(parents=True, exist_ok=True)
+
+    adapter = ApecAdapter(max_listings=50)
+    search_results = adapter.search(derived_queries)
+
+    normalized_listings = []
+    listing_errors: list[ListingError] = []
+    fetched_count = 0
+
+    for index, result in enumerate(search_results, start=1):
+        try:
+            html = adapter.fetch_listing_html(result.url)
+        except Exception as exc:  # pragma: no cover - defensive boundary
+            listing_errors.append(ListingError(url=result.url, stage="fetch_html", message=str(exc)))
+            continue
+
+        fetched_count += 1
+        snapshot_path = paths["snapshots"] / f"{index:04d}.html"
+        snapshot_path.write_text(html, encoding="utf-8")
+
+        try:
+            listing = normalize_apec_listing(
+                url=result.url,
+                html=html,
+                fetched_at=fetched_at,
+                source_job_id=result.source_job_id,
+            )
+        except Exception as exc:  # pragma: no cover - defensive boundary
+            listing_errors.append(ListingError(url=result.url, stage="normalize", message=str(exc)))
+            continue
+
+        normalized_listings.append(listing)
+
+    deduplicated_listings = dedupe_apec_listings(normalized_listings)
+    run_meta = ApecRunMeta(
+        derived_queries=derived_queries,
+        fetched_count=fetched_count,
+        normalized_count=len(normalized_listings),
+        deduplicated_count=len(deduplicated_listings),
+        failed_count=len(listing_errors),
+        listing_errors=listing_errors,
+    )
+
+    _write_yaml(paths["listings"], [listing.model_dump(mode="json") for listing in deduplicated_listings])
+    _write_yaml(paths["run_meta"], run_meta.model_dump(mode="json"))
+
+    typer.echo(
+        f"query={len(derived_queries)} fetched={fetched_count} normalized={len(normalized_listings)} "
+        f"deduplicated={len(deduplicated_listings)} failed={len(listing_errors)}"
+    )
+
+
 def main() -> None:
     app()
 

tests/test_apec_cli.py

@@ -0,0 +1,109 @@
+from datetime import datetime, timezone
+from textwrap import dedent
+
+import yaml
+from typer.testing import CliRunner
+
+from job_research.apec.adapter import ApecSearchResult
+from job_research.cli import app
+
+
+def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch) -> None:
+    data_root = tmp_path / "data"
+    data_root.mkdir()
+    (data_root / "candidate-profile.yaml").write_text(
+        dedent(
+            """
+            target_roles:
+              - Role From YAML
+            """
+        ).strip(),
+        encoding="utf-8",
+    )
+
+    html = dedent(
+        """
+        <html>
+          <body>
+            <h1>Role From YAML</h1>
+            <div class="company">Example Corp</div>
+            <div class="location">Paris</div>
+            <div class="contract">CDI</div>
+            <div class="description">Build pipelines</div>
+          </body>
+        </html>
+        """
+    ).strip()
+
+    class FakeApecAdapter:
+        instances: list["FakeApecAdapter"] = []
+
+        def __init__(self, max_listings: int = 50) -> None:
+            self.max_listings = max_listings
+            self.search_calls: list[list[str]] = []
+            self.fetch_calls: list[str] = []
+            FakeApecAdapter.instances.append(self)
+
+        def search(self, queries: list[str]) -> list[ApecSearchResult]:
+            self.search_calls.append(list(queries))
+            return [ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123")]
+
+        def fetch_listing_html(self, url: str) -> str:
+            self.fetch_calls.append(url)
+            return html
+
+    monkeypatch.setattr("job_research.cli.ApecAdapter", FakeApecAdapter)
+    monkeypatch.setattr(
+        "job_research.cli._utc_now",
+        lambda: datetime(2026, 6, 1, 10, 0, 0, tzinfo=timezone.utc),
+    )
+
+    result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)])
+
+    assert result.exit_code == 0
+    assert "query=1" in result.stdout
+    assert "fetched=1" in result.stdout
+    assert "normalized=1" in result.stdout
+    assert "deduplicated=1" in result.stdout
+    assert "failed=0" in result.stdout
+
+    adapter = FakeApecAdapter.instances[0]
+    assert adapter.max_listings == 50
+    assert adapter.search_calls == [["Role From YAML"]]
+    assert adapter.fetch_calls == ["https://example.test/job/123"]
+
+    runs_root = data_root / "apec" / "runs"
+    run_dirs = list(runs_root.iterdir())
+    assert len(run_dirs) == 1
+
+    run_dir = run_dirs[0]
+    snapshot_files = list((run_dir / "snapshots").glob("*.html"))
+    assert len(snapshot_files) == 1
+    assert snapshot_files[0].read_text(encoding="utf-8") == html
+
+    listings_payload = yaml.safe_load((run_dir / "listings.yaml").read_text(encoding="utf-8"))
+    assert listings_payload == [
+        {
+            "source": "apec",
+            "source_job_id": "job-123",
+            "url": "https://example.test/job/123",
+            "title": "Role From YAML",
+            "company": "Example Corp",
+            "location": "Paris",
+            "contract_type": "CDI",
+            "description_text": "Build pipelines",
+            "published_at": None,
+            "fetched_at": "2026-06-01T10:00:00Z",
+            "warnings": [],
+        }
+    ]
+
+    run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
+    assert run_meta_payload == {
+        "derived_queries": ["Role From YAML"],
+        "fetched_count": 1,
+        "normalized_count": 1,
+        "deduplicated_count": 1,
+        "failed_count": 0,
+        "listing_errors": [],
+    }