From ce73787f397df49fdcd60b60273322910c097c89 Mon Sep 17 00:00:00 2001 From: Antoine Date: Tue, 2 Jun 2026 21:14:40 +0200 Subject: [PATCH] fix: preserve Apec run metadata on snapshot failures --- src/job_research/cli.py | 18 ++-- src/job_research/models.py | 2 + tests/test_apec_cli.py | 166 ++++++++++++++++++++++++++++++++++--- tests/test_apec_storage.py | 4 + 4 files changed, 172 insertions(+), 18 deletions(-) diff --git a/src/job_research/cli.py b/src/job_research/cli.py index b2bf790..5c59bd6 100644 --- a/src/job_research/cli.py +++ b/src/job_research/cli.py @@ -115,7 +115,8 @@ def fetch_apec( current = _utc_now().astimezone(timezone.utc).replace(microsecond=0) run_id = current.strftime("%Y-%m-%dT%H-%M-%SZ") - fetched_at = current.strftime("%Y-%m-%dT%H:%M:%SZ") + run_started_at = current.strftime("%Y-%m-%dT%H:%M:%SZ") + fetched_at = run_started_at paths = apec_run_paths(data_root, run_id) paths["snapshots"].mkdir(parents=True, exist_ok=True) @@ -135,13 +136,6 @@ def fetch_apec( listing_errors.append(ListingError(url=result.url, stage="fetch_html", message=str(exc))) continue - try: - snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html" - snapshot_path.write_text(html, encoding="utf-8") - except Exception as exc: # pragma: no cover - defensive boundary - listing_errors.append(ListingError(url=result.url, stage="snapshot_write", message=str(exc))) - continue - try: listing = normalize_apec_listing( url=result.url, @@ -155,12 +149,20 @@ def fetch_apec( normalized_listings.append(listing) + try: + snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html" + snapshot_path.write_text(html, encoding="utf-8") + except Exception as exc: # pragma: no cover - defensive boundary + listing_errors.append(ListingError(url=result.url, stage="snapshot_write", message=str(exc))) + if fetched_count > 0 and not normalized_listings: typer.echo("No listings could be fetched or normalized from Apec", err=True) raise typer.Exit(code=1) deduplicated_listings = dedupe_apec_listings(normalized_listings) run_meta = ApecRunMeta( + run_id=run_id, + run_started_at=run_started_at, derived_queries=derived_queries, fetched_count=fetched_count, normalized_count=len(normalized_listings), diff --git a/src/job_research/models.py b/src/job_research/models.py index c396fd3..6646469 100644 --- a/src/job_research/models.py +++ b/src/job_research/models.py @@ -47,6 +47,8 @@ class ApecListing(BaseModel): class ApecRunMeta(BaseModel): + run_id: str + run_started_at: str derived_queries: list[str] = Field(default_factory=list) fetched_count: int = 0 normalized_count: int = 0 diff --git a/tests/test_apec_cli.py b/tests/test_apec_cli.py index 741fbcc..ddb4f98 100644 --- a/tests/test_apec_cli.py +++ b/tests/test_apec_cli.py @@ -160,6 +160,8 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) assert run_meta_payload == { + "run_id": "2026-06-01T10-00-00Z", + "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], "fetched_count": 1, "normalized_count": 1, @@ -248,6 +250,8 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings( run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) assert run_meta_payload == { + "run_id": "2026-06-01T10-00-00Z", + "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], "fetched_count": 2, "normalized_count": 1, @@ -263,7 +267,7 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings( } -def test_fetch_apec_records_snapshot_write_failures_without_losing_successful_listings( +def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_listings( tmp_path, monkeypatch, ) -> None: @@ -322,8 +326,8 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_successful_li assert result.exit_code == 0 assert "fetched=2" in result.stdout - assert "normalized=1" in result.stdout - assert "deduplicated=1" in result.stdout + assert "normalized=2" in result.stdout + assert "deduplicated=2" in result.stdout assert "failed=1" in result.stdout run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" @@ -332,6 +336,19 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_successful_li listings_payload = yaml.safe_load((run_dir / "listings.yaml").read_text(encoding="utf-8")) assert listings_payload == [ + { + "source": "apec", + "source_job_id": "job-123", + "url": "https://example.test/job/123", + "title": "Role From YAML", + "company": "Example Corp", + "location": "Paris - 75", + "contract_type": "CDI", + "description_text": "Build pipelines", + "published_at": "2026-04-20", + "fetched_at": "2026-06-01T10:00:00Z", + "warnings": [], + }, { "source": "apec", "source_job_id": "job-456", @@ -344,19 +361,148 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_successful_li "published_at": "2026-04-20", "fetched_at": "2026-06-01T10:00:00Z", "warnings": [], - } + }, ] run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) - assert run_meta_payload["failed_count"] == 1 - assert run_meta_payload["listing_errors"] == [ + assert run_meta_payload == { + "run_id": "2026-06-01T10-00-00Z", + "run_started_at": "2026-06-01T10:00:00Z", + "derived_queries": ["Role From YAML"], + "fetched_count": 2, + "normalized_count": 2, + "deduplicated_count": 2, + "failed_count": 1, + "listing_errors": [ + { + "url": first_result.url, + "stage": "snapshot_write", + "message": "disk full", + } + ], + } + + +def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails( + tmp_path, + monkeypatch, +) -> None: + data_root = tmp_path / "data" + data_root.mkdir() + (data_root / "candidate-profile.yaml").write_text( + dedent( + """ + target_roles: + - Role From YAML + """ + ).strip(), + encoding="utf-8", + ) + + html = _apec_detail_html() + first_result = ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123") + second_result = ApecSearchResult(url="https://example.test/job/456", source_job_id="job-456") + + class SnapshotWriteFailingAdapter: + instances: list["SnapshotWriteFailingAdapter"] = [] + + def __init__(self, max_listings: int = 50) -> None: + self.max_listings = max_listings + self.search_calls: list[list[str]] = [] + self.fetch_calls: list[str] = [] + SnapshotWriteFailingAdapter.instances.append(self) + + def search(self, queries: list[str]) -> list[ApecSearchResult]: + self.search_calls.append(list(queries)) + return [first_result, second_result] + + def fetch_listing_html(self, url: str) -> str: + self.fetch_calls.append(url) + return html + + original_write_text = Path.write_text + + def write_text_with_all_snapshot_failures( + self: Path, + data: str, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> int: + if self.parent.name == "snapshots": + raise OSError("disk full") + + return original_write_text(self, data, encoding=encoding, errors=errors, newline=newline) + + monkeypatch.setattr("job_research.cli.ApecAdapter", SnapshotWriteFailingAdapter) + monkeypatch.setattr("job_research.cli._utc_now", _fixed_now) + monkeypatch.setattr(Path, "write_text", write_text_with_all_snapshot_failures) + + result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)]) + + assert result.exit_code == 0 + assert "fetched=2" in result.stdout + assert "normalized=2" in result.stdout + assert "deduplicated=2" in result.stdout + assert "failed=2" in result.stdout + + run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + snapshot_files = sorted((run_dir / "snapshots").glob("*.html")) + assert snapshot_files == [] + + listings_payload = yaml.safe_load((run_dir / "listings.yaml").read_text(encoding="utf-8")) + assert listings_payload == [ { - "url": first_result.url, - "stage": "snapshot_write", - "message": "disk full", - } + "source": "apec", + "source_job_id": "job-123", + "url": "https://example.test/job/123", + "title": "Role From YAML", + "company": "Example Corp", + "location": "Paris - 75", + "contract_type": "CDI", + "description_text": "Build pipelines", + "published_at": "2026-04-20", + "fetched_at": "2026-06-01T10:00:00Z", + "warnings": [], + }, + { + "source": "apec", + "source_job_id": "job-456", + "url": "https://example.test/job/456", + "title": "Role From YAML", + "company": "Example Corp", + "location": "Paris - 75", + "contract_type": "CDI", + "description_text": "Build pipelines", + "published_at": "2026-04-20", + "fetched_at": "2026-06-01T10:00:00Z", + "warnings": [], + }, ] + run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) + assert run_meta_payload == { + "run_id": "2026-06-01T10-00-00Z", + "run_started_at": "2026-06-01T10:00:00Z", + "derived_queries": ["Role From YAML"], + "fetched_count": 2, + "normalized_count": 2, + "deduplicated_count": 2, + "failed_count": 2, + "listing_errors": [ + { + "url": first_result.url, + "stage": "snapshot_write", + "message": "disk full", + }, + { + "url": second_result.url, + "stage": "snapshot_write", + "message": "disk full", + }, + ], + } + def test_fetch_apec_fails_when_no_queries_are_derived(tmp_path, monkeypatch) -> None: data_root = tmp_path / "data" diff --git a/tests/test_apec_storage.py b/tests/test_apec_storage.py index 891818d..d71f625 100644 --- a/tests/test_apec_storage.py +++ b/tests/test_apec_storage.py @@ -24,6 +24,8 @@ def test_apec_models_serialize_expected_listing_shape() -> None: ], ) run_meta = ApecRunMeta( + run_id="2026-06-01T10-00-00Z", + run_started_at="2026-06-01T10:00:00Z", derived_queries=["Data Engineer"], fetched_count=1, normalized_count=1, @@ -34,6 +36,8 @@ def test_apec_models_serialize_expected_listing_shape() -> None: assert listing.model_dump()["source"] == "apec" assert listing.model_dump()["warnings"][0]["field"] == "location" + assert run_meta.model_dump()["run_id"] == "2026-06-01T10-00-00Z" + assert run_meta.model_dump()["run_started_at"] == "2026-06-01T10:00:00Z" assert run_meta.model_dump()["derived_queries"] == ["Data Engineer"]