From 450134c8086b3af320c2082ec2b2594462d329d9 Mon Sep 17 00:00:00 2001 From: Antoine Date: Wed, 3 Jun 2026 19:23:35 +0200 Subject: [PATCH] fix: preserve Apec artifacts on normalization failure --- src/job_research/cli.py | 21 ++++-- tests/test_apec_cli.py | 162 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 7 deletions(-) diff --git a/src/job_research/cli.py b/src/job_research/cli.py index 5c59bd6..4c4ad02 100644 --- a/src/job_research/cli.py +++ b/src/job_research/cli.py @@ -126,6 +126,7 @@ def fetch_apec( normalized_listings = [] listing_errors: list[ListingError] = [] fetched_count = 0 + successful_fetch_count = 0 for result in search_results: fetched_count += 1 @@ -136,6 +137,14 @@ def fetch_apec( listing_errors.append(ListingError(url=result.url, stage="fetch_html", message=str(exc))) continue + successful_fetch_count += 1 + + try: + snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html" + snapshot_path.write_text(html, encoding="utf-8") + except Exception as exc: # pragma: no cover - defensive boundary + listing_errors.append(ListingError(url=result.url, stage="snapshot_write", message=str(exc))) + try: listing = normalize_apec_listing( url=result.url, @@ -149,17 +158,12 @@ def fetch_apec( normalized_listings.append(listing) - try: - snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html" - snapshot_path.write_text(html, encoding="utf-8") - except Exception as exc: # pragma: no cover - defensive boundary - listing_errors.append(ListingError(url=result.url, stage="snapshot_write", message=str(exc))) - - if fetched_count > 0 and not normalized_listings: + if successful_fetch_count == 0: typer.echo("No listings could be fetched or normalized from Apec", err=True) raise typer.Exit(code=1) deduplicated_listings = dedupe_apec_listings(normalized_listings) + should_fail_after_write = successful_fetch_count > 0 and not normalized_listings run_meta = ApecRunMeta( run_id=run_id, run_started_at=run_started_at, @@ -179,6 +183,9 @@ def fetch_apec( f"deduplicated={len(deduplicated_listings)} failed={len(listing_errors)}" ) + if should_fail_after_write: + raise typer.Exit(code=1) + def main() -> None: app() diff --git a/tests/test_apec_cli.py b/tests/test_apec_cli.py index ddb4f98..40455ad 100644 --- a/tests/test_apec_cli.py +++ b/tests/test_apec_cli.py @@ -504,6 +504,168 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails( } +def test_fetch_apec_preserves_snapshot_and_run_meta_when_normalization_fails( + tmp_path, + monkeypatch, +) -> None: + data_root = tmp_path / "data" + data_root.mkdir() + (data_root / "candidate-profile.yaml").write_text( + dedent( + """ + target_roles: + - Role From YAML + """ + ).strip(), + encoding="utf-8", + ) + + html = _apec_detail_html() + result_item = ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123") + + class NormalizeFailingAdapter: + instances: list["NormalizeFailingAdapter"] = [] + + def __init__(self, max_listings: int = 50) -> None: + self.max_listings = max_listings + self.search_calls: list[list[str]] = [] + self.fetch_calls: list[str] = [] + NormalizeFailingAdapter.instances.append(self) + + def search(self, queries: list[str]) -> list[ApecSearchResult]: + self.search_calls.append(list(queries)) + return [result_item] + + def fetch_listing_html(self, url: str) -> str: + self.fetch_calls.append(url) + return html + + def fail_normalization(*args, **kwargs) -> None: + raise RuntimeError("normalize boom") + + monkeypatch.setattr("job_research.cli.ApecAdapter", NormalizeFailingAdapter) + monkeypatch.setattr("job_research.cli.normalize_apec_listing", fail_normalization) + monkeypatch.setattr("job_research.cli._utc_now", _fixed_now) + + result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)]) + + assert result.exit_code == 1 + assert "fetched=1" in result.stdout + assert "normalized=0" in result.stdout + assert "deduplicated=0" in result.stdout + assert "failed=1" in result.stdout + + run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + snapshot_file = run_dir / "snapshots" / "job-123.html" + assert snapshot_file.exists() + assert snapshot_file.read_text(encoding="utf-8") == html + + listings_payload = yaml.safe_load((run_dir / "listings.yaml").read_text(encoding="utf-8")) + assert listings_payload == [] + + run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) + assert run_meta_payload == { + "run_id": "2026-06-01T10-00-00Z", + "run_started_at": "2026-06-01T10:00:00Z", + "derived_queries": ["Role From YAML"], + "fetched_count": 1, + "normalized_count": 0, + "deduplicated_count": 0, + "failed_count": 1, + "listing_errors": [ + { + "url": result_item.url, + "stage": "normalize", + "message": "normalize boom", + } + ], + } + + +def test_fetch_apec_writes_artifacts_when_every_normalization_attempt_fails( + tmp_path, + monkeypatch, +) -> None: + data_root = tmp_path / "data" + data_root.mkdir() + (data_root / "candidate-profile.yaml").write_text( + dedent( + """ + target_roles: + - Role From YAML + """ + ).strip(), + encoding="utf-8", + ) + + html = _apec_detail_html() + first_result = ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123") + second_result = ApecSearchResult(url="https://example.test/job/456", source_job_id="job-456") + + class NormalizeFailingAdapter: + instances: list["NormalizeFailingAdapter"] = [] + + def __init__(self, max_listings: int = 50) -> None: + self.max_listings = max_listings + self.search_calls: list[list[str]] = [] + self.fetch_calls: list[str] = [] + NormalizeFailingAdapter.instances.append(self) + + def search(self, queries: list[str]) -> list[ApecSearchResult]: + self.search_calls.append(list(queries)) + return [first_result, second_result] + + def fetch_listing_html(self, url: str) -> str: + self.fetch_calls.append(url) + return html + + def fail_normalization(*args, **kwargs) -> None: + raise RuntimeError("normalize boom") + + monkeypatch.setattr("job_research.cli.ApecAdapter", NormalizeFailingAdapter) + monkeypatch.setattr("job_research.cli.normalize_apec_listing", fail_normalization) + monkeypatch.setattr("job_research.cli._utc_now", _fixed_now) + + result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)]) + + assert result.exit_code == 1 + assert "fetched=2" in result.stdout + assert "normalized=0" in result.stdout + assert "deduplicated=0" in result.stdout + assert "failed=2" in result.stdout + + run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + snapshot_files = sorted((run_dir / "snapshots").glob("*.html")) + assert [snapshot.name for snapshot in snapshot_files] == ["job-123.html", "job-456.html"] + assert [snapshot.read_text(encoding="utf-8") for snapshot in snapshot_files] == [html, html] + + listings_payload = yaml.safe_load((run_dir / "listings.yaml").read_text(encoding="utf-8")) + assert listings_payload == [] + + run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) + assert run_meta_payload == { + "run_id": "2026-06-01T10-00-00Z", + "run_started_at": "2026-06-01T10:00:00Z", + "derived_queries": ["Role From YAML"], + "fetched_count": 2, + "normalized_count": 0, + "deduplicated_count": 0, + "failed_count": 2, + "listing_errors": [ + { + "url": first_result.url, + "stage": "normalize", + "message": "normalize boom", + }, + { + "url": second_result.url, + "stage": "normalize", + "message": "normalize boom", + }, + ], + } + + def test_fetch_apec_fails_when_no_queries_are_derived(tmp_path, monkeypatch) -> None: data_root = tmp_path / "data" data_root.mkdir()