tonio/job-research
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: preserve Apec artifacts on normalization failure

Browse Source
This commit is contained in:
Antoine 2026-06-03 19:23:35 +02:00
parent ce73787f39
commit 450134c808
2 changed files with 176 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/job_research/cli.py
View File

@ -126,6 +126,7 @@ def fetch_apec(
normalized_listings = []
listing_errors: list[ListingError] = []
fetched_count = 0
successful_fetch_count = 0
for result in search_results:
fetched_count += 1
@ -136,6 +137,14 @@ def fetch_apec(
listing_errors.append(ListingError(url=result.url, stage="fetch_html", message=str(exc)))
continue
successful_fetch_count += 1
try:
snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html"
snapshot_path.write_text(html, encoding="utf-8")
except Exception as exc: # pragma: no cover - defensive boundary
listing_errors.append(ListingError(url=result.url, stage="snapshot_write", message=str(exc)))
try:
listing = normalize_apec_listing(
url=result.url,
@ -149,17 +158,12 @@ def fetch_apec(
normalized_listings.append(listing)
try:
snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html"
snapshot_path.write_text(html, encoding="utf-8")
except Exception as exc: # pragma: no cover - defensive boundary
listing_errors.append(ListingError(url=result.url, stage="snapshot_write", message=str(exc)))
if fetched_count > 0 and not normalized_listings:
if successful_fetch_count == 0:
typer.echo("No listings could be fetched or normalized from Apec", err=True)
raise typer.Exit(code=1)
deduplicated_listings = dedupe_apec_listings(normalized_listings)
should_fail_after_write = successful_fetch_count > 0 and not normalized_listings
run_meta = ApecRunMeta(
run_id=run_id,
run_started_at=run_started_at,
@ -179,6 +183,9 @@ def fetch_apec(
f"deduplicated={len(deduplicated_listings)} failed={len(listing_errors)}"
)
if should_fail_after_write:
raise typer.Exit(code=1)
def main() -> None:
app()

162
tests/test_apec_cli.py
View File

@ -504,6 +504,168 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails(
}
def test_fetch_apec_preserves_snapshot_and_run_meta_when_normalization_fails(
tmp_path,
monkeypatch,
) -> None:
data_root = tmp_path / "data"
data_root.mkdir()
(data_root / "candidate-profile.yaml").write_text(
dedent(
"""
target_roles:
- Role From YAML
"""
).strip(),
encoding="utf-8",
)
html = _apec_detail_html()
result_item = ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123")
class NormalizeFailingAdapter:
instances: list["NormalizeFailingAdapter"] = []
def __init__(self, max_listings: int = 50) -> None:
self.max_listings = max_listings
self.search_calls: list[list[str]] = []
self.fetch_calls: list[str] = []
NormalizeFailingAdapter.instances.append(self)
def search(self, queries: list[str]) -> list[ApecSearchResult]:
self.search_calls.append(list(queries))
return [result_item]
def fetch_listing_html(self, url: str) -> str:
self.fetch_calls.append(url)
return html
def fail_normalization(*args, **kwargs) -> None:
raise RuntimeError("normalize boom")
monkeypatch.setattr("job_research.cli.ApecAdapter", NormalizeFailingAdapter)
monkeypatch.setattr("job_research.cli.normalize_apec_listing", fail_normalization)
monkeypatch.setattr("job_research.cli._utc_now", _fixed_now)
result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)])
assert result.exit_code == 1
assert "fetched=1" in result.stdout
assert "normalized=0" in result.stdout
assert "deduplicated=0" in result.stdout
assert "failed=1" in result.stdout
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
snapshot_file = run_dir / "snapshots" / "job-123.html"
assert snapshot_file.exists()
assert snapshot_file.read_text(encoding="utf-8") == html
listings_payload = yaml.safe_load((run_dir / "listings.yaml").read_text(encoding="utf-8"))
assert listings_payload == []
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z",
"run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"],
"fetched_count": 1,
"normalized_count": 0,
"deduplicated_count": 0,
"failed_count": 1,
"listing_errors": [
{
"url": result_item.url,
"stage": "normalize",
"message": "normalize boom",
}
],
}
def test_fetch_apec_writes_artifacts_when_every_normalization_attempt_fails(
tmp_path,
monkeypatch,
) -> None:
data_root = tmp_path / "data"
data_root.mkdir()
(data_root / "candidate-profile.yaml").write_text(
dedent(
"""
target_roles:
- Role From YAML
"""
).strip(),
encoding="utf-8",
)
html = _apec_detail_html()
first_result = ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123")
second_result = ApecSearchResult(url="https://example.test/job/456", source_job_id="job-456")
class NormalizeFailingAdapter:
instances: list["NormalizeFailingAdapter"] = []
def __init__(self, max_listings: int = 50) -> None:
self.max_listings = max_listings
self.search_calls: list[list[str]] = []
self.fetch_calls: list[str] = []
NormalizeFailingAdapter.instances.append(self)
def search(self, queries: list[str]) -> list[ApecSearchResult]:
self.search_calls.append(list(queries))
return [first_result, second_result]
def fetch_listing_html(self, url: str) -> str:
self.fetch_calls.append(url)
return html
def fail_normalization(*args, **kwargs) -> None:
raise RuntimeError("normalize boom")
monkeypatch.setattr("job_research.cli.ApecAdapter", NormalizeFailingAdapter)
monkeypatch.setattr("job_research.cli.normalize_apec_listing", fail_normalization)
monkeypatch.setattr("job_research.cli._utc_now", _fixed_now)
result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)])
assert result.exit_code == 1
assert "fetched=2" in result.stdout
assert "normalized=0" in result.stdout
assert "deduplicated=0" in result.stdout
assert "failed=2" in result.stdout
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
assert [snapshot.name for snapshot in snapshot_files] == ["job-123.html", "job-456.html"]
assert [snapshot.read_text(encoding="utf-8") for snapshot in snapshot_files] == [html, html]
listings_payload = yaml.safe_load((run_dir / "listings.yaml").read_text(encoding="utf-8"))
assert listings_payload == []
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z",
"run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"],
"fetched_count": 2,
"normalized_count": 0,
"deduplicated_count": 0,
"failed_count": 2,
"listing_errors": [
{
"url": first_result.url,
"stage": "normalize",
"message": "normalize boom",
},
{
"url": second_result.url,
"stage": "normalize",
"message": "normalize boom",
},
],
}
def test_fetch_apec_fails_when_no_queries_are_derived(tmp_path, monkeypatch) -> None:
data_root = tmp_path / "data"
data_root.mkdir()