tonio/job-research
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: preserve Apec run metadata on snapshot failures

Browse Source
This commit is contained in:
Antoine 2026-06-02 21:14:40 +02:00
parent c28f804e23
commit ce73787f39
4 changed files with 172 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/job_research/cli.py
View File

@ -115,7 +115,8 @@ def fetch_apec(
current = _utc_now().astimezone(timezone.utc).replace(microsecond=0)
run_id = current.strftime("%Y-%m-%dT%H-%M-%SZ")
fetched_at = current.strftime("%Y-%m-%dT%H:%M:%SZ")
run_started_at = current.strftime("%Y-%m-%dT%H:%M:%SZ")
fetched_at = run_started_at
paths = apec_run_paths(data_root, run_id)
paths["snapshots"].mkdir(parents=True, exist_ok=True)
@ -135,13 +136,6 @@ def fetch_apec(
listing_errors.append(ListingError(url=result.url, stage="fetch_html", message=str(exc)))
continue
try:
snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html"
snapshot_path.write_text(html, encoding="utf-8")
except Exception as exc: # pragma: no cover - defensive boundary
listing_errors.append(ListingError(url=result.url, stage="snapshot_write", message=str(exc)))
continue
try:
listing = normalize_apec_listing(
url=result.url,
@ -155,12 +149,20 @@ def fetch_apec(
normalized_listings.append(listing)
try:
snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html"
snapshot_path.write_text(html, encoding="utf-8")
except Exception as exc: # pragma: no cover - defensive boundary
listing_errors.append(ListingError(url=result.url, stage="snapshot_write", message=str(exc)))
if fetched_count > 0 and not normalized_listings:
typer.echo("No listings could be fetched or normalized from Apec", err=True)
raise typer.Exit(code=1)
deduplicated_listings = dedupe_apec_listings(normalized_listings)
run_meta = ApecRunMeta(
run_id=run_id,
run_started_at=run_started_at,
derived_queries=derived_queries,
fetched_count=fetched_count,
normalized_count=len(normalized_listings),

2
src/job_research/models.py
View File

@ -47,6 +47,8 @@ class ApecListing(BaseModel):
class ApecRunMeta(BaseModel):
run_id: str
run_started_at: str
derived_queries: list[str] = Field(default_factory=list)
fetched_count: int = 0
normalized_count: int = 0

166
tests/test_apec_cli.py
View File

@ -160,6 +160,8 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z",
"run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"],
"fetched_count": 1,
"normalized_count": 1,
@ -248,6 +250,8 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings(
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z",
"run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"],
"fetched_count": 2,
"normalized_count": 1,
@ -263,7 +267,7 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings(
}
def test_fetch_apec_records_snapshot_write_failures_without_losing_successful_listings(
def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_listings(
tmp_path,
monkeypatch,
) -> None:
@ -322,8 +326,8 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_successful_li
assert result.exit_code == 0
assert "fetched=2" in result.stdout
assert "normalized=1" in result.stdout
assert "deduplicated=1" in result.stdout
assert "normalized=2" in result.stdout
assert "deduplicated=2" in result.stdout
assert "failed=1" in result.stdout
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
@ -332,6 +336,19 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_successful_li
listings_payload = yaml.safe_load((run_dir / "listings.yaml").read_text(encoding="utf-8"))
assert listings_payload == [
{
"source": "apec",
"source_job_id": "job-123",
"url": "https://example.test/job/123",
"title": "Role From YAML",
"company": "Example Corp",
"location": "Paris - 75",
"contract_type": "CDI",
"description_text": "Build pipelines",
"published_at": "2026-04-20",
"fetched_at": "2026-06-01T10:00:00Z",
"warnings": [],
},
{
"source": "apec",
"source_job_id": "job-456",
@ -344,19 +361,148 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_successful_li
"published_at": "2026-04-20",
"fetched_at": "2026-06-01T10:00:00Z",
"warnings": [],
}
},
]
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload["failed_count"] == 1
assert run_meta_payload["listing_errors"] == [
assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z",
"run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"],
"fetched_count": 2,
"normalized_count": 2,
"deduplicated_count": 2,
"failed_count": 1,
"listing_errors": [
{
"url": first_result.url,
"stage": "snapshot_write",
"message": "disk full",
}
],
}
def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails(
tmp_path,
monkeypatch,
) -> None:
data_root = tmp_path / "data"
data_root.mkdir()
(data_root / "candidate-profile.yaml").write_text(
dedent(
"""
target_roles:
- Role From YAML
"""
).strip(),
encoding="utf-8",
)
html = _apec_detail_html()
first_result = ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123")
second_result = ApecSearchResult(url="https://example.test/job/456", source_job_id="job-456")
class SnapshotWriteFailingAdapter:
instances: list["SnapshotWriteFailingAdapter"] = []
def __init__(self, max_listings: int = 50) -> None:
self.max_listings = max_listings
self.search_calls: list[list[str]] = []
self.fetch_calls: list[str] = []
SnapshotWriteFailingAdapter.instances.append(self)
def search(self, queries: list[str]) -> list[ApecSearchResult]:
self.search_calls.append(list(queries))
return [first_result, second_result]
def fetch_listing_html(self, url: str) -> str:
self.fetch_calls.append(url)
return html
original_write_text = Path.write_text
def write_text_with_all_snapshot_failures(
self: Path,
data: str,
encoding: str | None = None,
errors: str | None = None,
newline: str | None = None,
) -> int:
if self.parent.name == "snapshots":
raise OSError("disk full")
return original_write_text(self, data, encoding=encoding, errors=errors, newline=newline)
monkeypatch.setattr("job_research.cli.ApecAdapter", SnapshotWriteFailingAdapter)
monkeypatch.setattr("job_research.cli._utc_now", _fixed_now)
monkeypatch.setattr(Path, "write_text", write_text_with_all_snapshot_failures)
result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)])
assert result.exit_code == 0
assert "fetched=2" in result.stdout
assert "normalized=2" in result.stdout
assert "deduplicated=2" in result.stdout
assert "failed=2" in result.stdout
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
assert snapshot_files == []
listings_payload = yaml.safe_load((run_dir / "listings.yaml").read_text(encoding="utf-8"))
assert listings_payload == [
{
"url": first_result.url,
"stage": "snapshot_write",
"message": "disk full",
}
"source": "apec",
"source_job_id": "job-123",
"url": "https://example.test/job/123",
"title": "Role From YAML",
"company": "Example Corp",
"location": "Paris - 75",
"contract_type": "CDI",
"description_text": "Build pipelines",
"published_at": "2026-04-20",
"fetched_at": "2026-06-01T10:00:00Z",
"warnings": [],
},
{
"source": "apec",
"source_job_id": "job-456",
"url": "https://example.test/job/456",
"title": "Role From YAML",
"company": "Example Corp",
"location": "Paris - 75",
"contract_type": "CDI",
"description_text": "Build pipelines",
"published_at": "2026-04-20",
"fetched_at": "2026-06-01T10:00:00Z",
"warnings": [],
},
]
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
assert run_meta_payload == {
"run_id": "2026-06-01T10-00-00Z",
"run_started_at": "2026-06-01T10:00:00Z",
"derived_queries": ["Role From YAML"],
"fetched_count": 2,
"normalized_count": 2,
"deduplicated_count": 2,
"failed_count": 2,
"listing_errors": [
{
"url": first_result.url,
"stage": "snapshot_write",
"message": "disk full",
},
{
"url": second_result.url,
"stage": "snapshot_write",
"message": "disk full",
},
],
}
def test_fetch_apec_fails_when_no_queries_are_derived(tmp_path, monkeypatch) -> None:
data_root = tmp_path / "data"

4
tests/test_apec_storage.py
View File

@ -24,6 +24,8 @@ def test_apec_models_serialize_expected_listing_shape() -> None:
],
)
run_meta = ApecRunMeta(
run_id="2026-06-01T10-00-00Z",
run_started_at="2026-06-01T10:00:00Z",
derived_queries=["Data Engineer"],
fetched_count=1,
normalized_count=1,
@ -34,6 +36,8 @@ def test_apec_models_serialize_expected_listing_shape() -> None:
assert listing.model_dump()["source"] == "apec"
assert listing.model_dump()["warnings"][0]["field"] == "location"
assert run_meta.model_dump()["run_id"] == "2026-06-01T10-00-00Z"
assert run_meta.model_dump()["run_started_at"] == "2026-06-01T10:00:00Z"
assert run_meta.model_dump()["derived_queries"] == ["Data Engineer"]