fix: hard-fail fetch-apec contract gaps
This commit is contained in:
parent
97ed98c5de
commit
52b21b65c5
@ -94,6 +94,10 @@ def fetch_apec(
|
|||||||
profile = CandidateProfileOutput.model_validate(load_yaml(profile_path))
|
profile = CandidateProfileOutput.model_validate(load_yaml(profile_path))
|
||||||
derived_queries = derive_apec_queries(profile)
|
derived_queries = derive_apec_queries(profile)
|
||||||
|
|
||||||
|
if not derived_queries:
|
||||||
|
typer.echo("No usable Apec queries derived from candidate profile", err=True)
|
||||||
|
raise typer.Exit(code=1)
|
||||||
|
|
||||||
current = _utc_now().astimezone(timezone.utc).replace(microsecond=0)
|
current = _utc_now().astimezone(timezone.utc).replace(microsecond=0)
|
||||||
run_id = current.strftime("%Y-%m-%dT%H-%M-%SZ")
|
run_id = current.strftime("%Y-%m-%dT%H-%M-%SZ")
|
||||||
fetched_at = current.strftime("%Y-%m-%dT%H:%M:%SZ")
|
fetched_at = current.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
@ -101,20 +105,21 @@ def fetch_apec(
|
|||||||
paths["snapshots"].mkdir(parents=True, exist_ok=True)
|
paths["snapshots"].mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
adapter = ApecAdapter(max_listings=50)
|
adapter = ApecAdapter(max_listings=50)
|
||||||
search_results = adapter.search(derived_queries)
|
search_results = adapter.search(derived_queries)[:50]
|
||||||
|
|
||||||
normalized_listings = []
|
normalized_listings = []
|
||||||
listing_errors: list[ListingError] = []
|
listing_errors: list[ListingError] = []
|
||||||
fetched_count = 0
|
fetched_count = 0
|
||||||
|
|
||||||
for index, result in enumerate(search_results, start=1):
|
for index, result in enumerate(search_results, start=1):
|
||||||
|
fetched_count += 1
|
||||||
|
|
||||||
try:
|
try:
|
||||||
html = adapter.fetch_listing_html(result.url)
|
html = adapter.fetch_listing_html(result.url)
|
||||||
except Exception as exc: # pragma: no cover - defensive boundary
|
except Exception as exc: # pragma: no cover - defensive boundary
|
||||||
listing_errors.append(ListingError(url=result.url, stage="fetch_html", message=str(exc)))
|
listing_errors.append(ListingError(url=result.url, stage="fetch_html", message=str(exc)))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
fetched_count += 1
|
|
||||||
snapshot_path = paths["snapshots"] / f"{index:04d}.html"
|
snapshot_path = paths["snapshots"] / f"{index:04d}.html"
|
||||||
snapshot_path.write_text(html, encoding="utf-8")
|
snapshot_path.write_text(html, encoding="utf-8")
|
||||||
|
|
||||||
@ -131,6 +136,10 @@ def fetch_apec(
|
|||||||
|
|
||||||
normalized_listings.append(listing)
|
normalized_listings.append(listing)
|
||||||
|
|
||||||
|
if fetched_count > 0 and not normalized_listings:
|
||||||
|
typer.echo("No listings could be fetched or normalized from Apec", err=True)
|
||||||
|
raise typer.Exit(code=1)
|
||||||
|
|
||||||
deduplicated_listings = dedupe_apec_listings(normalized_listings)
|
deduplicated_listings = dedupe_apec_listings(normalized_listings)
|
||||||
run_meta = ApecRunMeta(
|
run_meta = ApecRunMeta(
|
||||||
derived_queries=derived_queries,
|
derived_queries=derived_queries,
|
||||||
|
|||||||
@ -8,6 +8,10 @@ from job_research.apec.adapter import ApecSearchResult
|
|||||||
from job_research.cli import app
|
from job_research.cli import app
|
||||||
|
|
||||||
|
|
||||||
|
def _fixed_now() -> datetime:
|
||||||
|
return datetime(2026, 6, 1, 10, 0, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch) -> None:
|
def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch) -> None:
|
||||||
data_root = tmp_path / "data"
|
data_root = tmp_path / "data"
|
||||||
data_root.mkdir()
|
data_root.mkdir()
|
||||||
@ -55,7 +59,7 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch
|
|||||||
monkeypatch.setattr("job_research.cli.ApecAdapter", FakeApecAdapter)
|
monkeypatch.setattr("job_research.cli.ApecAdapter", FakeApecAdapter)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"job_research.cli._utc_now",
|
"job_research.cli._utc_now",
|
||||||
lambda: datetime(2026, 6, 1, 10, 0, 0, tzinfo=timezone.utc),
|
_fixed_now,
|
||||||
)
|
)
|
||||||
|
|
||||||
result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)])
|
result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)])
|
||||||
@ -107,3 +111,158 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch
|
|||||||
"failed_count": 0,
|
"failed_count": 0,
|
||||||
"listing_errors": [],
|
"listing_errors": [],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_apec_fails_when_no_queries_are_derived(tmp_path, monkeypatch) -> None:
|
||||||
|
data_root = tmp_path / "data"
|
||||||
|
data_root.mkdir()
|
||||||
|
(data_root / "candidate-profile.yaml").write_text("target_roles: []\n", encoding="utf-8")
|
||||||
|
|
||||||
|
class SpyApecAdapter:
|
||||||
|
instances: list["SpyApecAdapter"] = []
|
||||||
|
|
||||||
|
def __init__(self, max_listings: int = 50) -> None:
|
||||||
|
self.max_listings = max_listings
|
||||||
|
SpyApecAdapter.instances.append(self)
|
||||||
|
|
||||||
|
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||||
|
raise AssertionError("search should not be called when there are no queries")
|
||||||
|
|
||||||
|
def fetch_listing_html(self, url: str) -> str:
|
||||||
|
raise AssertionError("fetch should not be called when there are no queries")
|
||||||
|
|
||||||
|
monkeypatch.setattr("job_research.cli.ApecAdapter", SpyApecAdapter)
|
||||||
|
monkeypatch.setattr("job_research.cli._utc_now", _fixed_now)
|
||||||
|
|
||||||
|
result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)])
|
||||||
|
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "No usable Apec queries derived from candidate profile" in result.stderr
|
||||||
|
assert SpyApecAdapter.instances == []
|
||||||
|
assert not (data_root / "apec").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_apec_processes_only_the_first_fifty_search_results(tmp_path, monkeypatch) -> None:
|
||||||
|
data_root = tmp_path / "data"
|
||||||
|
data_root.mkdir()
|
||||||
|
(data_root / "candidate-profile.yaml").write_text(
|
||||||
|
dedent(
|
||||||
|
"""
|
||||||
|
target_roles:
|
||||||
|
- Role From YAML
|
||||||
|
"""
|
||||||
|
).strip(),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
html = dedent(
|
||||||
|
"""
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<h1>Role From YAML</h1>
|
||||||
|
<div class="company">Example Corp</div>
|
||||||
|
<div class="location">Paris</div>
|
||||||
|
<div class="contract">CDI</div>
|
||||||
|
<div class="description">Build pipelines</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
).strip()
|
||||||
|
results = [
|
||||||
|
ApecSearchResult(url=f"https://example.test/job/{index}", source_job_id=f"job-{index}")
|
||||||
|
for index in range(51)
|
||||||
|
]
|
||||||
|
|
||||||
|
class FakeApecAdapter:
|
||||||
|
instances: list["FakeApecAdapter"] = []
|
||||||
|
|
||||||
|
def __init__(self, max_listings: int = 50) -> None:
|
||||||
|
self.max_listings = max_listings
|
||||||
|
self.search_calls: list[list[str]] = []
|
||||||
|
self.fetch_calls: list[str] = []
|
||||||
|
FakeApecAdapter.instances.append(self)
|
||||||
|
|
||||||
|
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||||
|
self.search_calls.append(list(queries))
|
||||||
|
return list(results)
|
||||||
|
|
||||||
|
def fetch_listing_html(self, url: str) -> str:
|
||||||
|
self.fetch_calls.append(url)
|
||||||
|
return html
|
||||||
|
|
||||||
|
monkeypatch.setattr("job_research.cli.ApecAdapter", FakeApecAdapter)
|
||||||
|
monkeypatch.setattr("job_research.cli._utc_now", _fixed_now)
|
||||||
|
|
||||||
|
result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)])
|
||||||
|
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "fetched=50" in result.stdout
|
||||||
|
assert "normalized=50" in result.stdout
|
||||||
|
assert "deduplicated=50" in result.stdout
|
||||||
|
assert "failed=0" in result.stdout
|
||||||
|
|
||||||
|
adapter = FakeApecAdapter.instances[0]
|
||||||
|
assert adapter.max_listings == 50
|
||||||
|
assert adapter.search_calls == [["Role From YAML"]]
|
||||||
|
assert adapter.fetch_calls == [result.url for result in results[:50]]
|
||||||
|
|
||||||
|
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
||||||
|
snapshot_files = sorted((run_dir / "snapshots").glob("*.html"))
|
||||||
|
assert len(snapshot_files) == 50
|
||||||
|
|
||||||
|
run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8"))
|
||||||
|
assert run_meta_payload["fetched_count"] == 50
|
||||||
|
assert run_meta_payload["normalized_count"] == 50
|
||||||
|
assert run_meta_payload["deduplicated_count"] == 50
|
||||||
|
assert run_meta_payload["failed_count"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_apec_fails_when_every_listing_attempt_fails(tmp_path, monkeypatch) -> None:
|
||||||
|
data_root = tmp_path / "data"
|
||||||
|
data_root.mkdir()
|
||||||
|
(data_root / "candidate-profile.yaml").write_text(
|
||||||
|
dedent(
|
||||||
|
"""
|
||||||
|
target_roles:
|
||||||
|
- Role From YAML
|
||||||
|
"""
|
||||||
|
).strip(),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
results = [
|
||||||
|
ApecSearchResult(url=f"https://example.test/job/{index}", source_job_id=f"job-{index}")
|
||||||
|
for index in range(2)
|
||||||
|
]
|
||||||
|
|
||||||
|
class FailingApecAdapter:
|
||||||
|
instances: list["FailingApecAdapter"] = []
|
||||||
|
|
||||||
|
def __init__(self, max_listings: int = 50) -> None:
|
||||||
|
self.max_listings = max_listings
|
||||||
|
self.search_calls: list[list[str]] = []
|
||||||
|
self.fetch_calls: list[str] = []
|
||||||
|
FailingApecAdapter.instances.append(self)
|
||||||
|
|
||||||
|
def search(self, queries: list[str]) -> list[ApecSearchResult]:
|
||||||
|
self.search_calls.append(list(queries))
|
||||||
|
return list(results)
|
||||||
|
|
||||||
|
def fetch_listing_html(self, url: str) -> str:
|
||||||
|
self.fetch_calls.append(url)
|
||||||
|
raise RuntimeError(f"boom: {url}")
|
||||||
|
|
||||||
|
monkeypatch.setattr("job_research.cli.ApecAdapter", FailingApecAdapter)
|
||||||
|
monkeypatch.setattr("job_research.cli._utc_now", _fixed_now)
|
||||||
|
|
||||||
|
result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)])
|
||||||
|
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "No listings could be fetched or normalized from Apec" in result.stderr
|
||||||
|
|
||||||
|
adapter = FailingApecAdapter.instances[0]
|
||||||
|
assert adapter.search_calls == [["Role From YAML"]]
|
||||||
|
assert adapter.fetch_calls == [result_item.url for result_item in results]
|
||||||
|
|
||||||
|
run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z"
|
||||||
|
assert not (run_dir / "listings.yaml").exists()
|
||||||
|
assert not (run_dir / "run-meta.yaml").exists()
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user