diff --git a/src/job_research/apec/query_derivation.py b/src/job_research/apec/query_derivation.py index 437acab..6dc9c56 100644 --- a/src/job_research/apec/query_derivation.py +++ b/src/job_research/apec/query_derivation.py @@ -5,15 +5,35 @@ def derive_apec_queries(profile: CandidateProfileOutput) -> list[str]: queries: list[str] = [] seen: set[str] = set() - for target_role in profile.target_roles: - query = target_role.strip() - if not query or query in seen: - continue + def add_query(raw_query: str) -> None: + query = " ".join(raw_query.split()) + if not query or query in seen or len(queries) == 5: + return seen.add(query) queries.append(query) - if len(queries) == 5: - break + unique_roles: list[str] = [] + for target_role in profile.target_roles: + query = " ".join(target_role.split()) + if not query or query in unique_roles: + continue + + unique_roles.append(query) + + for target_role in unique_roles[:2]: + add_query(target_role) + + if unique_roles: + primary_role = unique_roles[0] + for term in profile.strengths + profile.skills_to_emphasize: + add_query(f"{primary_role} {term}") + if len(queries) == 5: + break + else: + for term in profile.strengths + profile.skills_to_emphasize: + add_query(term) + if len(queries) == 5: + break return queries diff --git a/src/job_research/cli.py b/src/job_research/cli.py index 0700f0e..166d3e4 100644 --- a/src/job_research/cli.py +++ b/src/job_research/cli.py @@ -12,7 +12,7 @@ from job_research.apec.adapter import ApecAdapter from job_research.apec.dedupe import dedupe_apec_listings from job_research.apec.normalize import normalize_apec_listing from job_research.apec.query_derivation import derive_apec_queries -from job_research.models import ApecRunMeta, CandidateProfileOutput, ListingError +from job_research.models import ApecRunMeta, ApecSnapshotMeta, CandidateProfileOutput, ListingError from job_research.profile.cv_extractor import extract_cv_signals, extract_pdf_text from job_research.profile.merge import build_candidate_profile_output from job_research.profile.profile_parser import parse_profile_markdown @@ -48,6 +48,8 @@ def _load_candidate_profile(profile_path: Path) -> CandidateProfileOutput: return CandidateProfileOutput.model_validate(load_yaml(profile_path)) except FileNotFoundError as exc: raise ValueError(f"candidate-profile.yaml not found at {profile_path}") from exc + except (OSError, UnicodeDecodeError) as exc: + raise ValueError(f"candidate-profile.yaml not readable at {profile_path}: {exc}") from exc except (yaml.YAMLError, ValidationError, ValueError) as exc: raise ValueError(f"invalid candidate-profile.yaml at {profile_path}: {exc}") from exc @@ -132,14 +134,20 @@ def fetch_apec( run_id = current.strftime("%Y-%m-%dT%H-%M-%SZ") run_started_at = current.strftime("%Y-%m-%dT%H:%M:%SZ") fetched_at = run_started_at + + adapter = ApecAdapter(max_listings=50) + try: + search_results = adapter.search(derived_queries)[:50] + except Exception as exc: # pragma: no cover - defensive boundary + typer.echo(f"Unable to fetch Apec search results: {exc}", err=True) + raise typer.Exit(code=1) + paths = apec_run_paths(data_root, run_id) paths["snapshots"].mkdir(parents=True, exist_ok=True) - adapter = ApecAdapter(max_listings=50) - search_results = adapter.search(derived_queries)[:50] - normalized_listings = [] listing_errors: list[ListingError] = [] + snapshot_metadata: list[ApecSnapshotMeta] = [] fetched_count = 0 successful_fetch_count = 0 @@ -154,11 +162,22 @@ def fetch_apec( successful_fetch_count += 1 + snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html" + snapshot_meta = ApecSnapshotMeta( + url=result.url, + source_job_id=result.source_job_id, + snapshot_file=None, + fetched_at=fetched_at, + ) + try: - snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html" snapshot_path.write_text(html, encoding="utf-8") except Exception as exc: # pragma: no cover - defensive boundary listing_errors.append(ListingError(url=result.url, stage="snapshot_write", message=str(exc))) + else: + snapshot_meta.snapshot_file = snapshot_path.name + + snapshot_metadata.append(snapshot_meta) try: listing = normalize_apec_listing( @@ -179,14 +198,16 @@ def fetch_apec( deduplicated_listings = dedupe_apec_listings(normalized_listings) should_fail_after_write = successful_fetch_count > 0 and not normalized_listings + failed_count = len({error.url for error in listing_errors}) run_meta = ApecRunMeta( run_id=run_id, run_started_at=run_started_at, derived_queries=derived_queries, + snapshots=snapshot_metadata, fetched_count=fetched_count, normalized_count=len(normalized_listings), deduplicated_count=len(deduplicated_listings), - failed_count=len(listing_errors), + failed_count=failed_count, listing_errors=listing_errors, ) @@ -195,7 +216,7 @@ def fetch_apec( typer.echo( f"query={len(derived_queries)} fetched={fetched_count} normalized={len(normalized_listings)} " - f"deduplicated={len(deduplicated_listings)} failed={len(listing_errors)}" + f"deduplicated={len(deduplicated_listings)} failed={failed_count}" ) if should_fail_after_write: diff --git a/src/job_research/models.py b/src/job_research/models.py index 6646469..df68499 100644 --- a/src/job_research/models.py +++ b/src/job_research/models.py @@ -32,6 +32,13 @@ class ListingError(BaseModel): message: str +class ApecSnapshotMeta(BaseModel): + url: str + source_job_id: str | None = None + snapshot_file: str | None = None + fetched_at: str + + class ApecListing(BaseModel): source: str source_job_id: str | None = None @@ -50,6 +57,7 @@ class ApecRunMeta(BaseModel): run_id: str run_started_at: str derived_queries: list[str] = Field(default_factory=list) + snapshots: list[ApecSnapshotMeta] = Field(default_factory=list) fetched_count: int = 0 normalized_count: int = 0 deduplicated_count: int = 0 diff --git a/tests/apec/test_query_derivation.py b/tests/apec/test_query_derivation.py index 6f2f949..312a930 100644 --- a/tests/apec/test_query_derivation.py +++ b/tests/apec/test_query_derivation.py @@ -12,7 +12,9 @@ def test_derive_apec_queries_preserves_order_dedupes_and_caps_at_five() -> None: "Junior Data Platform Engineer", "ML Engineer", "Backend Engineer", - ] + ], + strengths=["Python", "SQL"], + skills_to_emphasize=["BigQuery", "Terraform"], ) queries = derive_apec_queries(profile) @@ -20,7 +22,7 @@ def test_derive_apec_queries_preserves_order_dedupes_and_caps_at_five() -> None: assert queries == [ "Data Engineer", "Analytics Engineer", - "BI Engineer", - "Junior Data Platform Engineer", - "ML Engineer", + "Data Engineer Python", + "Data Engineer SQL", + "Data Engineer BigQuery", ] diff --git a/tests/test_apec_cli.py b/tests/test_apec_cli.py index f1a75b1..e3d1823 100644 --- a/tests/test_apec_cli.py +++ b/tests/test_apec_cli.py @@ -6,6 +6,7 @@ import yaml from typer.testing import CliRunner from job_research.apec.adapter import ApecSearchResult +from job_research.apec.normalize import normalize_apec_listing as real_normalize_apec_listing from job_research.cli import app @@ -79,6 +80,19 @@ def _apec_detail_html( ).strip() +def _snapshot_meta( + url: str, + source_job_id: str | None, + snapshot_file: str | None, +) -> dict[str, str | None]: + return { + "url": url, + "source_job_id": source_job_id, + "snapshot_file": snapshot_file, + "fetched_at": "2026-06-01T10:00:00Z", + } + + def test_fetch_apec_fails_cleanly_when_candidate_profile_is_missing(tmp_path, monkeypatch) -> None: data_root = tmp_path / "data" data_root.mkdir() @@ -119,6 +133,82 @@ def test_fetch_apec_fails_cleanly_when_candidate_profile_yaml_is_malformed( assert not (data_root / "apec").exists() +def test_fetch_apec_fails_cleanly_when_candidate_profile_file_is_unreadable( + tmp_path, + monkeypatch, +) -> None: + data_root = tmp_path / "data" + data_root.mkdir() + (data_root / "candidate-profile.yaml").write_text( + dedent( + """ + target_roles: + - Role From YAML + """ + ).strip(), + encoding="utf-8", + ) + + def unreadable_load_yaml(path: Path) -> dict[str, object]: + raise PermissionError("Permission denied") + + class SpyApecAdapter: + def __init__(self, max_listings: int = 50) -> None: + raise AssertionError("adapter should not be created when the profile is unreadable") + + monkeypatch.setattr("job_research.cli.load_yaml", unreadable_load_yaml) + monkeypatch.setattr("job_research.cli.ApecAdapter", SpyApecAdapter) + + result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)]) + + assert result.exit_code == 1 + assert "candidate-profile.yaml not readable" in result.stderr + assert "Traceback" not in result.stderr + assert not (data_root / "apec").exists() + + +def test_fetch_apec_fails_cleanly_when_adapter_search_raises( + tmp_path, + monkeypatch, +) -> None: + data_root = tmp_path / "data" + data_root.mkdir() + (data_root / "candidate-profile.yaml").write_text( + dedent( + """ + target_roles: + - Role From YAML + """ + ).strip(), + encoding="utf-8", + ) + + class FailingSearchAdapter: + instances: list["FailingSearchAdapter"] = [] + + def __init__(self, max_listings: int = 50) -> None: + self.max_listings = max_listings + self.search_calls: list[list[str]] = [] + FailingSearchAdapter.instances.append(self) + + def search(self, queries: list[str]) -> list[ApecSearchResult]: + self.search_calls.append(list(queries)) + raise RuntimeError("search boom") + + def fetch_listing_html(self, url: str) -> str: + raise AssertionError("fetch should not be called when search fails") + + monkeypatch.setattr("job_research.cli.ApecAdapter", FailingSearchAdapter) + + result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)]) + + assert result.exit_code == 1 + assert "Unable to fetch Apec search results: search boom" in result.stderr + assert "Traceback" not in result.stderr + assert not (data_root / "apec").exists() + assert FailingSearchAdapter.instances[0].search_calls == [["Role From YAML"]] + + def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch) -> None: data_root = tmp_path / "data" data_root.mkdir() @@ -203,6 +293,13 @@ def test_fetch_apec_reads_profile_and_writes_run_artifacts(tmp_path, monkeypatch "run_id": "2026-06-01T10-00-00Z", "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], + "snapshots": [ + _snapshot_meta( + "https://example.test/job/123", + "job-123", + "job-123.html", + ) + ], "fetched_count": 1, "normalized_count": 1, "deduplicated_count": 1, @@ -293,6 +390,9 @@ def test_fetch_apec_records_partial_failures_without_losing_successful_listings( "run_id": "2026-06-01T10-00-00Z", "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], + "snapshots": [ + _snapshot_meta(good_result.url, "job-123", "job-123.html") + ], "fetched_count": 2, "normalized_count": 1, "deduplicated_count": 1, @@ -409,6 +509,10 @@ def test_fetch_apec_records_snapshot_write_failures_without_losing_normalized_li "run_id": "2026-06-01T10-00-00Z", "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], + "snapshots": [ + _snapshot_meta(first_result.url, "job-123", None), + _snapshot_meta(second_result.url, "job-456", "job-456.html"), + ], "fetched_count": 2, "normalized_count": 2, "deduplicated_count": 2, @@ -525,6 +629,10 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails( "run_id": "2026-06-01T10-00-00Z", "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], + "snapshots": [ + _snapshot_meta(first_result.url, "job-123", None), + _snapshot_meta(second_result.url, "job-456", None), + ], "fetched_count": 2, "normalized_count": 2, "deduplicated_count": 2, @@ -544,6 +652,109 @@ def test_fetch_apec_writes_run_meta_even_when_every_snapshot_write_fails( } +def test_fetch_apec_counts_each_failed_url_once_even_when_multiple_errors_are_recorded( + tmp_path, + monkeypatch, +) -> None: + data_root = tmp_path / "data" + data_root.mkdir() + (data_root / "candidate-profile.yaml").write_text( + dedent( + """ + target_roles: + - Role From YAML + """ + ).strip(), + encoding="utf-8", + ) + + html = _apec_detail_html() + first_result = ApecSearchResult(url="https://example.test/job/123", source_job_id="job-123") + second_result = ApecSearchResult(url="https://example.test/job/456", source_job_id="job-456") + + class MixedErrorAdapter: + instances: list["MixedErrorAdapter"] = [] + + def __init__(self, max_listings: int = 50) -> None: + self.max_listings = max_listings + self.search_calls: list[list[str]] = [] + self.fetch_calls: list[str] = [] + MixedErrorAdapter.instances.append(self) + + def search(self, queries: list[str]) -> list[ApecSearchResult]: + self.search_calls.append(list(queries)) + return [first_result, second_result] + + def fetch_listing_html(self, url: str) -> str: + self.fetch_calls.append(url) + return html + + original_write_text = Path.write_text + + def write_text_with_one_snapshot_failure( + self: Path, + data: str, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> int: + if self.parent.name == "snapshots" and self.name == "job-123.html": + raise OSError("disk full") + + return original_write_text(self, data, encoding=encoding, errors=errors, newline=newline) + + def normalize_with_first_url_failure(*args, **kwargs): + if kwargs["url"] == first_result.url: + raise RuntimeError("normalize boom") + + return real_normalize_apec_listing(*args, **kwargs) + + monkeypatch.setattr("job_research.cli.ApecAdapter", MixedErrorAdapter) + monkeypatch.setattr("job_research.cli.normalize_apec_listing", normalize_with_first_url_failure) + monkeypatch.setattr("job_research.cli._utc_now", _fixed_now) + monkeypatch.setattr(Path, "write_text", write_text_with_one_snapshot_failure) + + result = CliRunner().invoke(app, ["fetch-apec", "--data-root", str(data_root)]) + + assert result.exit_code == 0 + assert "fetched=2" in result.stdout + assert "normalized=1" in result.stdout + assert "deduplicated=1" in result.stdout + assert "failed=1" in result.stdout + + run_dir = data_root / "apec" / "runs" / "2026-06-01T10-00-00Z" + snapshot_files = sorted((run_dir / "snapshots").glob("*.html")) + assert [snapshot.name for snapshot in snapshot_files] == ["job-456.html"] + assert snapshot_files[0].read_text(encoding="utf-8") == html + + run_meta_payload = yaml.safe_load((run_dir / "run-meta.yaml").read_text(encoding="utf-8")) + assert run_meta_payload == { + "run_id": "2026-06-01T10-00-00Z", + "run_started_at": "2026-06-01T10:00:00Z", + "derived_queries": ["Role From YAML"], + "snapshots": [ + _snapshot_meta(first_result.url, "job-123", None), + _snapshot_meta(second_result.url, "job-456", "job-456.html"), + ], + "fetched_count": 2, + "normalized_count": 1, + "deduplicated_count": 1, + "failed_count": 1, + "listing_errors": [ + { + "url": first_result.url, + "stage": "snapshot_write", + "message": "disk full", + }, + { + "url": first_result.url, + "stage": "normalize", + "message": "normalize boom", + }, + ], + } + + def test_fetch_apec_preserves_snapshot_and_run_meta_when_normalization_fails( tmp_path, monkeypatch, @@ -608,6 +819,9 @@ def test_fetch_apec_preserves_snapshot_and_run_meta_when_normalization_fails( "run_id": "2026-06-01T10-00-00Z", "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], + "snapshots": [ + _snapshot_meta(result_item.url, "job-123", "job-123.html") + ], "fetched_count": 1, "normalized_count": 0, "deduplicated_count": 0, @@ -687,6 +901,10 @@ def test_fetch_apec_writes_artifacts_when_every_normalization_attempt_fails( "run_id": "2026-06-01T10-00-00Z", "run_started_at": "2026-06-01T10:00:00Z", "derived_queries": ["Role From YAML"], + "snapshots": [ + _snapshot_meta(first_result.url, "job-123", "job-123.html"), + _snapshot_meta(second_result.url, "job-456", "job-456.html"), + ], "fetched_count": 2, "normalized_count": 0, "deduplicated_count": 0, diff --git a/tests/test_apec_storage.py b/tests/test_apec_storage.py index d71f625..caa77c3 100644 --- a/tests/test_apec_storage.py +++ b/tests/test_apec_storage.py @@ -1,6 +1,6 @@ from pathlib import Path -from job_research.models import ApecListing, ApecRunMeta, ListingWarning +from job_research.models import ApecListing, ApecRunMeta, ApecSnapshotMeta, ListingWarning from job_research.storage import apec_run_paths @@ -27,6 +27,14 @@ def test_apec_models_serialize_expected_listing_shape() -> None: run_id="2026-06-01T10-00-00Z", run_started_at="2026-06-01T10:00:00Z", derived_queries=["Data Engineer"], + snapshots=[ + ApecSnapshotMeta( + url="https://example.test/job/123", + source_job_id="123", + snapshot_file="job-123.html", + fetched_at="2026-06-01T10:00:00Z", + ) + ], fetched_count=1, normalized_count=1, deduplicated_count=1, @@ -39,6 +47,14 @@ def test_apec_models_serialize_expected_listing_shape() -> None: assert run_meta.model_dump()["run_id"] == "2026-06-01T10-00-00Z" assert run_meta.model_dump()["run_started_at"] == "2026-06-01T10:00:00Z" assert run_meta.model_dump()["derived_queries"] == ["Data Engineer"] + assert run_meta.model_dump(mode="json")["snapshots"] == [ + { + "url": "https://example.test/job/123", + "source_job_id": "123", + "snapshot_file": "job-123.html", + "fetched_at": "2026-06-01T10:00:00Z", + } + ] def test_apec_run_paths_builds_expected_layout(tmp_path: Path) -> None: