job-research/src/job_research/cli.py

from datetime import datetime, timezone
import re
from pathlib import Path
from typing import Any
from urllib.parse import unquote, urlparse

import typer
import yaml
from pydantic import ValidationError

from job_research.apec.adapter import ApecAdapter, ApecSearchFilters
from job_research.apec.dedupe import dedupe_apec_listings
from job_research.apec.normalize import normalize_apec_listing
from job_research.apec.query_derivation import derive_apec_queries, derive_apec_search_filters
from job_research.models import ApecRunMeta, ApecSnapshotMeta, CandidateProfileOutput, ListingError
from job_research.profile.cv_extractor import extract_cv_signals, extract_pdf_text
from job_research.profile.merge import build_candidate_profile_output
from job_research.profile.profile_parser import parse_profile_markdown
from job_research.storage import apec_run_paths, load_yaml, save_candidate_profile_yaml

app = typer.Typer(help="Build one canonical candidate profile YAML")


def _utc_now() -> datetime:
    return datetime.now(timezone.utc)


def _snapshot_stem(url: str, source_job_id: str | None) -> str:
    if source_job_id:
        return source_job_id

    parsed_url = urlparse(url)
    fallback = parsed_url.path.rstrip("/").rsplit("/", 1)[-1] or parsed_url.netloc or "listing"
    if parsed_url.query:
        fallback = f"{fallback}-{parsed_url.query}"

    stem = re.sub(r"[^A-Za-z0-9]+", "-", unquote(fallback)).strip("-")
    return stem or "listing"


def _write_yaml(path: Path, payload: Any) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8")


def _load_candidate_profile(profile_path: Path) -> CandidateProfileOutput:
    try:
        return CandidateProfileOutput.model_validate(load_yaml(profile_path))
    except FileNotFoundError as exc:
        raise ValueError(f"candidate-profile.yaml not found at {profile_path}") from exc
    except (OSError, UnicodeDecodeError) as exc:
        raise ValueError(f"candidate-profile.yaml not readable at {profile_path}: {exc}") from exc
    except (yaml.YAMLError, ValidationError, ValueError) as exc:
        raise ValueError(f"invalid candidate-profile.yaml at {profile_path}: {exc}") from exc


@app.callback()
def main_command() -> None:
    pass

@app.command("build-profile")
def build_profile(
    cv: Path = typer.Option(
        ...,
        "--cv",
        exists=True,
        dir_okay=False,
        readable=True,
        help="Path to the CV PDF or UTF-8 text file.",
    ),
    profile: Path = typer.Option(
        ...,
        "--profile",
        exists=True,
        dir_okay=False,
        readable=True,
        help="Path to the light-template markdown profile.",
    ),
    out: Path = typer.Option(
        Path("data/candidate-profile.yaml"),
        "--out",
        dir_okay=False,
        help="Path to write the canonical YAML profile.",
    ),
) -> None:
    """Build candidate-profile.yaml from CV and markdown profile."""

    cv_text = extract_pdf_text(cv) if cv.suffix.lower() == ".pdf" else cv.read_text(encoding="utf-8")
    if not cv_text.strip():
        raise ValueError("No readable text found in CV input")

    authored_profile = parse_profile_markdown(profile.read_text(encoding="utf-8"))
    cv_signals = extract_cv_signals(cv_text)
    candidate_profile = build_candidate_profile_output(cv_signals, authored_profile)

    save_candidate_profile_yaml(out, candidate_profile)

    typer.echo(f"candidate profile written to {out}")
    warning_count = len(candidate_profile.warnings)
    if warning_count:
        typer.echo(f"Warnings included: {warning_count}")
    else:
        typer.echo("No warnings included.")


@app.command("fetch-apec")
def fetch_apec(
    data_root: Path = typer.Option(
        Path("data"),
        "--data-root",
        file_okay=False,
        dir_okay=True,
        help="Directory containing candidate-profile.yaml and Apec run artifacts.",
    ),
    ) -> None:
    """Fetch, normalize, dedupe, and persist Apec listings."""

    profile_path = data_root / "candidate-profile.yaml"
    try:
        profile = _load_candidate_profile(profile_path)
    except ValueError as exc:
        typer.echo(str(exc), err=True)
        raise typer.Exit(code=1)

    derived_queries = derive_apec_queries(profile)

    if not derived_queries:
        typer.echo("No usable Apec queries derived from candidate profile", err=True)
        raise typer.Exit(code=1)

    derived_search_filters = derive_apec_search_filters(profile)
    search_filters = ApecSearchFilters(
        location=derived_search_filters.location or "France",
        contract_type=derived_search_filters.contract_type or "CDI",
    )

    current = _utc_now().astimezone(timezone.utc).replace(microsecond=0)
    run_id = current.strftime("%Y-%m-%dT%H-%M-%SZ")
    run_started_at = current.strftime("%Y-%m-%dT%H:%M:%SZ")
    fetched_at = run_started_at

    adapter = ApecAdapter(max_listings=50)
    try:
        search_results = adapter.search(derived_queries, search_filters=search_filters)[:50]
    except Exception as exc:  # pragma: no cover - defensive boundary
        typer.echo(f"Unable to fetch Apec search results: {exc}", err=True)
        raise typer.Exit(code=1)

    paths = apec_run_paths(data_root, run_id)
    try:
        paths["snapshots"].mkdir(parents=True, exist_ok=True)
    except OSError as exc:  # pragma: no cover - defensive boundary
        typer.echo(f"Unable to create Apec snapshot directory: {exc}", err=True)
        raise typer.Exit(code=1)

    normalized_listings = []
    listing_errors: list[ListingError] = []
    snapshot_metadata: list[ApecSnapshotMeta] = []
    fetched_count = 0
    successful_fetch_count = 0

    for result in search_results:
        fetched_count += 1

        try:
            html = adapter.fetch_listing_html(result.url)
        except Exception as exc:  # pragma: no cover - defensive boundary
            listing_errors.append(ListingError(url=result.url, stage="fetch_html", message=str(exc)))
            continue

        successful_fetch_count += 1

        snapshot_path = paths["snapshots"] / f"{_snapshot_stem(result.url, result.source_job_id)}.html"
        snapshot_meta = ApecSnapshotMeta(
            url=result.url,
            source_job_id=result.source_job_id,
            snapshot_file=None,
            fetched_at=fetched_at,
        )

        try:
            snapshot_path.write_text(html, encoding="utf-8")
        except Exception as exc:  # pragma: no cover - defensive boundary
            listing_errors.append(ListingError(url=result.url, stage="snapshot_write", message=str(exc)))
        else:
            snapshot_meta.snapshot_file = snapshot_path.name

        snapshot_metadata.append(snapshot_meta)

        try:
            listing = normalize_apec_listing(
                url=result.url,
                html=html,
                fetched_at=fetched_at,
                source_job_id=result.source_job_id,
            )
        except Exception as exc:  # pragma: no cover - defensive boundary
            listing_errors.append(ListingError(url=result.url, stage="normalize", message=str(exc)))
            continue

        normalized_listings.append(listing)

    if search_results and successful_fetch_count == 0:
        typer.echo("No listings could be fetched or normalized from Apec", err=True)
        raise typer.Exit(code=1)

    deduplicated_listings = dedupe_apec_listings(normalized_listings)
    should_fail_after_write = successful_fetch_count > 0 and not normalized_listings
    failed_count = len({error.url for error in listing_errors})
    run_meta = ApecRunMeta(
        run_id=run_id,
        run_started_at=run_started_at,
        derived_queries=derived_queries,
        snapshots=snapshot_metadata,
        fetched_count=fetched_count,
        normalized_count=len(normalized_listings),
        deduplicated_count=len(deduplicated_listings),
        failed_count=failed_count,
        listing_errors=listing_errors,
    )

    artifact_write_errors: list[str] = []

    try:
        _write_yaml(paths["listings"], [listing.model_dump(mode="json") for listing in deduplicated_listings])
    except OSError as exc:  # pragma: no cover - defensive boundary
        artifact_write_errors.append(f"listings.yaml: {exc}")

    try:
        _write_yaml(paths["run_meta"], run_meta.model_dump(mode="json"))
    except OSError as exc:  # pragma: no cover - defensive boundary
        artifact_write_errors.append(f"run-meta.yaml: {exc}")

    if artifact_write_errors:
        typer.echo(f"Unable to write Apec run artifacts: {'; '.join(artifact_write_errors)}", err=True)
        raise typer.Exit(code=1)

    typer.echo(
        f"query={len(derived_queries)} fetched={fetched_count} normalized={len(normalized_listings)} "
        f"deduplicated={len(deduplicated_listings)} failed={failed_count}"
    )

    if should_fail_after_write:
        raise typer.Exit(code=1)


def main() -> None:
    app()


if __name__ == "__main__":
    main()