job-research/tests/apec/test_dedupe.py

from job_research.apec.dedupe import dedupe_apec_listings
from job_research.models import ApecListing


def test_dedupe_apec_listings_by_url_preserves_first_listing() -> None:
    first = ApecListing(
        source="apec",
        url="https://example.test/job/1",
        fetched_at="2026-06-01T10:00:00Z",
    )
    second = ApecListing(
        source="apec",
        url="https://example.test/job/1",
        fetched_at="2026-06-01T10:01:00Z",
    )

    deduped = dedupe_apec_listings([first, second])

    assert deduped == [first]


def test_dedupe_apec_listings_by_source_job_id_ignores_url_changes() -> None:
    first = ApecListing(
        source="apec",
        source_job_id="job-123",
        url="https://example.test/job/1",
        fetched_at="2026-06-01T10:00:00Z",
    )
    second = ApecListing(
        source="apec",
        source_job_id="job-123",
        url="https://example.test/job/2",
        fetched_at="2026-06-01T10:01:00Z",
    )

    deduped = dedupe_apec_listings([first, second])

    assert deduped == [first]


def test_dedupe_apec_listings_collapses_mixed_key_duplicates() -> None:
    first = ApecListing(
        source="apec",
        source_job_id="job-123",
        url="https://example.test/job/1",
        fetched_at="2026-06-01T10:00:00Z",
    )
    second = ApecListing(
        source="apec",
        source_job_id=None,
        url="https://example.test/job/1",
        fetched_at="2026-06-01T10:01:00Z",
    )

    deduped = dedupe_apec_listings([first, second])

    assert deduped == [first]


def test_dedupe_apec_listings_keeps_secondary_ids_from_skipped_rows() -> None:
    first = ApecListing(
        source="apec",
        source_job_id=None,
        url="url1",
        fetched_at="2026-06-01T10:00:00Z",
    )
    second = ApecListing(
        source="apec",
        source_job_id="job-123",
        url="url1",
        fetched_at="2026-06-01T10:01:00Z",
    )
    third = ApecListing(
        source="apec",
        source_job_id="job-123",
        url="url2",
        fetched_at="2026-06-01T10:02:00Z",
    )

    deduped = dedupe_apec_listings([first, second, third])

    assert deduped == [first]


def test_dedupe_apec_listings_merges_metadata_from_duplicate_rows() -> None:
    first = ApecListing(
        source="apec",
        source_job_id=None,
        published_at=None,
        refreshed_at=None,
        url="url1",
        fetched_at="2026-06-01T10:00:00Z",
    )
    second = ApecListing(
        source="apec",
        source_job_id="job-123",
        published_at="2026-06-01",
        refreshed_at="2026-06-02",
        url="url1",
        fetched_at="2026-06-01T10:01:00Z",
    )

    deduped = dedupe_apec_listings([first, second])

    assert len(deduped) == 1
    assert deduped[0].url == "url1"
    assert deduped[0].source_job_id == "job-123"
    assert deduped[0].published_at == "2026-06-01"
    assert deduped[0].refreshed_at == "2026-06-02"


def test_dedupe_apec_listings_merges_metadata_through_alias_chain() -> None:
    first = ApecListing(
        source="apec",
        source_job_id=None,
        url="u1",
        fetched_at="2026-06-01T10:00:00Z",
    )
    second = ApecListing(
        source="apec",
        source_job_id="i2",
        url="u2",
        fetched_at="2026-06-01T10:01:00Z",
    )
    third = ApecListing(
        source="apec",
        source_job_id="i4",
        url="u2",
        fetched_at="2026-06-01T10:02:00Z",
    )
    fourth = ApecListing(
        source="apec",
        source_job_id="i2",
        url="u1",
        fetched_at="2026-06-01T10:03:00Z",
    )
    fifth = ApecListing(
        source="apec",
        source_job_id="i4",
        url="u6",
        company="NewestCo",
        fetched_at="2026-06-01T10:04:00Z",
    )

    deduped = dedupe_apec_listings([first, second, third, fourth, fifth])

    assert len(deduped) == 1
    assert deduped[0].url == "u1"
    assert deduped[0].source_job_id == "i2"
    assert deduped[0].company == "NewestCo"


def test_dedupe_apec_listings_keeps_one_survivor_for_cluster_alias_chain() -> None:
    first = ApecListing(
        source="apec",
        source_job_id="id2",
        url="u2",
        fetched_at="2026-06-01T10:00:00Z",
    )
    second = ApecListing(
        source="apec",
        source_job_id="id3",
        url="u2",
        fetched_at="2026-06-01T10:01:00Z",
    )
    third = ApecListing(
        source="apec",
        source_job_id="id2",
        url="u1",
        fetched_at="2026-06-01T10:02:00Z",
    )
    fourth = ApecListing(
        source="apec",
        source_job_id="id3",
        url="u3",
        fetched_at="2026-06-01T10:03:00Z",
    )

    deduped = dedupe_apec_listings([first, second, third, fourth])

    assert len(deduped) == 1
    assert deduped[0].url == "u2"
    assert deduped[0].source_job_id == "id2"


def test_dedupe_apec_listings_keeps_first_listing_as_bridge_survivor() -> None:
    first = ApecListing(
        source="apec",
        source_job_id="id1",
        url="u1",
        fetched_at="2026-06-01T10:00:00Z",
    )
    second = ApecListing(
        source="apec",
        source_job_id="id2",
        url="u2",
        fetched_at="2026-06-01T10:01:00Z",
    )
    third = ApecListing(
        source="apec",
        source_job_id="id1",
        url="u2",
        company="NewestCo",
        fetched_at="2026-06-01T10:02:00Z",
    )

    deduped = dedupe_apec_listings([first, second, third])

    assert len(deduped) == 1
    assert deduped[0].url == "u1"
    assert deduped[0].source_job_id == "id1"
    assert deduped[0].company == "NewestCo"