from job_research.apec.dedupe import dedupe_apec_listings from job_research.models import ApecListing def test_dedupe_apec_listings_by_url_preserves_first_listing() -> None: first = ApecListing( source="apec", url="https://example.test/job/1", fetched_at="2026-06-01T10:00:00Z", ) second = ApecListing( source="apec", url="https://example.test/job/1", fetched_at="2026-06-01T10:01:00Z", ) deduped = dedupe_apec_listings([first, second]) assert deduped == [first] def test_dedupe_apec_listings_by_source_job_id_ignores_url_changes() -> None: first = ApecListing( source="apec", source_job_id="job-123", url="https://example.test/job/1", fetched_at="2026-06-01T10:00:00Z", ) second = ApecListing( source="apec", source_job_id="job-123", url="https://example.test/job/2", fetched_at="2026-06-01T10:01:00Z", ) deduped = dedupe_apec_listings([first, second]) assert deduped == [first] def test_dedupe_apec_listings_collapses_mixed_key_duplicates() -> None: first = ApecListing( source="apec", source_job_id="job-123", url="https://example.test/job/1", fetched_at="2026-06-01T10:00:00Z", ) second = ApecListing( source="apec", source_job_id=None, url="https://example.test/job/1", fetched_at="2026-06-01T10:01:00Z", ) deduped = dedupe_apec_listings([first, second]) assert deduped == [first] def test_dedupe_apec_listings_keeps_secondary_ids_from_skipped_rows() -> None: first = ApecListing( source="apec", source_job_id=None, url="url1", fetched_at="2026-06-01T10:00:00Z", ) second = ApecListing( source="apec", source_job_id="job-123", url="url1", fetched_at="2026-06-01T10:01:00Z", ) third = ApecListing( source="apec", source_job_id="job-123", url="url2", fetched_at="2026-06-01T10:02:00Z", ) deduped = dedupe_apec_listings([first, second, third]) assert deduped == [first] def test_dedupe_apec_listings_merges_metadata_from_duplicate_rows() -> None: first = ApecListing( source="apec", source_job_id=None, published_at=None, refreshed_at=None, url="url1", fetched_at="2026-06-01T10:00:00Z", ) second = ApecListing( source="apec", source_job_id="job-123", published_at="2026-06-01", refreshed_at="2026-06-02", url="url1", fetched_at="2026-06-01T10:01:00Z", ) deduped = dedupe_apec_listings([first, second]) assert len(deduped) == 1 assert deduped[0].url == "url1" assert deduped[0].source_job_id == "job-123" assert deduped[0].published_at == "2026-06-01" assert deduped[0].refreshed_at == "2026-06-02" def test_dedupe_apec_listings_merges_metadata_through_alias_chain() -> None: first = ApecListing( source="apec", source_job_id=None, url="u1", fetched_at="2026-06-01T10:00:00Z", ) second = ApecListing( source="apec", source_job_id="i2", url="u2", fetched_at="2026-06-01T10:01:00Z", ) third = ApecListing( source="apec", source_job_id="i4", url="u2", fetched_at="2026-06-01T10:02:00Z", ) fourth = ApecListing( source="apec", source_job_id="i2", url="u1", fetched_at="2026-06-01T10:03:00Z", ) fifth = ApecListing( source="apec", source_job_id="i4", url="u6", company="NewestCo", fetched_at="2026-06-01T10:04:00Z", ) deduped = dedupe_apec_listings([first, second, third, fourth, fifth]) assert len(deduped) == 1 assert deduped[0].url == "u1" assert deduped[0].source_job_id == "i2" assert deduped[0].company == "NewestCo" def test_dedupe_apec_listings_keeps_one_survivor_for_cluster_alias_chain() -> None: first = ApecListing( source="apec", source_job_id="id2", url="u2", fetched_at="2026-06-01T10:00:00Z", ) second = ApecListing( source="apec", source_job_id="id3", url="u2", fetched_at="2026-06-01T10:01:00Z", ) third = ApecListing( source="apec", source_job_id="id2", url="u1", fetched_at="2026-06-01T10:02:00Z", ) fourth = ApecListing( source="apec", source_job_id="id3", url="u3", fetched_at="2026-06-01T10:03:00Z", ) deduped = dedupe_apec_listings([first, second, third, fourth]) assert len(deduped) == 1 assert deduped[0].url == "u2" assert deduped[0].source_job_id == "id2" def test_dedupe_apec_listings_keeps_first_listing_as_bridge_survivor() -> None: first = ApecListing( source="apec", source_job_id="id1", url="u1", fetched_at="2026-06-01T10:00:00Z", ) second = ApecListing( source="apec", source_job_id="id2", url="u2", fetched_at="2026-06-01T10:01:00Z", ) third = ApecListing( source="apec", source_job_id="id1", url="u2", company="NewestCo", fetched_at="2026-06-01T10:02:00Z", ) deduped = dedupe_apec_listings([first, second, third]) assert len(deduped) == 1 assert deduped[0].url == "u1" assert deduped[0].source_job_id == "id1" assert deduped[0].company == "NewestCo"