From 00f37179959443c2907752f9b42d9afd45c15a2d Mon Sep 17 00:00:00 2001 From: Antoine Date: Mon, 1 Jun 2026 13:22:53 +0200 Subject: [PATCH] fix: dedupe mixed-key Apec listings --- src/job_research/apec/dedupe.py | 12 +++++++----- tests/apec/test_dedupe.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/job_research/apec/dedupe.py b/src/job_research/apec/dedupe.py index f07adf5..4bbd3db 100644 --- a/src/job_research/apec/dedupe.py +++ b/src/job_research/apec/dedupe.py @@ -2,16 +2,18 @@ from job_research.models import ApecListing def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]: - seen_keys: set[str] = set() + seen_urls: set[str] = set() + seen_source_job_ids: set[str] = set() deduped: list[ApecListing] = [] for listing in listings: - key = listing.source_job_id or listing.url - - if key in seen_keys: + source_job_id = listing.source_job_id + if listing.url in seen_urls or (source_job_id and source_job_id in seen_source_job_ids): continue - seen_keys.add(key) + seen_urls.add(listing.url) + if source_job_id: + seen_source_job_ids.add(source_job_id) deduped.append(listing) return deduped diff --git a/tests/apec/test_dedupe.py b/tests/apec/test_dedupe.py index 561d0a5..efc43c2 100644 --- a/tests/apec/test_dedupe.py +++ b/tests/apec/test_dedupe.py @@ -36,3 +36,22 @@ def test_dedupe_apec_listings_by_source_job_id_ignores_url_changes() -> None: deduped = dedupe_apec_listings([first, second]) assert deduped == [first] + + +def test_dedupe_apec_listings_collapses_mixed_key_duplicates() -> None: + first = ApecListing( + source="apec", + source_job_id="job-123", + url="https://example.test/job/1", + fetched_at="2026-06-01T10:00:00Z", + ) + second = ApecListing( + source="apec", + source_job_id=None, + url="https://example.test/job/1", + fetched_at="2026-06-01T10:01:00Z", + ) + + deduped = dedupe_apec_listings([first, second]) + + assert deduped == [first]