diff --git a/src/job_research/apec/dedupe.py b/src/job_research/apec/dedupe.py index 4bbd3db..96cfb3d 100644 --- a/src/job_research/apec/dedupe.py +++ b/src/job_research/apec/dedupe.py @@ -8,12 +8,16 @@ def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]: for listing in listings: source_job_id = listing.source_job_id - if listing.url in seen_urls or (source_job_id and source_job_id in seen_source_job_ids): - continue + is_duplicate = listing.url in seen_urls or ( + source_job_id is not None and source_job_id in seen_source_job_ids + ) seen_urls.add(listing.url) - if source_job_id: + if source_job_id is not None: seen_source_job_ids.add(source_job_id) + + if is_duplicate: + continue deduped.append(listing) return deduped diff --git a/tests/apec/test_dedupe.py b/tests/apec/test_dedupe.py index efc43c2..5376893 100644 --- a/tests/apec/test_dedupe.py +++ b/tests/apec/test_dedupe.py @@ -55,3 +55,28 @@ def test_dedupe_apec_listings_collapses_mixed_key_duplicates() -> None: deduped = dedupe_apec_listings([first, second]) assert deduped == [first] + + +def test_dedupe_apec_listings_keeps_secondary_ids_from_skipped_rows() -> None: + first = ApecListing( + source="apec", + source_job_id=None, + url="url1", + fetched_at="2026-06-01T10:00:00Z", + ) + second = ApecListing( + source="apec", + source_job_id="job-123", + url="url1", + fetched_at="2026-06-01T10:01:00Z", + ) + third = ApecListing( + source="apec", + source_job_id="job-123", + url="url2", + fetched_at="2026-06-01T10:02:00Z", + ) + + deduped = dedupe_apec_listings([first, second, third]) + + assert deduped == [first]