diff --git a/src/job_research/apec/dedupe.py b/src/job_research/apec/dedupe.py index e66bad6..412b6eb 100644 --- a/src/job_research/apec/dedupe.py +++ b/src/job_research/apec/dedupe.py @@ -36,6 +36,18 @@ def _register_listing( source_job_id_to_listing[listing.source_job_id] = survivor +def _repoint_listing_aliases( + url_to_listing: dict[str, ApecListing], + source_job_id_to_listing: dict[str, ApecListing], + removed: ApecListing, + survivor: ApecListing, +) -> None: + for mapping in (url_to_listing, source_job_id_to_listing): + for alias, listing in list(mapping.items()): + if listing is removed: + mapping[alias] = survivor + + def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]: url_to_listing: dict[str, ApecListing] = {} source_job_id_to_listing: dict[str, ApecListing] = {} @@ -66,7 +78,7 @@ def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]: _merge_listing_metadata(survivor, other) deduped[:] = [item for item in deduped if item is not other] - _register_listing(url_to_listing, source_job_id_to_listing, other, survivor) + _repoint_listing_aliases(url_to_listing, source_job_id_to_listing, other, survivor) _merge_listing_metadata(survivor, listing) _register_listing(url_to_listing, source_job_id_to_listing, listing, survivor) diff --git a/tests/apec/test_dedupe.py b/tests/apec/test_dedupe.py index 39638fe..2e9c34f 100644 --- a/tests/apec/test_dedupe.py +++ b/tests/apec/test_dedupe.py @@ -104,3 +104,77 @@ def test_dedupe_apec_listings_merges_metadata_from_duplicate_rows() -> None: assert deduped[0].url == "url1" assert deduped[0].source_job_id == "job-123" assert deduped[0].published_at == "2026-06-01" + + +def test_dedupe_apec_listings_merges_metadata_through_alias_chain() -> None: + first = ApecListing( + source="apec", + source_job_id=None, + url="u1", + fetched_at="2026-06-01T10:00:00Z", + ) + second = ApecListing( + source="apec", + source_job_id="i2", + url="u2", + fetched_at="2026-06-01T10:01:00Z", + ) + third = ApecListing( + source="apec", + source_job_id="i4", + url="u2", + fetched_at="2026-06-01T10:02:00Z", + ) + fourth = ApecListing( + source="apec", + source_job_id="i2", + url="u1", + fetched_at="2026-06-01T10:03:00Z", + ) + fifth = ApecListing( + source="apec", + source_job_id="i4", + url="u6", + company="NewestCo", + fetched_at="2026-06-01T10:04:00Z", + ) + + deduped = dedupe_apec_listings([first, second, third, fourth, fifth]) + + assert len(deduped) == 1 + assert deduped[0].url == "u1" + assert deduped[0].source_job_id == "i2" + assert deduped[0].company == "NewestCo" + + +def test_dedupe_apec_listings_keeps_one_survivor_for_cluster_alias_chain() -> None: + first = ApecListing( + source="apec", + source_job_id="id2", + url="u2", + fetched_at="2026-06-01T10:00:00Z", + ) + second = ApecListing( + source="apec", + source_job_id="id3", + url="u2", + fetched_at="2026-06-01T10:01:00Z", + ) + third = ApecListing( + source="apec", + source_job_id="id2", + url="u1", + fetched_at="2026-06-01T10:02:00Z", + ) + fourth = ApecListing( + source="apec", + source_job_id="id3", + url="u3", + fetched_at="2026-06-01T10:03:00Z", + ) + + deduped = dedupe_apec_listings([first, second, third, fourth]) + + assert len(deduped) == 1 + assert deduped[0].url == "u2" + assert deduped[0].source_job_id == "id2"