fix: reassign Apec alias chains

This commit is contained in:
Antoine 2026-06-01 14:07:26 +02:00
parent 96b4cd37ec
commit debaab0947
2 changed files with 87 additions and 1 deletions

View File

@ -36,6 +36,18 @@ def _register_listing(
source_job_id_to_listing[listing.source_job_id] = survivor
def _repoint_listing_aliases(
url_to_listing: dict[str, ApecListing],
source_job_id_to_listing: dict[str, ApecListing],
removed: ApecListing,
survivor: ApecListing,
) -> None:
for mapping in (url_to_listing, source_job_id_to_listing):
for alias, listing in list(mapping.items()):
if listing is removed:
mapping[alias] = survivor
def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]:
url_to_listing: dict[str, ApecListing] = {}
source_job_id_to_listing: dict[str, ApecListing] = {}
@ -66,7 +78,7 @@ def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]:
_merge_listing_metadata(survivor, other)
deduped[:] = [item for item in deduped if item is not other]
_register_listing(url_to_listing, source_job_id_to_listing, other, survivor)
_repoint_listing_aliases(url_to_listing, source_job_id_to_listing, other, survivor)
_merge_listing_metadata(survivor, listing)
_register_listing(url_to_listing, source_job_id_to_listing, listing, survivor)

View File

@ -104,3 +104,77 @@ def test_dedupe_apec_listings_merges_metadata_from_duplicate_rows() -> None:
assert deduped[0].url == "url1"
assert deduped[0].source_job_id == "job-123"
assert deduped[0].published_at == "2026-06-01"
def test_dedupe_apec_listings_merges_metadata_through_alias_chain() -> None:
first = ApecListing(
source="apec",
source_job_id=None,
url="u1",
fetched_at="2026-06-01T10:00:00Z",
)
second = ApecListing(
source="apec",
source_job_id="i2",
url="u2",
fetched_at="2026-06-01T10:01:00Z",
)
third = ApecListing(
source="apec",
source_job_id="i4",
url="u2",
fetched_at="2026-06-01T10:02:00Z",
)
fourth = ApecListing(
source="apec",
source_job_id="i2",
url="u1",
fetched_at="2026-06-01T10:03:00Z",
)
fifth = ApecListing(
source="apec",
source_job_id="i4",
url="u6",
company="NewestCo",
fetched_at="2026-06-01T10:04:00Z",
)
deduped = dedupe_apec_listings([first, second, third, fourth, fifth])
assert len(deduped) == 1
assert deduped[0].url == "u1"
assert deduped[0].source_job_id == "i2"
assert deduped[0].company == "NewestCo"
def test_dedupe_apec_listings_keeps_one_survivor_for_cluster_alias_chain() -> None:
first = ApecListing(
source="apec",
source_job_id="id2",
url="u2",
fetched_at="2026-06-01T10:00:00Z",
)
second = ApecListing(
source="apec",
source_job_id="id3",
url="u2",
fetched_at="2026-06-01T10:01:00Z",
)
third = ApecListing(
source="apec",
source_job_id="id2",
url="u1",
fetched_at="2026-06-01T10:02:00Z",
)
fourth = ApecListing(
source="apec",
source_job_id="id3",
url="u3",
fetched_at="2026-06-01T10:03:00Z",
)
deduped = dedupe_apec_listings([first, second, third, fourth])
assert len(deduped) == 1
assert deduped[0].url == "u2"
assert deduped[0].source_job_id == "id2"