fix: preserve identifiers in Apec dedupe
This commit is contained in:
parent
95b17a7c50
commit
96a7dcbe94
@ -8,12 +8,16 @@ def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]:
|
|||||||
|
|
||||||
for listing in listings:
|
for listing in listings:
|
||||||
source_job_id = listing.source_job_id
|
source_job_id = listing.source_job_id
|
||||||
if listing.url in seen_urls or (source_job_id and source_job_id in seen_source_job_ids):
|
is_duplicate = listing.url in seen_urls or (
|
||||||
continue
|
source_job_id is not None and source_job_id in seen_source_job_ids
|
||||||
|
)
|
||||||
|
|
||||||
seen_urls.add(listing.url)
|
seen_urls.add(listing.url)
|
||||||
if source_job_id:
|
if source_job_id is not None:
|
||||||
seen_source_job_ids.add(source_job_id)
|
seen_source_job_ids.add(source_job_id)
|
||||||
|
|
||||||
|
if is_duplicate:
|
||||||
|
continue
|
||||||
deduped.append(listing)
|
deduped.append(listing)
|
||||||
|
|
||||||
return deduped
|
return deduped
|
||||||
|
|||||||
@ -55,3 +55,28 @@ def test_dedupe_apec_listings_collapses_mixed_key_duplicates() -> None:
|
|||||||
deduped = dedupe_apec_listings([first, second])
|
deduped = dedupe_apec_listings([first, second])
|
||||||
|
|
||||||
assert deduped == [first]
|
assert deduped == [first]
|
||||||
|
|
||||||
|
|
||||||
|
def test_dedupe_apec_listings_keeps_secondary_ids_from_skipped_rows() -> None:
|
||||||
|
first = ApecListing(
|
||||||
|
source="apec",
|
||||||
|
source_job_id=None,
|
||||||
|
url="url1",
|
||||||
|
fetched_at="2026-06-01T10:00:00Z",
|
||||||
|
)
|
||||||
|
second = ApecListing(
|
||||||
|
source="apec",
|
||||||
|
source_job_id="job-123",
|
||||||
|
url="url1",
|
||||||
|
fetched_at="2026-06-01T10:01:00Z",
|
||||||
|
)
|
||||||
|
third = ApecListing(
|
||||||
|
source="apec",
|
||||||
|
source_job_id="job-123",
|
||||||
|
url="url2",
|
||||||
|
fetched_at="2026-06-01T10:02:00Z",
|
||||||
|
)
|
||||||
|
|
||||||
|
deduped = dedupe_apec_listings([first, second, third])
|
||||||
|
|
||||||
|
assert deduped == [first]
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user