fix: dedupe mixed-key Apec listings

This commit is contained in:
Antoine 2026-06-01 13:22:53 +02:00
parent 96e2a7b178
commit 00f3717995
2 changed files with 26 additions and 5 deletions

View File

@ -2,16 +2,18 @@ from job_research.models import ApecListing
def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]:
seen_keys: set[str] = set()
seen_urls: set[str] = set()
seen_source_job_ids: set[str] = set()
deduped: list[ApecListing] = []
for listing in listings:
key = listing.source_job_id or listing.url
if key in seen_keys:
source_job_id = listing.source_job_id
if listing.url in seen_urls or (source_job_id and source_job_id in seen_source_job_ids):
continue
seen_keys.add(key)
seen_urls.add(listing.url)
if source_job_id:
seen_source_job_ids.add(source_job_id)
deduped.append(listing)
return deduped

View File

@ -36,3 +36,22 @@ def test_dedupe_apec_listings_by_source_job_id_ignores_url_changes() -> None:
deduped = dedupe_apec_listings([first, second])
assert deduped == [first]
def test_dedupe_apec_listings_collapses_mixed_key_duplicates() -> None:
first = ApecListing(
source="apec",
source_job_id="job-123",
url="https://example.test/job/1",
fetched_at="2026-06-01T10:00:00Z",
)
second = ApecListing(
source="apec",
source_job_id=None,
url="https://example.test/job/1",
fetched_at="2026-06-01T10:01:00Z",
)
deduped = dedupe_apec_listings([first, second])
assert deduped == [first]