fix: dedupe Apec listings by source job id
This commit is contained in:
parent
e86257cbcc
commit
96e2a7b178
@ -2,14 +2,16 @@ from job_research.models import ApecListing
|
||||
|
||||
|
||||
def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]:
|
||||
seen_urls: set[str] = set()
|
||||
seen_keys: set[str] = set()
|
||||
deduped: list[ApecListing] = []
|
||||
|
||||
for listing in listings:
|
||||
if listing.url in seen_urls:
|
||||
key = listing.source_job_id or listing.url
|
||||
|
||||
if key in seen_keys:
|
||||
continue
|
||||
|
||||
seen_urls.add(listing.url)
|
||||
seen_keys.add(key)
|
||||
deduped.append(listing)
|
||||
|
||||
return deduped
|
||||
|
||||
@ -17,3 +17,22 @@ def test_dedupe_apec_listings_by_url_preserves_first_listing() -> None:
|
||||
deduped = dedupe_apec_listings([first, second])
|
||||
|
||||
assert deduped == [first]
|
||||
|
||||
|
||||
def test_dedupe_apec_listings_by_source_job_id_ignores_url_changes() -> None:
|
||||
first = ApecListing(
|
||||
source="apec",
|
||||
source_job_id="job-123",
|
||||
url="https://example.test/job/1",
|
||||
fetched_at="2026-06-01T10:00:00Z",
|
||||
)
|
||||
second = ApecListing(
|
||||
source="apec",
|
||||
source_job_id="job-123",
|
||||
url="https://example.test/job/2",
|
||||
fetched_at="2026-06-01T10:01:00Z",
|
||||
)
|
||||
|
||||
deduped = dedupe_apec_listings([first, second])
|
||||
|
||||
assert deduped == [first]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user