tonio/job-research
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: dedupe Apec listings by source job id

Browse Source
This commit is contained in:
Antoine 2026-06-01 13:13:24 +02:00
parent e86257cbcc
commit 96e2a7b178
2 changed files with 24 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/job_research/apec/dedupe.py
View File

@ -2,14 +2,16 @@ from job_research.models import ApecListing
def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]: def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]:
seen_urls: set[str] = set() seen_keys: set[str] = set()
deduped: list[ApecListing] = [] deduped: list[ApecListing] = []
for listing in listings: for listing in listings:
if listing.url in seen_urls: key = listing.source_job_id or listing.url
if key in seen_keys:
continue continue
seen_urls.add(listing.url) seen_keys.add(key)
deduped.append(listing) deduped.append(listing)
return deduped return deduped

19
tests/apec/test_dedupe.py
View File

@ -17,3 +17,22 @@ def test_dedupe_apec_listings_by_url_preserves_first_listing() -> None:
deduped = dedupe_apec_listings([first, second]) deduped = dedupe_apec_listings([first, second])
assert deduped == [first] assert deduped == [first]
def test_dedupe_apec_listings_by_source_job_id_ignores_url_changes() -> None:
first = ApecListing(
source="apec",
source_job_id="job-123",
url="https://example.test/job/1",
fetched_at="2026-06-01T10:00:00Z",
)
second = ApecListing(
source="apec",
source_job_id="job-123",
url="https://example.test/job/2",
fetched_at="2026-06-01T10:01:00Z",
)
deduped = dedupe_apec_listings([first, second])
assert deduped == [first]