diff --git a/src/job_research/apec/dedupe.py b/src/job_research/apec/dedupe.py index 96cfb3d..e66bad6 100644 --- a/src/job_research/apec/dedupe.py +++ b/src/job_research/apec/dedupe.py @@ -1,23 +1,74 @@ from job_research.models import ApecListing +_MERGEABLE_FIELDS = ( + "source_job_id", + "title", + "company", + "location", + "contract_type", + "description_text", + "published_at", +) + + +def _merge_listing_metadata(survivor: ApecListing, source: ApecListing) -> None: + for field_name in _MERGEABLE_FIELDS: + if getattr(survivor, field_name) is None: + value = getattr(source, field_name) + if value is not None: + setattr(survivor, field_name, value) + + for warning in source.warnings: + if warning not in survivor.warnings: + survivor.warnings.append(warning) + + +def _register_listing( + url_to_listing: dict[str, ApecListing], + source_job_id_to_listing: dict[str, ApecListing], + listing: ApecListing, + survivor: ApecListing, +) -> None: + url_to_listing[listing.url] = survivor + + if listing.source_job_id is not None: + source_job_id_to_listing[listing.source_job_id] = survivor + + def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]: - seen_urls: set[str] = set() - seen_source_job_ids: set[str] = set() + url_to_listing: dict[str, ApecListing] = {} + source_job_id_to_listing: dict[str, ApecListing] = {} deduped: list[ApecListing] = [] for listing in listings: source_job_id = listing.source_job_id - is_duplicate = listing.url in seen_urls or ( - source_job_id is not None and source_job_id in seen_source_job_ids - ) + matches: list[ApecListing] = [] + + url_match = url_to_listing.get(listing.url) + if url_match is not None: + matches.append(url_match) - seen_urls.add(listing.url) if source_job_id is not None: - seen_source_job_ids.add(source_job_id) + source_job_id_match = source_job_id_to_listing.get(source_job_id) + if source_job_id_match is not None and source_job_id_match not in matches: + matches.append(source_job_id_match) - if is_duplicate: + if not matches: + deduped.append(listing) + _register_listing(url_to_listing, source_job_id_to_listing, listing, listing) continue - deduped.append(listing) + + survivor = matches[0] + for other in matches[1:]: + if other is survivor: + continue + + _merge_listing_metadata(survivor, other) + deduped[:] = [item for item in deduped if item is not other] + _register_listing(url_to_listing, source_job_id_to_listing, other, survivor) + + _merge_listing_metadata(survivor, listing) + _register_listing(url_to_listing, source_job_id_to_listing, listing, survivor) return deduped diff --git a/tests/apec/test_dedupe.py b/tests/apec/test_dedupe.py index 5376893..39638fe 100644 --- a/tests/apec/test_dedupe.py +++ b/tests/apec/test_dedupe.py @@ -80,3 +80,27 @@ def test_dedupe_apec_listings_keeps_secondary_ids_from_skipped_rows() -> None: deduped = dedupe_apec_listings([first, second, third]) assert deduped == [first] + + +def test_dedupe_apec_listings_merges_metadata_from_duplicate_rows() -> None: + first = ApecListing( + source="apec", + source_job_id=None, + published_at=None, + url="url1", + fetched_at="2026-06-01T10:00:00Z", + ) + second = ApecListing( + source="apec", + source_job_id="job-123", + published_at="2026-06-01", + url="url1", + fetched_at="2026-06-01T10:01:00Z", + ) + + deduped = dedupe_apec_listings([first, second]) + + assert len(deduped) == 1 + assert deduped[0].url == "url1" + assert deduped[0].source_job_id == "job-123" + assert deduped[0].published_at == "2026-06-01"