From 96e2a7b1780aee8038d9dff3836d35f73b649cdf Mon Sep 17 00:00:00 2001
From: Antoine <antoinevdp@live.fr>
Date: Mon, 1 Jun 2026 13:13:24 +0200
Subject: [PATCH] fix: dedupe Apec listings by source job id

---
 src/job_research/apec/dedupe.py |  8 +++++---
 tests/apec/test_dedupe.py       | 19 +++++++++++++++++++
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/src/job_research/apec/dedupe.py b/src/job_research/apec/dedupe.py
index 6db9c61..f07adf5 100644
--- a/src/job_research/apec/dedupe.py
+++ b/src/job_research/apec/dedupe.py
@@ -2,14 +2,16 @@ from job_research.models import ApecListing
 
 
 def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]:
-    seen_urls: set[str] = set()
+    seen_keys: set[str] = set()
     deduped: list[ApecListing] = []
 
     for listing in listings:
-        if listing.url in seen_urls:
+        key = listing.source_job_id or listing.url
+
+        if key in seen_keys:
             continue
 
-        seen_urls.add(listing.url)
+        seen_keys.add(key)
         deduped.append(listing)
 
     return deduped
diff --git a/tests/apec/test_dedupe.py b/tests/apec/test_dedupe.py
index 5b030e0..561d0a5 100644
--- a/tests/apec/test_dedupe.py
+++ b/tests/apec/test_dedupe.py
@@ -17,3 +17,22 @@ def test_dedupe_apec_listings_by_url_preserves_first_listing() -> None:
     deduped = dedupe_apec_listings([first, second])
 
     assert deduped == [first]
+
+
+def test_dedupe_apec_listings_by_source_job_id_ignores_url_changes() -> None:
+    first = ApecListing(
+        source="apec",
+        source_job_id="job-123",
+        url="https://example.test/job/1",
+        fetched_at="2026-06-01T10:00:00Z",
+    )
+    second = ApecListing(
+        source="apec",
+        source_job_id="job-123",
+        url="https://example.test/job/2",
+        fetched_at="2026-06-01T10:01:00Z",
+    )
+
+    deduped = dedupe_apec_listings([first, second])
+
+    assert deduped == [first]