feat: add Apec normalization and dedupe

2026-06-01 13:06:37 +02:00 · 2026-06-01 13:06:37 +02:00 · e86257cbcc
commit e86257cbcc
parent ac78bee74b
4 changed files with 88 additions and 0 deletions
--- a/src/job_research/apec/dedupe.py
+++ b/src/job_research/apec/dedupe.py
@ -0,0 +1,15 @@
+from job_research.models import ApecListing
+
+
+def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]:
+    seen_urls: set[str] = set()
+    deduped: list[ApecListing] = []
+
+    for listing in listings:
+        if listing.url in seen_urls:
+            continue
+
+        seen_urls.add(listing.url)
+        deduped.append(listing)
+
+    return deduped
--- a/src/job_research/apec/normalize.py
+++ b/src/job_research/apec/normalize.py
@ -0,0 +1,24 @@
+from bs4 import BeautifulSoup
+
+from job_research.models import ApecListing
+
+
+def normalize_apec_listing(url: str, html: str, fetched_at: str) -> ApecListing:
+    soup = BeautifulSoup(html, "html.parser")
+
+    title = soup.find("h1")
+    company = soup.select_one(".company")
+    location = soup.select_one(".location")
+    contract = soup.select_one(".contract")
+    description = soup.select_one(".description")
+
+    return ApecListing(
+        source="apec",
+        url=url,
+        title=title.get_text(strip=True) if title else None,
+        company=company.get_text(strip=True) if company else None,
+        location=location.get_text(strip=True) if location else None,
+        contract_type=contract.get_text(strip=True) if contract else None,
+        description_text=description.get_text(" ", strip=True) if description else None,
+        fetched_at=fetched_at,
+    )
--- a/tests/apec/test_dedupe.py
+++ b/tests/apec/test_dedupe.py
@ -0,0 +1,19 @@
+from job_research.apec.dedupe import dedupe_apec_listings
+from job_research.models import ApecListing
+
+
+def test_dedupe_apec_listings_by_url_preserves_first_listing() -> None:
+    first = ApecListing(
+        source="apec",
+        url="https://example.test/job/1",
+        fetched_at="2026-06-01T10:00:00Z",
+    )
+    second = ApecListing(
+        source="apec",
+        url="https://example.test/job/1",
+        fetched_at="2026-06-01T10:01:00Z",
+    )
+
+    deduped = dedupe_apec_listings([first, second])
+
+    assert deduped == [first]
--- a/tests/apec/test_normalize.py
+++ b/tests/apec/test_normalize.py
@ -0,0 +1,30 @@
+from job_research.apec.normalize import normalize_apec_listing
+
+
+def test_normalize_apec_listing_extracts_minimal_shape() -> None:
+    html = """
+    <html>
+      <body>
+        <h1>Data Engineer</h1>
+        <div class="company">Example Corp</div>
+        <div class="location">Paris</div>
+        <div class="contract">CDI</div>
+        <div class="description">Build pipelines</div>
+      </body>
+    </html>
+    """
+
+    listing = normalize_apec_listing(
+        url="https://example.test/job/123",
+        html=html,
+        fetched_at="2026-06-01T10:00:00Z",
+    )
+
+    assert listing.source == "apec"
+    assert listing.url == "https://example.test/job/123"
+    assert listing.title == "Data Engineer"
+    assert listing.company == "Example Corp"
+    assert listing.location == "Paris"
+    assert listing.contract_type == "CDI"
+    assert listing.description_text == "Build pipelines"
+    assert listing.fetched_at == "2026-06-01T10:00:00Z"