tonio/job-research
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Apec normalization and dedupe

Browse Source
This commit is contained in:
Antoine 2026-06-01 13:06:37 +02:00
parent ac78bee74b
commit e86257cbcc
4 changed files with 88 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/job_research/apec/dedupe.py Normal file
View File

@ -0,0 +1,15 @@
from job_research.models import ApecListing
def dedupe_apec_listings(listings: list[ApecListing]) -> list[ApecListing]:
seen_urls: set[str] = set()
deduped: list[ApecListing] = []
for listing in listings:
if listing.url in seen_urls:
continue
seen_urls.add(listing.url)
deduped.append(listing)
return deduped

24
src/job_research/apec/normalize.py Normal file
View File

@ -0,0 +1,24 @@
from bs4 import BeautifulSoup
from job_research.models import ApecListing
def normalize_apec_listing(url: str, html: str, fetched_at: str) -> ApecListing:
soup = BeautifulSoup(html, "html.parser")
title = soup.find("h1")
company = soup.select_one(".company")
location = soup.select_one(".location")
contract = soup.select_one(".contract")
description = soup.select_one(".description")
return ApecListing(
source="apec",
url=url,
title=title.get_text(strip=True) if title else None,
company=company.get_text(strip=True) if company else None,
location=location.get_text(strip=True) if location else None,
contract_type=contract.get_text(strip=True) if contract else None,
description_text=description.get_text(" ", strip=True) if description else None,
fetched_at=fetched_at,
)

19
tests/apec/test_dedupe.py Normal file
View File

@ -0,0 +1,19 @@
from job_research.apec.dedupe import dedupe_apec_listings
from job_research.models import ApecListing
def test_dedupe_apec_listings_by_url_preserves_first_listing() -> None:
first = ApecListing(
source="apec",
url="https://example.test/job/1",
fetched_at="2026-06-01T10:00:00Z",
)
second = ApecListing(
source="apec",
url="https://example.test/job/1",
fetched_at="2026-06-01T10:01:00Z",
)
deduped = dedupe_apec_listings([first, second])
assert deduped == [first]

30
tests/apec/test_normalize.py Normal file
View File

@ -0,0 +1,30 @@
from job_research.apec.normalize import normalize_apec_listing
def test_normalize_apec_listing_extracts_minimal_shape() -> None:
html = """
<html>
<body>
<h1>Data Engineer</h1>
<div class="company">Example Corp</div>
<div class="location">Paris</div>
<div class="contract">CDI</div>
<div class="description">Build pipelines</div>
</body>
</html>
"""
listing = normalize_apec_listing(
url="https://example.test/job/123",
html=html,
fetched_at="2026-06-01T10:00:00Z",
)
assert listing.source == "apec"
assert listing.url == "https://example.test/job/123"
assert listing.title == "Data Engineer"
assert listing.company == "Example Corp"
assert listing.location == "Paris"
assert listing.contract_type == "CDI"
assert listing.description_text == "Build pipelines"
assert listing.fetched_at == "2026-06-01T10:00:00Z"