From ed1af201bc709778b85223949a20f6cd1f2a05e9 Mon Sep 17 00:00:00 2001 From: Antoine Date: Fri, 5 Jun 2026 13:37:22 +0200 Subject: [PATCH] fix: harden profile and Apec normalization warnings --- src/job_research/apec/normalize.py | 54 ++++++++++++++-------- src/job_research/profile/cv_extractor.py | 32 +++++++++++++ src/job_research/profile/merge.py | 10 ++++ src/job_research/profile/profile_parser.py | 19 +++++++- tests/apec/test_normalize.py | 41 ++++++++++++++++ tests/profile/test_cv_extractor.py | 17 +++++++ tests/profile/test_merge.py | 32 +++++++++++++ tests/profile/test_profile_parser.py | 35 ++++++++++++++ 8 files changed, 220 insertions(+), 20 deletions(-) diff --git a/src/job_research/apec/normalize.py b/src/job_research/apec/normalize.py index 78ac275..b5e6884 100644 --- a/src/job_research/apec/normalize.py +++ b/src/job_research/apec/normalize.py @@ -103,29 +103,34 @@ def _warning(field: str, message: str) -> ListingWarning: return ListingWarning(field=field, message=message) -def _extract_published_at(soup: BeautifulSoup) -> str | None: +def _extract_listing_date( + soup: BeautifulSoup, + pattern: re.Pattern[str], + *, + field: str, + missing_message: str | None = None, + invalid_message: str, + warnings: list[ListingWarning], + warn_on_missing: bool, +) -> str | None: card_offer = soup.select_one(".card-offer") if card_offer is None: + if warn_on_missing and missing_message is not None: + warnings.append(_warning(field, missing_message)) return None - match = _PUBLISHED_AT_PATTERN.search(card_offer.get_text(" ", strip=True)) + match = pattern.search(card_offer.get_text(" ", strip=True)) if match is None: + if warn_on_missing and missing_message is not None: + warnings.append(_warning(field, missing_message)) return None - return datetime.strptime(match.group(1), "%d/%m/%Y").date().isoformat() - - -def _extract_refreshed_at(soup: BeautifulSoup) -> str | None: - card_offer = soup.select_one(".card-offer") - if card_offer is None: + try: + return datetime.strptime(match.group(1), "%d/%m/%Y").date().isoformat() + except ValueError: + warnings.append(_warning(field, invalid_message)) return None - match = _REFRESHED_AT_PATTERN.search(card_offer.get_text(" ", strip=True)) - if match is None: - return None - - return datetime.strptime(match.group(1), "%d/%m/%Y").date().isoformat() - def _extract_source_job_id(soup: BeautifulSoup, source_job_id: str | None) -> str | None: if source_job_id is not None: @@ -272,11 +277,24 @@ def normalize_apec_listing( warnings.append(_warning("company", "Company is empty or placeholder text")) company_text = None - published_at_value = published_at or _extract_published_at(soup) - if published_at_value is None: - warnings.append(_warning("published_at", "Published date missing from Apec listing")) + published_at_value = published_at or _extract_listing_date( + soup, + _PUBLISHED_AT_PATTERN, + field="published_at", + missing_message="Published date missing from Apec listing", + invalid_message="Published date is invalid", + warnings=warnings, + warn_on_missing=True, + ) - refreshed_at_value = refreshed_at or _extract_refreshed_at(soup) + refreshed_at_value = refreshed_at or _extract_listing_date( + soup, + _REFRESHED_AT_PATTERN, + field="refreshed_at", + invalid_message="Refreshed date is invalid", + warnings=warnings, + warn_on_missing=False, + ) return ApecListing( source="apec", diff --git a/src/job_research/profile/cv_extractor.py b/src/job_research/profile/cv_extractor.py index 1ccf55b..55b5a22 100644 --- a/src/job_research/profile/cv_extractor.py +++ b/src/job_research/profile/cv_extractor.py @@ -5,6 +5,8 @@ from pathlib import Path from pypdf import PdfReader +from job_research.models import WarningItem + EXPERIENCE_LINE_CONNECTORS = (" at ", " chez ", " au ", " à ") @@ -150,6 +152,15 @@ YEARS_OF_EXPERIENCE_PATTERNS = ( re.compile(r"^ann[ée]es d['’]exp[ée]rience\s*:\s*(\d+)\s*$", re.IGNORECASE), ) +LOW_CONFIDENCE_NAME_PATTERNS = ( + re.compile(r"\b(cv|resume|curriculum vitae|profile)\b", re.IGNORECASE), + re.compile(r"[|/@]"), + re.compile( + r"\b(data engineer|software engineer|developer|analyst|scientist|consultant|architect|manager|product owner|backend|frontend|full stack)\b", + re.IGNORECASE, + ), +) + def extract_pdf_text(path: Path) -> str: reader = PdfReader(str(path)) @@ -168,8 +179,17 @@ def extract_pdf_text(path: Path) -> str: def extract_cv_signals(text: str) -> dict[str, object]: lines = [_normalize_line(line) for line in text.splitlines()] non_empty_lines = [line for line in lines if line] + warnings: list[WarningItem] = [] name = non_empty_lines[0] if non_empty_lines else None + if name is not None and _looks_like_low_confidence_name(name): + warnings.append( + WarningItem( + field="name", + message="First CV line looks like a header or tagline; review manually.", + ) + ) + location = None languages: list[str] = [] skills: list[str] = [] @@ -247,6 +267,7 @@ def extract_cv_signals(text: str) -> dict[str, object]: "skills": skills, "experience_entries": experience_entries, "education_entries": education_entries, + "warnings": warnings, } if years_of_experience is not None: @@ -369,3 +390,14 @@ def _looks_like_experience_title(title: str) -> bool: def _looks_like_prose_company(company: str) -> bool: return any(pattern.search(company) for pattern in EXPERIENCE_PROSE_COMPANY_PATTERNS) + + +def _looks_like_low_confidence_name(name: str) -> bool: + normalized = " ".join(name.split()) + if not normalized: + return True + + if len(normalized.split()) > 4: + return True + + return any(pattern.search(normalized) for pattern in LOW_CONFIDENCE_NAME_PATTERNS) diff --git a/src/job_research/profile/merge.py b/src/job_research/profile/merge.py index 263fbac..fa0795f 100644 --- a/src/job_research/profile/merge.py +++ b/src/job_research/profile/merge.py @@ -33,6 +33,7 @@ def build_candidate_profile_output( warnings: list[WarningItem] = [] _append_years_of_experience_warning(cv_signals, authored.notes, warnings) + _append_cv_extraction_warnings(cv_signals, warnings) _append_missing_cv_fact_warnings(cv_signals, warnings) merged_skills: list[str] = [] @@ -99,6 +100,15 @@ def _append_missing_cv_fact_warnings( warnings.append(WarningItem(field=field, message=message)) +def _append_cv_extraction_warnings( + cv_signals: dict[str, object], warnings: list[WarningItem] +) -> None: + for warning in cv_signals.get("warnings") or []: + warnings.append( + warning if isinstance(warning, WarningItem) else WarningItem.model_validate(warning) + ) + + def _note_years_of_experience(note: str) -> int | None: normalized = note.casefold().replace("’", "'") if not any(marker in normalized for marker in EXPERIENCE_NOTE_MARKERS): diff --git a/src/job_research/profile/profile_parser.py b/src/job_research/profile/profile_parser.py index 1e4ee35..8ab1843 100644 --- a/src/job_research/profile/profile_parser.py +++ b/src/job_research/profile/profile_parser.py @@ -17,14 +17,15 @@ class AuthoredProfile: notes: list[str] = field(default_factory=list) -REQUIRED_SECTIONS = { +REQUIRED_SECTION_NAMES = ( "summary", "target roles", "strengths", "skills to emphasize", "constraints", "notes", -} +) +REQUIRED_SECTIONS = set(REQUIRED_SECTION_NAMES) def parse_profile_markdown(markdown: str) -> AuthoredProfile: @@ -45,6 +46,10 @@ def parse_profile_markdown(markdown: str) -> AuthoredProfile: missing_text = ", ".join(sorted(missing)) raise ValueError(f"Missing required markdown sections: {missing_text}") + for section_name in REQUIRED_SECTION_NAMES: + if not _has_usable_section_content(sections[section_name]): + raise ValueError(f"Missing usable content in section '{section_name}'") + return AuthoredProfile( summary=" ".join(sections["summary"]), target_roles=_parse_list_section("target roles", sections["target roles"]), @@ -64,6 +69,8 @@ def _parse_list_section(section_name: str, lines: list[str]) -> list[str]: item = _strip_list_marker(line) if item is None: raise ValueError(f"Unsupported content in section '{section_name}': {line}") + if not item: + raise ValueError(f"Missing usable content in section '{section_name}'") items.append(item) return items @@ -74,6 +81,8 @@ def _parse_notes_section(lines: list[str]) -> list[str]: for line in lines: item = _strip_list_marker(line) + if item == "": + raise ValueError("Missing usable content in section 'notes'") notes.append(item if item is not None else line) return notes @@ -81,7 +90,13 @@ def _parse_notes_section(lines: list[str]) -> list[str]: def _strip_list_marker(line: str) -> str | None: for marker in LIST_MARKERS: + if line == marker.strip(): + return "" if line.startswith(marker): return line[len(marker):].strip() return None + + +def _has_usable_section_content(lines: list[str]) -> bool: + return any(line not in {"-", "*", "+"} for line in lines) diff --git a/tests/apec/test_normalize.py b/tests/apec/test_normalize.py index c474010..bb4eb16 100644 --- a/tests/apec/test_normalize.py +++ b/tests/apec/test_normalize.py @@ -71,6 +71,47 @@ def test_normalize_apec_listing_extracts_minimal_shape() -> None: assert listing.fetched_at == "2026-06-01T10:00:00Z" +def test_normalize_apec_listing_warns_and_returns_none_for_invalid_dates() -> None: + html = """ + + +
+

Data Engineer F/H

+
+
Ref. Apec : 178554452W
+
    +
  • CLOUD TEMPLE
    • +
  • 1 CDI
    • +
  • Puteaux - 92
    • +
+

Publiée le 32/13/2026 Actualisée le 31/02/2026

+
+
+
+ CLOUD TEMPLE +
+
+
+

Descriptif du poste

+

Build pipelines

+
+
+ + + """ + + listing = normalize_apec_listing( + url="https://example.test/job/123", + html=html, + fetched_at="2026-06-01T10:00:00Z", + source_job_id="job-123", + ) + + assert listing.published_at is None + assert listing.refreshed_at is None + assert [warning.field for warning in listing.warnings] == ["published_at", "refreshed_at"] + + def test_normalize_apec_listing_uses_details_offer_list_company_fallback() -> None: html = """ diff --git a/tests/profile/test_cv_extractor.py b/tests/profile/test_cv_extractor.py index 7a9bc44..4dd8577 100644 --- a/tests/profile/test_cv_extractor.py +++ b/tests/profile/test_cv_extractor.py @@ -29,6 +29,23 @@ def test_extract_cv_signals_reads_basic_fields_from_text() -> None: assert len(extracted["experience_entries"]) == 2 +def test_extract_cv_signals_flags_low_confidence_first_line_as_name() -> None: + text = dedent( + """ + Data Engineer | Python | GCP + Location: France + Languages: French, English + Skills: Python, SQL + Data Engineer at Company A + """ + ).strip() + + extracted = extract_cv_signals(text) + + assert extracted["name"] == "Data Engineer | Python | GCP" + assert [warning.field for warning in extracted["warnings"]] == ["name"] + + @pytest.mark.parametrize( ("line", "expected"), [ diff --git a/tests/profile/test_merge.py b/tests/profile/test_merge.py index 1aeecbc..c3e05b6 100644 --- a/tests/profile/test_merge.py +++ b/tests/profile/test_merge.py @@ -1,5 +1,6 @@ from job_research.profile.merge import build_candidate_profile_output from job_research.profile.profile_parser import AuthoredProfile +from job_research.models import WarningItem def test_build_candidate_profile_output_writes_warning_when_facts_conflict() -> None: @@ -72,3 +73,34 @@ def test_build_candidate_profile_output_warns_on_missing_core_cv_facts() -> None "skills", "education_entries", ] + + +def test_build_candidate_profile_output_propagates_cv_extraction_warnings() -> None: + cv_signals = { + "name": "Data Engineer | Python | GCP", + "location": "France", + "languages": ["French", "English"], + "skills": ["Python", "SQL"], + "experience_entries": [{"title": "Data Engineer", "company": "A"}], + "education_entries": [{"credential": "MSc", "institution": "Example University"}], + "warnings": [ + WarningItem( + field="name", + message="First CV line looks like a header or tagline; review manually.", + ) + ], + } + authored = AuthoredProfile( + summary="Junior data engineer focused on GCP.", + target_roles=["Data Engineer"], + strengths=["Python"], + skills_to_emphasize=["BigQuery", "GCP"], + constraints=["CDI only"], + notes=[], + ) + + output = build_candidate_profile_output(cv_signals, authored) + + assert output.warnings == [ + WarningItem(field="name", message="First CV line looks like a header or tagline; review manually.") + ] diff --git a/tests/profile/test_profile_parser.py b/tests/profile/test_profile_parser.py index 8df1c82..5f3d33a 100644 --- a/tests/profile/test_profile_parser.py +++ b/tests/profile/test_profile_parser.py @@ -72,3 +72,38 @@ def test_parse_profile_markdown_rejects_unsupported_list_content() -> None: with pytest.raises(ValueError, match="Unsupported content in section 'target roles'"): parse_profile_markdown(markdown) + + +@pytest.mark.parametrize("section_name", ["Target Roles", "Notes"]) +def test_parse_profile_markdown_rejects_blank_bullet_only_required_sections( + section_name: str, +) -> None: + target_roles = "- " if section_name == "Target Roles" else "- Data Engineer" + notes = "- " if section_name == "Notes" else "Slight preference for French listings." + + markdown = dedent( + f""" + # Candidate Profile + + ## Summary + Junior data engineer focused on Python and GCP. + + ## Target Roles + {target_roles} + + ## Strengths + - Python + + ## Skills To Emphasize + - BigQuery + + ## Constraints + - CDI only + + ## Notes + {notes} + """ + ).strip() + + with pytest.raises(ValueError, match=f"Missing usable content in section '{section_name.lower()}'"): + parse_profile_markdown(markdown)