fix: broaden cv parsing for french labels
Recognize French section labels, allow clear title connector company lines across French and English forms, and keep prose fragments out of experience parsing.
This commit is contained in:
parent
77d2f0f5c6
commit
19a1731349
@ -6,28 +6,34 @@ from pathlib import Path
|
||||
from pypdf import PdfReader
|
||||
|
||||
|
||||
EXPERIENCE_TITLE_ENDINGS = {
|
||||
"administrator",
|
||||
"analyst",
|
||||
"architect",
|
||||
"consultant",
|
||||
"développeur",
|
||||
"developpeur",
|
||||
"data engineer",
|
||||
"data scientist",
|
||||
"developer",
|
||||
"director",
|
||||
"ingénieur",
|
||||
"ingenieur",
|
||||
"engineer",
|
||||
"lead",
|
||||
"manager",
|
||||
"researcher",
|
||||
"scientist",
|
||||
"specialist",
|
||||
}
|
||||
EXPERIENCE_LINE_CONNECTORS = (" at ", " chez ", " au ", " à ")
|
||||
|
||||
EXPERIENCE_LINE_CONNECTORS = (" at ", " chez ", " au ")
|
||||
EXPERIENCE_TITLE_STOPWORDS = {
|
||||
"a",
|
||||
"an",
|
||||
"and",
|
||||
"as",
|
||||
"at",
|
||||
"after",
|
||||
"before",
|
||||
"by",
|
||||
"for",
|
||||
"from",
|
||||
"in",
|
||||
"into",
|
||||
"of",
|
||||
"on",
|
||||
"or",
|
||||
"the",
|
||||
"to",
|
||||
"with",
|
||||
"within",
|
||||
"without",
|
||||
"while",
|
||||
"during",
|
||||
"since",
|
||||
"because",
|
||||
}
|
||||
|
||||
EXPERIENCE_PROSE_MARKERS = {
|
||||
"after",
|
||||
@ -55,6 +61,8 @@ EXPERIENCE_PROSE_MARKERS = {
|
||||
"until",
|
||||
"while",
|
||||
"working",
|
||||
"work",
|
||||
"worked",
|
||||
}
|
||||
|
||||
EDUCATION_TITLE_KEYWORDS = {
|
||||
@ -125,7 +133,7 @@ def extract_cv_signals(text: str) -> dict[str, object]:
|
||||
for line in non_empty_lines[1:]:
|
||||
lowered = line.lower()
|
||||
|
||||
if lowered.startswith("education:"):
|
||||
if lowered.startswith(("education:", "formation:")):
|
||||
in_education_section = True
|
||||
pending_education_credential = None
|
||||
remainder = line.split(":", 1)[1].strip()
|
||||
@ -164,10 +172,10 @@ def extract_cv_signals(text: str) -> dict[str, object]:
|
||||
if lowered.startswith("location:"):
|
||||
location = line.split(":", 1)[1].strip() or None
|
||||
continue
|
||||
if lowered.startswith("languages:"):
|
||||
if lowered.startswith(("languages:", "langues:")):
|
||||
languages = _parse_csv_field(line)
|
||||
continue
|
||||
if lowered.startswith("skills:"):
|
||||
if lowered.startswith(("skills:", "compétences:")):
|
||||
skills = _parse_csv_field(line)
|
||||
continue
|
||||
education_entry = _parse_education_entry(line)
|
||||
@ -247,8 +255,7 @@ def _parse_experience_entry(line: str) -> dict[str, str] | None:
|
||||
if not title or not company:
|
||||
continue
|
||||
|
||||
title_words = title.split()
|
||||
if title_words[-1].lower() not in EXPERIENCE_TITLE_ENDINGS:
|
||||
if not _looks_like_experience_title(title):
|
||||
continue
|
||||
|
||||
if _looks_like_prose_company(company):
|
||||
@ -266,6 +273,23 @@ def _looks_like_experience_line(line: str) -> bool:
|
||||
return _parse_experience_entry(line) is not None
|
||||
|
||||
|
||||
def _looks_like_experience_title(title: str) -> bool:
|
||||
title_words = [word.strip(".,;:!?()[]{}") for word in title.split()]
|
||||
if not title_words:
|
||||
return False
|
||||
|
||||
first_word = title_words[0]
|
||||
if len(first_word) < 2 or not first_word[0].isupper():
|
||||
return False
|
||||
|
||||
if first_word.lower() in EXPERIENCE_TITLE_STOPWORDS:
|
||||
return False
|
||||
|
||||
return not any(
|
||||
word.lower() in EXPERIENCE_PROSE_MARKERS for word in title_words if word
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_prose_company(company: str) -> bool:
|
||||
company_words = [word.strip(".,;:!?()[]{}") for word in company.split()]
|
||||
return any(
|
||||
|
||||
@ -107,6 +107,26 @@ def test_extract_cv_signals_normalizes_en_dash_bullet_prefixed_fields_and_experi
|
||||
]
|
||||
|
||||
|
||||
def test_extract_cv_signals_recognizes_french_field_labels() -> None:
|
||||
text = dedent(
|
||||
"""
|
||||
Tonio
|
||||
Formation:
|
||||
M.Sc. in Data Engineering at EPITA
|
||||
Langues: French, English
|
||||
Compétences: Python, SQL
|
||||
"""
|
||||
).strip()
|
||||
|
||||
extracted = extract_cv_signals(text)
|
||||
|
||||
assert extracted["languages"] == ["French", "English"]
|
||||
assert extracted["skills"] == ["Python", "SQL"]
|
||||
assert extracted["education_entries"] == [
|
||||
{"credential": "M.Sc. in Data Engineering", "institution": "EPITA"}
|
||||
]
|
||||
|
||||
|
||||
def test_extract_cv_signals_parses_french_experience_connectors() -> None:
|
||||
text = dedent(
|
||||
"""
|
||||
@ -125,6 +145,30 @@ def test_extract_cv_signals_parses_french_experience_connectors() -> None:
|
||||
]
|
||||
|
||||
|
||||
def test_extract_cv_signals_parses_clear_titles_with_french_and_english_connectors() -> None:
|
||||
text = dedent(
|
||||
"""
|
||||
Tonio
|
||||
Location: France
|
||||
Ingénieur à Thales
|
||||
Ingénieur Data chez BNP Paribas
|
||||
Consultant BI chez Accenture
|
||||
Head of Data at Qonto
|
||||
Product Owner at Qonto
|
||||
"""
|
||||
).strip()
|
||||
|
||||
extracted = extract_cv_signals(text)
|
||||
|
||||
assert extracted["experience_entries"] == [
|
||||
{"title": "Ingénieur", "company": "Thales"},
|
||||
{"title": "Ingénieur Data", "company": "BNP Paribas"},
|
||||
{"title": "Consultant BI", "company": "Accenture"},
|
||||
{"title": "Head of Data", "company": "Qonto"},
|
||||
{"title": "Product Owner", "company": "Qonto"},
|
||||
]
|
||||
|
||||
|
||||
def test_extract_cv_signals_ignores_french_prose_continuations() -> None:
|
||||
text = dedent(
|
||||
"""
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user