a353e1c683
All checks were successful
CI / Lint, PostgreSQL tests, and package (push) Successful in 12s
237 lines
9.9 KiB
Python
237 lines
9.9 KiB
Python
"""Conservative, offline normalization of a Leboncoin observation.
|
|
|
|
Missing values remain None. Supplied values that cannot be represented safely
|
|
produce field/code issues, never copies of raw data. The input is not mutated.
|
|
"""
|
|
|
|
from datetime import datetime
|
|
from decimal import Decimal, InvalidOperation, localcontext
|
|
import re
|
|
|
|
|
|
_NUMBER = re.compile(r"[+-]?[0-9]+(?:\.[0-9]+)?\Z")
|
|
_MAX_PRICE = Decimal("999999999999.99") # PostgreSQL numeric(14, 2).
|
|
_ATTRIBUTE_FIELDS = {
|
|
"make": "brand",
|
|
"model": "model",
|
|
"model_year": "regdate",
|
|
"mileage_km": "mileage",
|
|
"fuel": "fuel",
|
|
"gearbox": "gearbox",
|
|
"body_style": "vehicle_type",
|
|
"power_hp": "horse_power_din",
|
|
"fiscal_power_cv": "horsepower",
|
|
}
|
|
|
|
|
|
def clean_observation(payload: dict, observed_at: datetime) -> tuple[dict, list[dict]]:
|
|
"""Return the fixed silver field contract and sanitized validation issues.
|
|
|
|
Euro prices must be singleton arrays; cents are scalar integers. Prices
|
|
allow zero and at most two fractional digits (no rounding). Numeric
|
|
strings use plain decimal notation, without units or grouping punctuation.
|
|
Years are 1886..observation year + 1, mileage 0..10,000,000 km, DIN power
|
|
1..3,000 hp and fiscal power 1..1,000 CV. Coordinates use global bounds.
|
|
Publication/update dates must carry their own timezone; observed_at is not
|
|
a substitute timestamp. Explicit nulls and empty price arrays are missing.
|
|
"""
|
|
result = dict.fromkeys((
|
|
"price_eur", "currency", *_ATTRIBUTE_FIELDS,
|
|
"source_published_at", "source_updated_at", "title", "description",
|
|
"city", "postcode", "department_code", "latitude", "longitude",
|
|
"location_precision",
|
|
))
|
|
issues = []
|
|
|
|
def issue(field, code):
|
|
entry = {"field": field, "code": code}
|
|
if entry not in issues:
|
|
issues.append(entry)
|
|
|
|
def number(value, field, minimum, maximum, places=0):
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, bool) or not isinstance(value, (str, int, float, Decimal)):
|
|
issue(field, "invalid_value")
|
|
return None
|
|
if isinstance(value, str) and not _NUMBER.fullmatch(value.strip()):
|
|
issue(field, "invalid_value")
|
|
return None
|
|
try:
|
|
parsed = Decimal(str(value).strip())
|
|
except (InvalidOperation, ValueError):
|
|
issue(field, "invalid_value")
|
|
return None
|
|
if not parsed.is_finite():
|
|
issue(field, "invalid_value")
|
|
return None
|
|
if not minimum <= parsed <= maximum:
|
|
issue(field, "out_of_range")
|
|
return None
|
|
if places is not None:
|
|
# Inspect digits rather than round under the active Decimal context.
|
|
digits = parsed.as_tuple().digits
|
|
exponent = parsed.as_tuple().exponent
|
|
trailing = 0
|
|
for digit in reversed(digits):
|
|
if digit != 0:
|
|
break
|
|
trailing += 1
|
|
if parsed and exponent + trailing < -places:
|
|
issue(field, "invalid_precision")
|
|
return None
|
|
return parsed
|
|
|
|
def text(value, field, nonempty=False):
|
|
if value is None:
|
|
return None
|
|
if not isinstance(value, str) or (nonempty and not value.strip()):
|
|
issue(field, "invalid_value")
|
|
return None
|
|
return value
|
|
|
|
price = payload.get("price")
|
|
price_bad = False
|
|
if isinstance(price, list):
|
|
if len(price) > 1:
|
|
issue("price_eur", "ambiguous_value")
|
|
price_bad = True
|
|
price = None
|
|
else:
|
|
price = price[0] if price else None
|
|
elif price is not None:
|
|
issue("price_eur", "invalid_value")
|
|
price_bad = True
|
|
price = None
|
|
euros = number(price, "price_eur", 0, _MAX_PRICE, 2)
|
|
cents_raw = payload.get("price_cents")
|
|
cents = number(cents_raw, "price_eur", 0, 99999999999999)
|
|
price_bad |= (price is not None and euros is None) or (cents_raw is not None and cents is None)
|
|
with localcontext() as context:
|
|
context.prec = 32
|
|
if cents is not None:
|
|
cents = cents / 100
|
|
if euros is not None and cents is not None and euros != cents:
|
|
issue("price_eur", "conflicting_prices")
|
|
price_bad = True
|
|
if not price_bad:
|
|
chosen = euros if euros is not None else cents
|
|
if chosen is not None:
|
|
result["price_eur"] = chosen.quantize(Decimal("0.01"))
|
|
result["currency"] = "EUR"
|
|
|
|
bounds = {
|
|
"model_year": (1886, observed_at.year + 1),
|
|
"mileage_km": (0, 10_000_000),
|
|
"power_hp": (1, 3_000),
|
|
"fiscal_power_cv": (1, 1_000),
|
|
}
|
|
attributes = payload.get("attributes")
|
|
if attributes is None:
|
|
attributes = []
|
|
if not isinstance(attributes, list):
|
|
issue("attributes", "invalid_value")
|
|
attributes = []
|
|
for attribute in attributes:
|
|
if not isinstance(attribute, dict) or not isinstance(attribute.get("key"), str):
|
|
issue("attributes", "invalid_value")
|
|
|
|
for field, key in _ATTRIBUTE_FIELDS.items():
|
|
candidates = []
|
|
signatures = []
|
|
for attribute in attributes:
|
|
if not isinstance(attribute, dict) or attribute.get("key") != key:
|
|
continue
|
|
value = attribute.get("value")
|
|
values = attribute.get("values")
|
|
invalid = False
|
|
# model_all is a search ancestor only when a real scalar model
|
|
# is corroborated by the array. Do not generalize to other *_all codes.
|
|
if (key == "model" and isinstance(value, str) and value.strip()
|
|
and value != "model_all" and isinstance(values, list)
|
|
and value in values):
|
|
values = [entry for entry in values if entry != "model_all"]
|
|
if values is not None:
|
|
if not isinstance(values, list):
|
|
issue(field, "invalid_value")
|
|
invalid = True
|
|
elif len(values) > 1:
|
|
issue(field, "ambiguous_value")
|
|
invalid = True
|
|
elif values:
|
|
if value is not None and str(value) != str(values[0]):
|
|
issue(field, "conflicting_attributes")
|
|
invalid = True
|
|
elif value is None:
|
|
value = values[0]
|
|
label = attribute.get("value_label")
|
|
signatures.append((str(value) if value is not None else None, label))
|
|
if field in bounds:
|
|
parsed = number(value, field, *bounds[field])
|
|
candidate = int(parsed) if parsed is not None else None
|
|
elif field in ("fuel", "gearbox", "body_style"):
|
|
candidate = text(label, field, nonempty=True) if label is not None else None
|
|
if label is None and value is not None:
|
|
if isinstance(value, (int, float, Decimal)) and not isinstance(value, bool):
|
|
issue(field, "unmapped_code")
|
|
elif isinstance(value, str) and _NUMBER.fullmatch(value.strip()):
|
|
issue(field, "unmapped_code")
|
|
else:
|
|
candidate = text(value, field, nonempty=True)
|
|
else:
|
|
candidate = text(value, field, nonempty=True)
|
|
candidates.append(None if invalid else candidate)
|
|
if signatures and any(signature != signatures[0] for signature in signatures[1:]):
|
|
issue(field, "conflicting_attributes")
|
|
elif candidates:
|
|
result[field] = candidates[0] if all(c == candidates[0] for c in candidates) else None
|
|
elif field == "make" and (payload.get("attributes") is None
|
|
or isinstance(payload["attributes"], list)):
|
|
result[field] = text(payload.get("brand"), field, nonempty=True)
|
|
|
|
for field, key in (("source_published_at", "first_publication_date"),
|
|
("source_updated_at", "index_date")):
|
|
value = payload.get(key)
|
|
if value is None:
|
|
continue
|
|
if not isinstance(value, str):
|
|
issue(field, "invalid_value")
|
|
continue
|
|
try:
|
|
parsed = datetime.fromisoformat(value)
|
|
except ValueError:
|
|
issue(field, "invalid_value")
|
|
continue
|
|
if parsed.utcoffset() is None:
|
|
issue(field, "timezone_missing")
|
|
else:
|
|
result[field] = parsed
|
|
|
|
result["title"] = text(payload.get("subject"), "title")
|
|
result["description"] = text(payload.get("body"), "description")
|
|
location = payload.get("location")
|
|
if location is None:
|
|
location = {}
|
|
if not isinstance(location, dict):
|
|
issue("location", "invalid_value")
|
|
location = {}
|
|
for field, key in (("city", "city"), ("postcode", "zipcode"),
|
|
("department_code", "department_id")):
|
|
result[field] = text(location.get(key), field, nonempty=True)
|
|
latitude = number(location.get("lat"), "latitude", -90, 90, places=None)
|
|
longitude = number(location.get("lng"), "longitude", -180, 180, places=None)
|
|
if ((location.get("lat") is None) != (location.get("lng") is None)
|
|
or (latitude is None) != (longitude is None)):
|
|
issue("latitude", "incomplete_coordinates")
|
|
issue("longitude", "incomplete_coordinates")
|
|
elif latitude is not None:
|
|
result["latitude"], result["longitude"] = latitude, longitude
|
|
precision = location.get("origin_type")
|
|
if precision in (None, ""):
|
|
precision = location.get("type")
|
|
if precision in (None, ""):
|
|
# Observed source metadata; "address" does not prove an exact map point.
|
|
precision = location.get("source") if location.get("source") in ("city", "address") else None
|
|
result["location_precision"] = text(precision, "location_precision", nonempty=True)
|
|
return result, issues
|