"""Conservative, offline normalization of a Leboncoin observation. Missing values remain None. Supplied values that cannot be represented safely produce field/code issues, never copies of raw data. The input is not mutated. """ from datetime import datetime from decimal import Decimal, InvalidOperation, localcontext import re _NUMBER = re.compile(r"[+-]?[0-9]+(?:\.[0-9]+)?\Z") _MAX_PRICE = Decimal("999999999999.99") # PostgreSQL numeric(14, 2). _ATTRIBUTE_FIELDS = { "make": "brand", "model": "model", "model_year": "regdate", "mileage_km": "mileage", "fuel": "fuel", "gearbox": "gearbox", "body_style": "vehicle_type", "power_hp": "horse_power_din", "fiscal_power_cv": "horsepower", } def clean_observation(payload: dict, observed_at: datetime) -> tuple[dict, list[dict]]: """Return the fixed silver field contract and sanitized validation issues. Euro prices must be singleton arrays; cents are scalar integers. Prices allow zero and at most two fractional digits (no rounding). Numeric strings use plain decimal notation, without units or grouping punctuation. Years are 1886..observation year + 1, mileage 0..10,000,000 km, DIN power 1..3,000 hp and fiscal power 1..1,000 CV. Coordinates use global bounds. Publication/update dates must carry their own timezone; observed_at is not a substitute timestamp. Explicit nulls and empty price arrays are missing. """ result = dict.fromkeys(( "price_eur", "currency", *_ATTRIBUTE_FIELDS, "source_published_at", "source_updated_at", "title", "description", "city", "postcode", "department_code", "latitude", "longitude", "location_precision", )) issues = [] def issue(field, code): entry = {"field": field, "code": code} if entry not in issues: issues.append(entry) def number(value, field, minimum, maximum, places=0): if value is None: return None if isinstance(value, bool) or not isinstance(value, (str, int, float, Decimal)): issue(field, "invalid_value") return None if isinstance(value, str) and not _NUMBER.fullmatch(value.strip()): issue(field, "invalid_value") return None try: parsed = Decimal(str(value).strip()) except (InvalidOperation, ValueError): issue(field, "invalid_value") return None if not parsed.is_finite(): issue(field, "invalid_value") return None if not minimum <= parsed <= maximum: issue(field, "out_of_range") return None if places is not None: # Inspect digits rather than round under the active Decimal context. digits = parsed.as_tuple().digits exponent = parsed.as_tuple().exponent trailing = 0 for digit in reversed(digits): if digit != 0: break trailing += 1 if parsed and exponent + trailing < -places: issue(field, "invalid_precision") return None return parsed def text(value, field, nonempty=False): if value is None: return None if not isinstance(value, str) or (nonempty and not value.strip()): issue(field, "invalid_value") return None return value price = payload.get("price") price_bad = False if isinstance(price, list): if len(price) > 1: issue("price_eur", "ambiguous_value") price_bad = True price = None else: price = price[0] if price else None elif price is not None: issue("price_eur", "invalid_value") price_bad = True price = None euros = number(price, "price_eur", 0, _MAX_PRICE, 2) cents_raw = payload.get("price_cents") cents = number(cents_raw, "price_eur", 0, 99999999999999) price_bad |= (price is not None and euros is None) or (cents_raw is not None and cents is None) with localcontext() as context: context.prec = 32 if cents is not None: cents = cents / 100 if euros is not None and cents is not None and euros != cents: issue("price_eur", "conflicting_prices") price_bad = True if not price_bad: chosen = euros if euros is not None else cents if chosen is not None: result["price_eur"] = chosen.quantize(Decimal("0.01")) result["currency"] = "EUR" bounds = { "model_year": (1886, observed_at.year + 1), "mileage_km": (0, 10_000_000), "power_hp": (1, 3_000), "fiscal_power_cv": (1, 1_000), } attributes = payload.get("attributes") if attributes is None: attributes = [] if not isinstance(attributes, list): issue("attributes", "invalid_value") attributes = [] for attribute in attributes: if not isinstance(attribute, dict) or not isinstance(attribute.get("key"), str): issue("attributes", "invalid_value") for field, key in _ATTRIBUTE_FIELDS.items(): candidates = [] signatures = [] for attribute in attributes: if not isinstance(attribute, dict) or attribute.get("key") != key: continue value = attribute.get("value") values = attribute.get("values") invalid = False # model_all is a search ancestor only when a real scalar model # is corroborated by the array. Do not generalize to other *_all codes. if (key == "model" and isinstance(value, str) and value.strip() and value != "model_all" and isinstance(values, list) and value in values): values = [entry for entry in values if entry != "model_all"] if values is not None: if not isinstance(values, list): issue(field, "invalid_value") invalid = True elif len(values) > 1: issue(field, "ambiguous_value") invalid = True elif values: if value is not None and str(value) != str(values[0]): issue(field, "conflicting_attributes") invalid = True elif value is None: value = values[0] label = attribute.get("value_label") signatures.append((str(value) if value is not None else None, label)) if field in bounds: parsed = number(value, field, *bounds[field]) candidate = int(parsed) if parsed is not None else None elif field in ("fuel", "gearbox", "body_style"): candidate = text(label, field, nonempty=True) if label is not None else None if label is None and value is not None: if isinstance(value, (int, float, Decimal)) and not isinstance(value, bool): issue(field, "unmapped_code") elif isinstance(value, str) and _NUMBER.fullmatch(value.strip()): issue(field, "unmapped_code") else: candidate = text(value, field, nonempty=True) else: candidate = text(value, field, nonempty=True) candidates.append(None if invalid else candidate) if signatures and any(signature != signatures[0] for signature in signatures[1:]): issue(field, "conflicting_attributes") elif candidates: result[field] = candidates[0] if all(c == candidates[0] for c in candidates) else None elif field == "make" and (payload.get("attributes") is None or isinstance(payload["attributes"], list)): result[field] = text(payload.get("brand"), field, nonempty=True) for field, key in (("source_published_at", "first_publication_date"), ("source_updated_at", "index_date")): value = payload.get(key) if value is None: continue if not isinstance(value, str): issue(field, "invalid_value") continue try: parsed = datetime.fromisoformat(value) except ValueError: issue(field, "invalid_value") continue if parsed.utcoffset() is None: issue(field, "timezone_missing") else: result[field] = parsed result["title"] = text(payload.get("subject"), "title") result["description"] = text(payload.get("body"), "description") location = payload.get("location") if location is None: location = {} if not isinstance(location, dict): issue("location", "invalid_value") location = {} for field, key in (("city", "city"), ("postcode", "zipcode"), ("department_code", "department_id")): result[field] = text(location.get(key), field, nonempty=True) latitude = number(location.get("lat"), "latitude", -90, 90, places=None) longitude = number(location.get("lng"), "longitude", -180, 180, places=None) if ((location.get("lat") is None) != (location.get("lng") is None) or (latitude is None) != (longitude is None)): issue("latitude", "incomplete_coordinates") issue("longitude", "incomplete_coordinates") elif latitude is not None: result["latitude"], result["longitude"] = latitude, longitude precision = location.get("origin_type") if precision in (None, ""): precision = location.get("type") if precision in (None, ""): # Observed source metadata; "address" does not prove an exact map point. precision = location.get("source") if location.get("source") in ("city", "address") else None result["location_precision"] = text(precision, "location_precision", nonempty=True) return result, issues