diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml new file mode 100644 index 0000000..26fff53 --- /dev/null +++ b/.gitea/workflows/ci.yaml @@ -0,0 +1,60 @@ +name: CI + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + test: + name: Lint, PostgreSQL tests, and package + runs-on: python-ci + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: ci-test-only + POSTGRES_DB: car_traker_test + options: >- + --health-cmd "pg_isready -U postgres -d car_traker_test" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + env: + # Disposable service only. Never replace this with the homelab database URL. + TEST_DATABASE_URL: postgresql://postgres:ci-test-only@postgres:5432/car_traker_test + UV_NO_PROGRESS: "1" + steps: + - name: Check out repository + uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Install uv + uses: https://github.com/astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + with: + version: "0.9.3" + python-version: "3.13" + enable-cache: false + add-problem-matchers: false + # Gitea's job token must not be sent to the GitHub release API. + github-token: "" + + - name: Install locked dependencies + run: uv sync --locked --dev + + - name: Lint + run: uv run --locked ruff check src tests + + - name: Run offline and PostgreSQL integration tests + run: timeout 10m uv run --locked python -m unittest discover -s tests -v + + - name: Build package + run: uv build --no-sources + + - name: Check installed wheel and packaged migrations + run: | + uv venv /tmp/car-traker-wheel + uv pip install --python /tmp/car-traker-wheel/bin/python --no-deps dist/*.whl + /tmp/car-traker-wheel/bin/python -c 'from importlib.resources import files; assert files("car_traker").joinpath("migrations", "001_silver.sql").is_file(), "SQL migration missing from wheel"' diff --git a/.gitignore b/.gitignore index 505a3b1..2147240 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,18 @@ wheels/ # Virtual environments .venv + +# Local source snapshots and browser inspection artifacts +data/ +.playwright-mcp/ + +# Local database credentials +.env +.env.* +!.env.example +.pgpass +.pg_service.conf + +# Development caches and runner credentials +.ruff_cache/ +.runner diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md new file mode 100644 index 0000000..7642be3 --- /dev/null +++ b/PROJECT_PLAN.md @@ -0,0 +1,213 @@ +# Youngtimer Market Observatory: Project Handoff + +## Implementation Update (2026-09-08) + +The user subsequently selected live Leboncoin collection and explicitly requested +titles, descriptions, maximum vehicle attributes, and source-provided precise +locations for a future map dashboard, including visits to individual ads. This +supersedes the earlier outstanding input decision and the exclusions of free text +and precise location below. Seller/contact objects and photos remain excluded; +free text may still contain personal information and must remain private until +reviewed/redacted for downstream publication. Source coordinates may be approximate. + +The collector now lives in `src/car_traker/`, storing separate search/detail +observations in versioned compressed JSON Lines with a coverage manifest. See +`README.md` for current commands, capture policy, retention, and verified live +results. The original handoff below remains planning context, not a description +of the current implementation. + +The next milestone is implemented: SQL migrations, conservative silver cleaning, +atomic/idempotent PostgreSQL import, and views for observations, price history, +coverage, and quality issues. Development/CLI execution is on the user's laptop; +the intended database is the existing remote homelab PostgreSQL instance. Local +PostgreSQL containers are disposable tests only. Homelab provisioning and remote +connection setup remain user actions documented in `docs/POSTGRESQL.md`; no homelab +database has been accessed or modified during implementation. + +## Purpose + +Build a French used-car asking-price observatory focused initially on youngtimers. +The goals are market analytics and a data engineering portfolio, not primarily a +car-buying assistant. The user is an experienced data engineer. + +Track listing observations over time, normalize vehicle variants, preserve price +history, and eventually identify listings priced below suitable comparables. +Asking prices are not transaction prices. No confirmed sale-price source has been +identified. + +## Confirmed Requirements + +- Initial families: Saab 900, Mercedes-Benz 124-series, and Volvo 850. +- Include broad variant coverage: both Saab 900 generations; Mercedes sedans, + estates, coupes, and cabriolets; Volvo 850 body styles and engine variants. +- Keep the vehicle catalog extensible to many more models later. +- Geographic scope: all France. +- Target collection cadence: daily, subject to source access feasibility. +- Preferred first platform: Leboncoin. +- No paid data sources. +- Python and SQL; FastAPI if an API becomes necessary, but no API needed initially. +- Polars is optional, not a requirement. Use it only where it adds value over SQL. +- No distributed infrastructure. +- Docker Compose deployment on the user's homelab. +- No deadline or fixed time budget. +- First milestone: a working end-to-end pipeline, not just an exploratory notebook. + +## Environment + +- Homelab: Ryzen 5 7500F, 32 GB RAM. +- PostgreSQL already runs on a Debian VM; reuse it rather than require a second + production database. +- Repository hosted on the user's own Gitea instance. +- Current workspace: `/home/tonio/Workspace/perso/car-traker`. +- At inspection, the repository contained `.gitignore`, `.python-version`, + `main.py`, `pyproject.toml`, and an empty `README.md`. +- `pyproject.toml` specifies Python >=3.13, project name `car-traker`, version + `0.1.0`, and no dependencies. +- The worktree was clean before this handoff document was added. +- No application implementation was changed during the planning discussion. +- The user previously wrote a Leboncoin scraper, but no longer has its code or + sample responses. + +Reinspect the repository before implementation; do not assume it is unchanged. + +## Source Access + +The user prefers Leboncoin and has experience scraping it. They stated that +publicly available data can be collected under French law, then argued that +Leboncoin's allowance for ChatGPT bots permits this project to scrape it. + + +### Outstanding Decision + +The previous agent offered two paths: + +1. Build the source-independent foundation using clearly synthetic snapshots + while source access is investigated. +2. Investigate alternative free sources before implementing the foundation. + +The recommendation was path 1, but the user has not selected either path. Their +latest request was only to write this handoff document. Do not treat synthetic +data, an alternative platform, or live Leboncoin collection as an agreed choice. + +## Proposed Architecture + +These are recommendations discussed with the user, not a fully approved technical +design or an implemented system. + +### Collection and Processing + +- A small Python CLI with source-specific collection/parsing separated from the + normalized import path. +- Prefer a direct HTTP client where an accessible source exposes the required + data without browser rendering. +- Use Playwright when rendering or interaction is genuinely required. The user + suggested it but is open to another suitable approach. +- Do not choose a browser framework before inspecting the source. Playwright + does not itself resolve access restrictions or guarantee anti-bot compatibility. +- No need initially for Scrapy, distributed queues, or a large orchestrator. +- Configurable model families and source search definitions rather than + hardcoded logic for three models. + +### Persistence and Scheduling + +- Existing PostgreSQL for collection runs, listings, observations, normalized + attributes, and analytical SQL views. +- Local mounted storage for minimized source snapshots where retention is + appropriate, enabling replay and parser debugging. +- Snapshot storage outside Git, with a documented retention policy. +- A Compose job triggered daily by a host systemd timer is the proposed simplest + deployment. Compose is confirmed; the timer mechanism is not yet finalized. +- Runtime database configuration supplied outside committed code. PostgreSQL + version, network reachability, credentials, TLS, and available privileges still + need to be established. Never request or commit secrets in the handoff. +- No FastAPI service until there is an actual API consumer. +- SQL-first analytics; optional Polars for exploration or file processing without + duplicating SQL transformations unnecessarily. + +## Data Modeling Principles + +- A source listing is not a physical vehicle. Keep source listing identity + separate from tentative vehicle/repost matches. +- Preserve timestamped observations and asking-price changes, including source + provenance. Repeat imports must not create duplicate observations. +- Separate observation time from any source-provided publication/update time. +- Track collection-run success, failure, partial coverage, and scope so collection + outages or incomplete pagination cannot masquerade as market changes. +- A listing disappearing means it is no longer observed, not that it sold. +- Last observed price is not sale price. +- Observed listing duration is not confirmed total time on market. +- Youngtimers may appreciate. Changes in asking prices or cohort composition are + not automatically vehicle depreciation. +- Missing condition, mileage, or service-history information stays unknown. +- Preserve uncertain normalization rather than inventing precise variants. +- Do not mix synthetic fixtures into real market analytics. + +### Initial Vehicle Distinctions + +| Family | Important distinctions | +| --- | --- | +| Saab 900 | Classic vs second generation, body style, turbo vs naturally aspirated, engine, transmission | +| Mercedes 124-series | Sedan/estate/coupe/cabriolet, chassis designation where known, engine designation, transmission | +| Volvo 850 | Sedan/estate, engine variant, T-5/T-5R/R, transmission | + +Retain these distinctions even when discovery uses broad family searches. Avoid +pooling materially different variants in price comparisons. + +Exclude or explicitly classify parts, donor cars, restoration projects, and +wanted ads. Condition and originality may explain substantial price differences; +below-comparable asking prices are candidates for investigation, not guaranteed +bargains. + +## Privacy and Retention + +The analytics do not need seller names, phone numbers, exact addresses, +registration plates, or photos. Avoid collecting them by default. + +Descriptions and full page responses may contain personal information. Prefer +minimized source records, and define filtering and retention before persisting +raw content. Do not assume retaining every page indefinitely is necessary for +reproducibility. Source identifiers and URLs should also be assessed for their +linkability and retention needs. + +## Proposed First Milestone + +1. Establish the initial data-input path and document the source-access decision. +2. Define an extensible vehicle catalog and a validated observation contract. +3. Implement schema setup/migrations for collection runs, source listings, and + immutable timestamped observations. +4. Implement an idempotent import CLI with validation and transaction handling. +5. Add analytical SQL views for listing price history, comparable cohorts, + observed duration, and collection coverage. +6. Package the job in Docker Compose, connecting to the existing PostgreSQL. +7. Add tests and operational documentation, including scheduling and replay. + +### Acceptance Criteria + +- A small input dataset can be ingested end to end into PostgreSQL. +- Replaying the same observations creates no duplicates or false price changes. +- A later observation preserves earlier values and exposes genuine price changes. +- Vehicle families, generations, and materially different variants remain + distinguishable in analytics. +- Invalid or ambiguous records are handled explicitly, not silently coerced. +- Failed/partial collection cannot falsely mark listings as sold or disappeared. +- Tests can run without contacting Leboncoin or using private data. +- Compose configuration and documented commands support the homelab deployment. +- Documentation clearly distinguishes a replayable foundation from a functioning + live daily collector; synthetic imports alone do not prove live acquisition. + +## Deferred Work + +- Deal scoring and alerts until there is sufficient comparable data. +- Dashboard and FastAPI service until the ingestion/modeling foundation works. +- Probabilistic repost/vehicle deduplication with uncertainty tracking. +- Additional free listing sources, subject to access investigation. +- Government enrichment datasets for market context, not assumed sale prices. +- More advanced longitudinal price analysis that accounts for variant and + condition differences. + +## Guidance for the Next Agent + +Read this document and inspect the current files first. Keep the implementation +small, testable, and source-independent. Do not introduce paid services, +distributed infrastructure, or GitHub-specific deployment assumptions into this +Gitea/homelab project. diff --git a/README.md b/README.md index e69de29..eca48ff 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,226 @@ +# Youngtimer Market Observatory + +First implementation: collect unnormalized listing observations from Leboncoin +search pages and individual ads, including descriptions and source-provided locations. +Finalized raw runs can now be cleaned and imported idempotently into PostgreSQL. +Scheduling and deployment are not implemented yet. +Asking prices are not sale prices; disappearing listings do not prove a sale. + +## Run + +Python 3.13+ and `uv`: + +```sh +uv sync +uv run playwright install chromium +uv run car-traker --family saab-900 --max-pages 1 --max-details 1 --headed +``` + +The smoke command deliberately caps coverage and returns exit code 1 if further +pages or details exist. Collect all configured searches and their individual ads with: + +```sh +uv run car-traker --headed +``` + +Options: `--config searches.toml`, `--output data/raw/leboncoin`, `--family`, +`--max-pages 10` (per search), `--delay 5` (seconds between requests), `--headed`. +Detail visits are enabled by default, once per unique listing ID per run. +`--max-details N` caps detail requests across the whole run (unlimited by default). +`--search-only` disables detail visits. Delay applies to both search and detail +requests. More detail visits mean substantially longer runs and greater exposure +to source access failures. +Linux may need Playwright's system libraries installed by an administrator. +Headed mode requires a graphical session. Omitting `--headed` runs headless, but +headless access failed in the initial live test on this machine. +Do not run overlapping jobs. No credentials, proxies, CAPTCHA solvers, stealth +plugins, or automatic access-block retries are used. + +`searches.toml` defines broad nationwide car-category searches. Add queries/families +there without changing code. Saab 900 and Volvo 850 queries do not restrict +generation, body style, or engine. Mercedes 124 queries include chassis spellings, +but ads using only badges such as "300 CE" may be missed. Search coverage is not a +complete census, and false positives remain for later cleaning. Overlapping query +results are intentionally preserved with their query provenance. + +## Project Structure + +```text +src/car_traker/ # Installed Python package + __init__.py + __main__.py # python -m car_traker entry point + collect.py # Collector and CLI + database.py # PostgreSQL migrations and import CLI + transform.py # Pure, conservative silver field cleaning + migrations/ # Versioned SQL schema and analytical views +tests/ # Offline tests +docs/POSTGRESQL.md # Homelab provisioning, import, queries, database tests +docs/CI.md # Gitea 1.23 CI and repository-scoped runner registration +ops/ci-runner/ # Optional CI runner infrastructure, not app deployment +searches.toml # Search configuration +pyproject.toml # Dependencies, build configuration, and CLI entry point +data/ # Local snapshots (ignored by Git) +``` + +`uv sync` installs the package in editable mode from `src/`. Run commands from the +repository root so the default configuration and data paths resolve correctly. +`uv run python -m car_traker` also works as an alternative to `uv run car-traker`. + +## Silver Import + +Run the importer **on your laptop**, connecting to PostgreSQL **on your homelab**. +No local application database is required. See +[PostgreSQL setup and silver storage](docs/POSTGRESQL.md) for the one-time homelab +role/database setup, network access, and laptop connection configuration. + +Once the remote connection is configured: + +```sh +uv run car-traker-db migrate +uv run car-traker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json +``` + +Importing an unchanged run again is a no-op. Search and detail observations retain +separate timestamps; source coverage and data-quality issues remain explicit. +SQL views expose typed listing history, latest observations, price history, coverage, +and validation issues. Generation/trim inference, redaction, and scheduling are deferred. + +## Raw Format + +**Gzip-compressed JSON Lines (`.jsonl.gz`)**, with one observation per line and one +file per search page or individual ad. JSON preserves nested attributes and original types, including +missing fields and source strings. Gzip reduces storage using standard Python tools. +CSV would flatten or discard structure. Parquet is better considered later, once a +stable cleaned schema exists. These are filtered raw records, not full-page archives. + +```text +data/raw/leboncoin/YYYY-MM-DD// + manifest.json + search-001-page-0001.jsonl.gz + search-001-page-0002.jsonl.gz + detail-3248166132.jsonl.gz +``` + +Each line contains `schema_version: 2`, `record_type` (`search` or `detail`), `source`, +`run_id`, UTC `observed_at`, and source `payload`. Search rows also include discovery +`family`, `search_text`, `search_url`, and `page`; detail rows include `listing_url`. +Join them using `(source, run_id, payload.list_id)`. A listing discovered through +multiple queries has multiple search rows but only one detail visit per run. +Family is a search label, not a validated vehicle classification. + +Search observations are saved before detail requests. Detail observations have +their own timestamps and never overwrite search prices or other source values. +Prices and vehicle attributes are not coerced or normalized. Publication timestamps +remain distinct from observation timestamps. A new collection creates a new run, +never overwriting previous runs. Existing version-1 snapshots remain unchanged; +they lack the newly captured fields and cannot be enriched without recollection. + +Files are finalized with an atomic rename. The manifest is updated after each page +and records counts, source-reported totals/page limits, the planned searches, and +coverage status. Use only finalized files referenced in a manifest for replay. +An interrupted process may leave `.tmp` files; ignore them. A hard kill can leave +`status: running`; treat that run as incomplete. Storage failures may also prevent +the final manifest update. + +`complete` means all pages reported by the source for each configured query were +visited and every discovered unique listing had a successful detail visit (unless +explicitly disabled), not that every relevant ad was found. Results can change +during pagination; reported totals are diagnostic, not transactional guarantees. +`partial` means a page/detail cap, unavailable detail, interruption, or failure after +some pages; `failed` means no pages were saved. Exit code is 0 only for a complete run. + +Search coverage and `details.status` are tracked separately. Detail coverage refers +only to discovered IDs, not ads on unvisited search pages. `details.listings` maps +each discovered ID to `complete`, `unavailable`, `failed`, or `not_attempted`, plus +its snapshot filename or failure/skip reason. Successful empty discovery needs no +detail requests; failed discovery does not imply successful detail coverage. +HTTP 404/410 marks a detail unavailable and continues, without inferring a sale. +A block (including 403/429), malformed payload, or repeated search page stops the +whole run without retries. Inspect `error_type`, `reason`, and `http_status` in the +manifest; exception bodies are not stored to avoid leaking page contents. + +Read a snapshot without cleaning: + +```python +import gzip +import json + +with gzip.open("data/raw/leboncoin///search-001-page-0001.jsonl.gz", "rt") as f: + for line in f: + observation = json.loads(line) + print(observation["observed_at"], observation["payload"]["price"]) +``` + +## Privacy And Retention + +The expanded capture was explicitly requested for later vehicle analysis and a map +dashboard. It retains listing IDs, timestamps, category/status, prices, title +(`subject`), description (`body`), URL, brand, listing options/counters, and the full +source `location` object. This includes city/postcode, latitude/longitude, and +location provenance/precision metadata when supplied. Coordinates are **not proof +of an exact vehicle or seller address**; they may identify a city centroid or an +approximate point. Missing coordinates stay missing; no geocoding is performed. + +Attributes are now retained by default, including newly introduced vehicle fields, +except known seller/profile/rating/contact fields, plate/VIN identifiers, and the +external vehicle-history report link. Structured seller objects, contact buttons, +photos, and related-listing objects are excluded. Unknown top-level fields remain +excluded; this is not a promise to capture every field the website may ever expose. + +Titles and descriptions are stored **verbatim, without redaction**. They can contain +names, phone numbers, addresses, or other personal information supplied by sellers. +New attribute keys and location fields can also contain unexpected information; +the field filter is not an anonymization guarantee. Do not publish raw text or +location records directly in the dashboard. Add privacy review/redaction and an +appropriate map precision policy before exposing a public dataset or dashboard. + +Listing IDs are linkable. Treat the raw directory as private: restrict filesystem +access, do not publish snapshots, and exclude it from Git (already configured). +New run directories are created with owner-only permissions (`0700` on Linux). +Initial retention policy: delete whole run directories after **30 days**, including +backups. Retention enforcement is currently manual, not an automatic deletion job; +set it up before unattended collection. Full page HTML is processed in browser +memory, not archived. Do not enable browser tracing or save browser profiles by default. + +## Source Access + +On 2026-09-08, direct HTTP search returned 403, while the inspection browser loaded +a Saab 900 search and exposed 35 records in +`#__NEXT_DATA__.props.pageProps.searchData` (101 reported results, three pages). +The collector therefore uses Playwright Chromium and reads that embedded JSON. +Individual ads expose `#__NEXT_DATA__.props.pageProps.ad`; inspection confirmed a +description was populated there while the search record's `body` was empty. +Access can differ across browser environments, +IP addresses, and runs. This observation is not proof of reliable daily access or +permission to collect; review applicable source terms and requirements before +unattended deployment. A challenge or changed payload is a failure, not zero ads. + +The original search-only Python CLI was also tested on 2026-09-08: headless collection failed +before saving records; headed Chromium saved **35 real Saab 900 observations**. +A subsequent headed pagination test saved page 1, then received HTTP 403 on page 2. +Its manifest correctly reports `partial`, `SearchAccessError`, and `http_status: 403`. +Live first-page acquisition is verified; full pagination, all-family coverage, and +reliable unattended daily collection are **not** verified. The collector stops on +that access failure rather than retrying or presenting partial data as complete. + +The expanded CLI was tested with `--max-pages 1 --max-details 1 --headed` on the +same date: it saved 35 search observations and one successful individual-ad snapshot +under run `ed585bcc-188d-401e-bc9f-8bc1318d4ca6`. The run is intentionally partial +because of both limits. Full detail coverage and reliable unattended collection +remain unverified. + +## Tests + +```sh +uv sync --locked --dev +uv run --locked ruff check src tests +uv run python -m unittest discover -s tests -v +``` + +Tests use clearly synthetic in-memory responses and temporary storage. They never +contact Leboncoin, and fixtures are not mixed with real observations. + +[Gitea CI](docs/CI.md) runs linting, the full suite against disposable PostgreSQL, +and package validation on pushes and pull requests. Enable Repository Actions and +register the documented `python-ci` runner before expecting remote jobs to execute. +CI never scrapes Leboncoin or connects to the homelab application database. diff --git a/docs/CI.md b/docs/CI.md new file mode 100644 index 0000000..920ad1a --- /dev/null +++ b/docs/CI.md @@ -0,0 +1,147 @@ +# Gitea CI + +This repository targets **Gitea 1.23.8**. The workflow is in +`.gitea/workflows/ci.yaml`; it runs on pushes, pull requests, and manual dispatch. +It adds continuous integration only: no application deployment, scheduled scraping, +production migrations, or access to the homelab application database. + +## What Runs + +One Docker-backed job, selected by the runner label `python-ci`, performs: + +1. Checkout using a commit-pinned action, without persisting checkout credentials. +2. Install uv 0.9.3 and Python 3.13, then `uv sync --locked --dev`. +3. Run Ruff's correctness-focused checks over `src` and `tests`. +4. Run the full unittest suite, including integration tests against a fresh + PostgreSQL 16 service named `postgres` and database `car_traker_test`. +5. Build the source distribution and wheel, install that wheel in an independent + environment, and verify that its SQL migrations are packaged. + +The service password `ci-test-only` is deliberately public and only for the +ephemeral test database. No database port is published on the runner host. Never +replace `TEST_DATABASE_URL` with a production connection string: the integration +tests truncate their test tables. No browser installation or live scraping occurs. +No raw data, browser profiles, password files, or laptop environment is mounted. + +The job image is `node:22-bookworm` so JavaScript actions have Node, Git, and the +basic Linux tools they need; uv supplies Python. The image is not your application +deployment image. Both action references use explicit GitHub URLs and commit SHAs. +The setup-uv action receives an empty `github-token`, so the Gitea job token is not +sent to GitHub's release API. Caching and problem matchers are disabled for this +first Gitea setup. PostgreSQL and Node images track their declared version tags; +action commits, uv, and the Python dependency lockfile are explicitly pinned. + +Gitea 1.23 ignores several GitHub workflow keys, including job-level +`timeout-minutes`, `permissions`, and `concurrency`; the workflow does not pretend +they provide protections. Runner configuration enforces capacity 1 and a 30-minute +job timeout; the test command also has a 10-minute timeout. + +## Enable Repository Actions + +In Gitea, open the `tonio/car-tracker` repository's **Settings** and enable +**Repository Actions**. Actions are enabled globally by default since Gitea 1.21, +but disabled for repositories by default. If the repository option is unavailable, +check the instance's `[actions] ENABLED` setting and your permissions. + +Pushing a workflow is not enough: a runner with the matching label must be online. +Until then, runs can remain waiting. If Actions were disabled when the workflow +was pushed, enable them and use the manual run button afterward. + +## Register A Runner + +Use a dedicated Linux CI VM with Docker and Docker Compose, preferably separate +from the Gitea and PostgreSQL hosts. A laptop can host a temporary runner, but jobs +will stop being serviced when it sleeps or disconnects. The included configuration +uses `gitea/act_runner:0.2.13`, with the configuration format verified from that +release. Do not substitute the newer renamed runner without checking compatibility +with your Gitea version. + +The runner daemon needs Docker socket access, which is effectively root access to +the CI host. Do not put production database credentials, backups, or raw listing +data on that host. Job containers do not receive the socket or arbitrary host bind +mounts. These restrictions reduce exposure but do not make self-hosted execution +safe for arbitrary hostile workflows. Register at repository scope, accept only +trusted contributors/workflows, and review fork pull requests before allowing them +to run. Runner and server updates should be planned independently of this initial +setup; Gitea 1.23.8 is an older release. + +On the CI host, check out this repository, then run from its root: + +```sh +docker compose -f ops/ci-runner/compose.yaml pull +docker compose -f ops/ci-runner/compose.yaml run --rm runner \ + --config /config.yaml register +``` + +Registration is interactive, so a registration token is not placed in shell +arguments or a committed environment file. Obtain a **repository-scoped** token +from `tonio/car-tracker` -> **Settings -> Actions -> Runners**. Enter: + +| Prompt | Value | +| --- | --- | +| Instance URL | Your Gitea **HTTP(S) web URL**, including its port if needed | +| Registration token | The repository token, entered locally; do not send it in chat | +| Runner name | For example `car-tracker-ci` | +| Labels | `python-ci:docker://node:22-bookworm` | + +The instance URL is **not** the Git SSH URL ending in port 2222. It must resolve +and be reachable from both the runner container and its job containers. Avoid +`localhost`, which would refer to a container. If using a VPN-only hostname, +verify DNS and routing inside Docker as well as on the CI host. HTTPS certificate +verification remains enabled; configure trust for private certificates rather +than disabling it. + +Registration writes credentials to `/data/.runner` in the persistent Docker volume. +Treat that volume as a secret. No registration token is needed in Compose's +environment. Start the registered daemon: + +```sh +docker compose -f ops/ci-runner/compose.yaml up -d +docker compose -f ops/ci-runner/compose.yaml logs --tail=100 runner +``` + +Confirm in Gitea that the runner is online and advertises **`python-ci`**, then +trigger the CI workflow. The runner needs outbound access to your Gitea web URL, +GitHub/action releases, PyPI/Python downloads, and the container registry. It does +not need your application PostgreSQL credentials or an HBA rule for that database. +Restrict CI-host network access to production services where practical. + +Do not run `docker compose down -v` unless you intend to erase runner registration. +Normal container recreation preserves the registration volume. + +## First Run Checks + +- A job stuck waiting usually means Actions are disabled, the runner is offline, + or its label does not match `python-ci`. +- Checkout failures often mean job containers cannot resolve/reach Gitea's web URL + or trust its TLS certificate. Successful Git SSH access from the laptop does not + prove HTTP(S) checkout works from a Docker container. +- `postgres` must resolve inside the job's Docker network. Leave `container.network` + empty so act_runner creates the service network; do not use host networking. +- The integration tests must execute, not be skipped. `TEST_DATABASE_URL` is set + in the workflow and points only to the disposable service. +- Caching is deliberately absent, so no cache-server ports or credentials are needed. + +The workflow was executed locally using `act_runner 0.2.13` against an archive of +the staged source, with no laptop credentials or raw snapshots included. Linting, +all 82 tests (including PostgreSQL integration tests), the build, and installed-wheel +migration checks passed. Local execution substituted the source archive for remote +checkout; Gitea HTTP(S) checkout and server-triggered execution still need verification +after a runner is registered and the repository's Actions feature is enabled. + +## Local Checks + +```sh +uv sync --locked --dev +uv run --locked ruff check src tests +uv run --locked python -m unittest discover -s tests -v +uv build --no-sources +``` + +Without `TEST_DATABASE_URL`, database integration tests are skipped locally. Use +the [disposable PostgreSQL instructions](POSTGRESQL.md#tests) to run them, never the +homelab application database. Keep `.env`, `.pgpass`, `.runner`, and `data/` out of Git. + +References: [Gitea 1.23 quick start](https://docs.gitea.com/1.23/usage/actions/quickstart), +[runner setup](https://docs.gitea.com/1.23/usage/actions/act-runner), and +[compatibility differences](https://docs.gitea.com/1.23/usage/actions/comparison). diff --git a/docs/POSTGRESQL.md b/docs/POSTGRESQL.md new file mode 100644 index 0000000..3f55f70 --- /dev/null +++ b/docs/POSTGRESQL.md @@ -0,0 +1,310 @@ +# Silver Storage And Homelab Setup + +This milestone loads finalized local raw runs into your **existing PostgreSQL +server**. It does not deploy a second production server or schedule scraping. +Development and CLI execution currently happen on your laptop. Raw files remain +on the laptop, and only the database connection crosses the network: + +```text +Laptop: raw JSONL -> Python silver importer -> Homelab VM: PostgreSQL/car_traker +``` + +You do not need to install PostgreSQL on the laptop or move the repository to the +homelab for this milestone. The optional local container at the end of this document +is exclusively for isolated tests, not the application database. +PostgreSQL 14+ is required; PostgreSQL 16 is used for integration testing. +No PostGIS or other extension is needed for this first map-ready dataset. + +## Actions On The Homelab + +Create a dedicated database and an application login. Run these commands as your +PostgreSQL administrator, for example with `sudo -u postgres psql` on the Debian VM: + +```sql +CREATE ROLE car_traker LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; +SET password_encryption = 'scram-sha-256'; +\password car_traker +CREATE DATABASE car_traker OWNER car_traker; +REVOKE ALL ON DATABASE car_traker FROM PUBLIC; +\connect car_traker +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +``` + +`\password` prompts locally without putting the password in SQL history. These are +one-time provisioning commands; inspect existing roles/databases rather than +rerunning blindly if the names already exist. The importer connects as `car_traker`, +not `postgres`. Owning this dedicated database allows it to create its `silver` +schema and run migrations without cluster-wide administrative privileges. + +If importing from another machine, verify PostgreSQL is already listening on the +VM's LAN address. If not, adjust `listen_addresses` in the existing cluster's +`postgresql.conf`. A change to that setting requires a cluster restart, which +affects other clients: schedule it rather than restarting the VM blindly. + +Restrict the firewall to the importer machine's IP and PostgreSQL port, normally +5432. Do not expose PostgreSQL publicly. With server TLS configured, add a narrowly +scoped `pg_hba.conf` rule, replacing `` with the actual client address: + +```text +hostssl car_traker car_traker /32 scram-sha-256 +``` + +Rules are first-match; review existing rules and their order. Reload authentication +changes with `SELECT pg_reload_conf();`. `hostssl` requires PostgreSQL server TLS; +the client hostname must match the server certificate for `verify-full` below. +If your server has no TLS yet, configure it or use a secured SSH/VPN connection +appropriate to your network before sending data/passwords across it. Do not solve +connection errors by adding a broad `trust` rule or opening access to all hosts. + +## Local Connection Configuration + +Run the following on the **laptop**, pointing `PGHOST` at the **homelab VM**, not +`localhost`. In the server access rule above, `` is currently the +laptop's LAN/VPN address as seen by PostgreSQL, not a future deployment container. + +The CLI uses standard libpq environment variables or a `DATABASE_URL` supplied +outside the repository. Prefer a password file rather than a password in commands: + +```sh +export PGHOST=postgres.your-lan.example +export PGPORT=5432 +export PGDATABASE=car_traker +export PGUSER=car_traker +export PGSSLMODE=verify-full +export PGSSLROOTCERT=/path/to/your/postgres-ca.crt +export PGPASSFILE="$HOME/.pgpass" +``` + +The matching password-file entry has this form: + +```text +postgres.your-lan.example:5432:car_traker:car_traker: +``` + +Protect it with `chmod 600 "$HOME/.pgpass"`. Escape `:` and `\` in passwords using +libpq's password-file rules. The example hostname and certificate path are +placeholders. Do not commit credentials or send them in chat. `.env` is ignored by +Git, but it is **not automatically loaded** by this CLI. `DATABASE_URL`, if present, +can override the equivalent `PG*` settings; unset stale values when switching DBs. +Without an explicit host, service, or connection URL, the CLI refuses to run instead +of guessing that a local PostgreSQL server exists. + +With `psql` installed, verify connectivity before importing: + +```sh +psql -c 'SELECT current_database(), current_user, version();' +``` + +Alternatively, no local `psql` installation is needed for the read-only CLI check: + +```sh +uv run car-traker-db check +``` + +It connects with the same settings as migrations/imports and executes only `SELECT 1`. + +Enable backups for the new database and include it in your existing monitoring. +Do not point integration tests at it. + +## Connection Troubleshooting + +The CLI reports safe error categories, not raw connection exceptions that may +contain credentials or private data. `no pg_hba.conf entry` means PostgreSQL was +reached, but its access rules do not allow the client/database/role combination. +The rule must use the **client IP seen by PostgreSQL**, which can be a VPN or NAT +address rather than the laptop's Wi-Fi/LAN address. Use the actual database and +role names you created; `car_traker` in this document is an example, not a hardcoded +application requirement. + +On the homelab, locate the active configuration and inspect parsed rules as an +administrator: + +```sql +SHOW hba_file; +SELECT line_number, type, database, user_name, address, auth_method, error +FROM pg_hba_file_rules; +``` + +Add the appropriate narrow `hostssl` rule in the file returned by `SHOW hba_file`, +then run `SELECT pg_reload_conf();`. Check the `error` column and server logs if it +still fails; a successful reload request alone does not prove every rule is valid. +Do not change passwords or use `trust` to work around a missing HBA rule. Once access +is permitted, rerun `car-traker-db check` from the laptop before migrating. + +Other categories point to DNS/VPN, refused/timed-out connections, authentication, +TLS verification, missing databases, or insufficient privileges. This check does +not mutate schema or verify that the role has migration privileges. + +## Migrate And Import + +From the repository root, with the connection configured: + +```sh +uv sync +uv run car-traker-db migrate +uv run car-traker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json +``` + +The example uses the existing 35-search/one-detail live run. Choose any finalized +manifest to import another run. Repeating the same command returns +`status: already_imported`, without additional observations or false price changes. +Migrations are packaged SQL files applied transactionally with stored checksums. +Add a new migration to evolve the schema; never edit an already-applied migration. +Run migrations explicitly before imports. They are not performed implicitly on each run. + +Verified on 2026-09-08 against disposable PostgreSQL 16 using a non-superuser +database owner: 36 observations accepted, zero rejected, 35 latest listings with +coordinates and source location metadata. An identical replay inserted nothing. +The sample has 72 field issues: both source timestamps lack timezones on all 36 +observations. No timezone was invented to make those issues disappear. This test +did not connect to or change the homelab database. + +Import exit codes: + +| Code | Meaning | +| --- | --- | +| 0 | Import committed (possibly with field issues), or identical run already imported | +| 1 | Input integrity, connection, or database failure; import did not commit | +| 2 | Import or replay has rejected records; inspect `v_validation_issues` | + +A successful import is not a successful collection. Importing a partial or failed +source run is intentional and preserves its coverage status. A `running` manifest +is refused because files may still be changing; investigate interrupted runs before +finalizing their metadata. The importer never changes the raw files or source status. + +## Schema And Cleaning + +All objects live in the private `silver` schema: + +| Table | Purpose | +| --- | --- | +| `schema_migrations` | Applied migration checksums | +| `collection_runs` | Original manifest, source status, timestamps, transform version | +| `raw_files` | Referenced snapshot names, compressed-byte checksums, accepted/rejected counts | +| `search_coverage` | Every planned query, including queries never attempted | +| `detail_coverage` | Discovered detail IDs and their collection outcomes | +| `listings` | Source + listing ID, not a physical-vehicle identity | +| `observations` | Immutable-on-import typed observations, original payload, field issues | +| `rejected_records` | Invalid record provenance, reason, and input hash; no text copied into errors | + +The initial transformation contract is version 1: + +- Euro prices come from a singleton `price` array and/or `price_cents / 100`, using + decimal arithmetic. Conflicting units, multiple candidate prices, fractional + cents, negative values, and invalid formats become NULL with an issue. Currency + is EUR under this Leboncoin source contract, not inferred for other platforms. + Numeric JSON tokens are decoded and persisted with decimal precision, without + an intermediate binary-float rounding step. +- Make/model remain source values. Model taxonomy markers such as `model_all` are + not alternative vehicle models. Fiscal power reads `horsepower`; DIN power reads + `horse_power_din`. Fuel, gearbox, and body style use source labels when available; + unlabeled numeric category codes remain unknown rather than guessed. +- Year, mileage, and power are parsed conservatively with documented bounds in + `transform.py`. Missing stays NULL. Unknown variants and generations are not inferred + from year, title, or discovery family. +- Observation times must have a timezone and match the manifest. Source publication + and update times lacking a timezone become NULL with `timezone_missing`; their + original strings remain in `raw_payload`. No assumed UTC or Paris offset is added. +- Coordinates must be a complete valid latitude/longitude pair. Bounds include + overseas France. Location precision is source metadata, never an assertion that + a map pin is the seller's exact address. When `origin_type`/`type` are blank, the + observed `location.source` values `city` or `address` are retained as metadata. + No geocoding occurs. +- Titles/descriptions and raw attributes are retained privately, without NLP or + redaction. Missing description in a search row is not filled from a later detail. + +Version-1 and version-2 raw files are supported because both already exist locally. +The importer reads only files referenced by the manifest, rejects path traversal, +verifies row counts, and ignores orphan/temp files. Missing or corrupt files fail +the whole run. Invalid individual record envelopes are quarantined as rejected +records; valid records with invalid field values remain observations with NULLs +and issues. A run imports in one transaction. Per-run database locks serialize +concurrent replays; original manifest/file checksums prevent silently accepting +changed input under an existing run ID. + +Replays do not apply newer cleaning rules retroactively. A future transformation +upgrade needs an explicit, tested reprocessing migration using preserved payloads, +not a normal replay that overwrites historical observations. + +## Query The Milestone + +Inspect source coverage and silver quality separately: + +```sql +SELECT * FROM silver.v_collection_coverage ORDER BY started_at DESC; +SELECT * FROM silver.search_coverage ORDER BY run_id, search_index; +SELECT * FROM silver.v_validation_issues ORDER BY run_id, raw_file, raw_line; +``` + +Map-ready latest observations: + +```sql +SELECT source_listing_id, price_eur, make, model, model_year, mileage_km, + city, postcode, latitude, longitude, location_precision, + observed_at, record_type, collection_status, quality_issue_count +FROM silver.v_latest_listings +WHERE latitude IS NOT NULL AND longitude IS NOT NULL; +``` + +`v_latest_listings` selects a single latest observation per listing, preferring +detail only on an equal timestamp. A newer search row can have fewer fields than +an older detail row; the view does not manufacture a combined observation. Search +family remains provenance, not a validated model match. Detail family can be found +by joining search observations on `(source, run_id, source_listing_id)`; multiple +discovery families are possible. + +Price history and observed changes: + +```sql +SELECT * FROM silver.v_price_history +WHERE previous_observed_price_eur IS NOT NULL + AND price_eur <> previous_observed_price_eur +ORDER BY source, source_listing_id, observed_at; +``` + +`v_listing_observations` provides the full typed history. Price history includes +both search and detail records with their timestamps; differences can occur within +a run. An unavailable detail or absence from a later search does not set a sold +flag. All these views are private analytical tools, not a public API. + +## Privacy And Retention + +Base tables contain raw descriptions and precise source locations. SQL views omit +free text/raw payloads but still expose linkable IDs and map points; that does not +make them anonymized. No public grants are made. Do not give a public dashboard +the owner account; define a separate read-only role and reviewed/redacted views +before exposing it. + +Deleting 30-day-old raw files does **not** remove their copies from PostgreSQL. +Apply the same 30-day initial retention to private `title`, `description`, and +`raw_payload` copies and relevant backups before unattended operation. Retention +enforcement remains a separate operational task, not implemented automatically in +this milestone. Keep typed historical market data private too; decide its retention +and location precision explicitly before publication. Raw checksums provide +integrity/provenance, not recovery after the files are deleted. + +## Tests + +All ordinary tests are offline and use synthetic data: + +```sh +uv run python -m unittest discover -s tests -v +``` + +Database integration tests are skipped unless `TEST_DATABASE_URL` is set. They +**truncate the silver data tables** and require an explicit database name ending +`_test`. Use only a disposable database, never the homelab production database. +One local example (test password only; localhost-only port; ephemeral data): + +```sh +docker run --detach --rm --name car-traker-silver-test \ + --env POSTGRES_PASSWORD=local-test-only --env POSTGRES_DB=car_traker_test \ + --publish 127.0.0.1:55432:5432 --tmpfs /var/lib/postgresql/data postgres:16 +docker exec car-traker-silver-test pg_isready -U postgres -d car_traker_test +TEST_DATABASE_URL='postgresql://postgres:local-test-only@127.0.0.1:55432/car_traker_test' \ + uv run python -m unittest discover -s tests -v +docker stop car-traker-silver-test +``` + +Wait until `pg_isready` succeeds before running the tests. This temporary container +is only a testing option, not the production deployment architecture. diff --git a/main.py b/main.py deleted file mode 100644 index 719f37c..0000000 --- a/main.py +++ /dev/null @@ -1,6 +0,0 @@ -def main(): - print("Hello from car-traker!") - - -if __name__ == "__main__": - main() diff --git a/ops/ci-runner/compose.yaml b/ops/ci-runner/compose.yaml new file mode 100644 index 0000000..2bd3d5b --- /dev/null +++ b/ops/ci-runner/compose.yaml @@ -0,0 +1,15 @@ +# CI infrastructure only; this does not deploy the application or PostgreSQL. +services: + runner: + image: gitea/act_runner:0.2.13 + restart: unless-stopped + entrypoint: ["act_runner"] + command: ["--config", "/config.yaml", "daemon"] + working_dir: /data + volumes: + - runner-data:/data + - ./config.yaml:/config.yaml:ro + - /var/run/docker.sock:/var/run/docker.sock + +volumes: + runner-data: diff --git a/ops/ci-runner/config.yaml b/ops/ci-runner/config.yaml new file mode 100644 index 0000000..60257a4 --- /dev/null +++ b/ops/ci-runner/config.yaml @@ -0,0 +1,23 @@ +log: + level: info + +runner: + file: /data/.runner + capacity: 1 + timeout: 30m + shutdown_timeout: 30s + insecure: false + env_file: "" + labels: + - "python-ci:docker://node:22-bookworm" + +cache: + enabled: false + +container: + network: "" + privileged: false + valid_volumes: [] + # The daemon uses Docker, but jobs/services do not receive the Docker socket. + docker_host: "-" + force_pull: true diff --git a/pyproject.toml b/pyproject.toml index 13b4daf..205e4cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,30 @@ [project] name = "car-traker" version = "0.1.0" -description = "Add your description here" +description = "French youngtimer asking-price observatory" readme = "README.md" requires-python = ">=3.13" -dependencies = [] +dependencies = ["playwright>=1.58,<2", "psycopg[binary]>=3.2,<4", "simplejson>=3.20,<4"] + +[dependency-groups] +dev = ["ruff>=0.14,<0.15"] + +[tool.ruff] +target-version = "py313" + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F"] + +[project.scripts] +car-traker = "car_traker.collect:main" +car-traker-db = "car_traker.database:main" + +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +car_traker = ["migrations/*.sql"] diff --git a/searches.toml b/searches.toml new file mode 100644 index 0000000..868c2f6 --- /dev/null +++ b/searches.toml @@ -0,0 +1,28 @@ +# Broad discovery only; matching and vehicle normalization happen later. +[[searches]] +family = "saab-900" +text = "saab 900" + +[[searches]] +family = "volvo-850" +text = "volvo 850" + +[[searches]] +family = "mercedes-124" +text = "mercedes 124" + +[[searches]] +family = "mercedes-124" +text = "mercedes w124" + +[[searches]] +family = "mercedes-124" +text = "mercedes s124" + +[[searches]] +family = "mercedes-124" +text = "mercedes c124" + +[[searches]] +family = "mercedes-124" +text = "mercedes a124" diff --git a/src/car_traker/__init__.py b/src/car_traker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/car_traker/__main__.py b/src/car_traker/__main__.py new file mode 100644 index 0000000..1d82a74 --- /dev/null +++ b/src/car_traker/__main__.py @@ -0,0 +1,5 @@ +from .collect import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/car_traker/collect.py b/src/car_traker/collect.py new file mode 100644 index 0000000..54c5083 --- /dev/null +++ b/src/car_traker/collect.py @@ -0,0 +1,309 @@ +"""Collect unnormalized Leboncoin search and individual listing observations.""" + +import argparse +import gzip +import json +import math +import time +import tomllib +import uuid +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlencode + + +FIELDS = { + "list_id", "first_publication_date", "expiration_date", "index_date", + "status", "category_id", "category_name", "ad_type", "price", "price_cents", + "subject", "body", "url", "brand", "is_boosted", "options", "counters", +} +EXCLUDED_ATTRIBUTES = { + "phone", "phone_number", "email", "name", "first_name", "last_name", + "siren", "siret", "licence_plate", "license_plate", "registration_number", "vin", + "vehicle_history_report_public_url", +} +EXCLUDED_ATTRIBUTE_PREFIXES = ("owner_", "seller_", "profile_", "rating_", "contact_") + + +class SourceAccessError(RuntimeError): + def __init__(self, status): + super().__init__("Source access failed; stopping without retries") + self.status = status + + +def now(): + return datetime.now(timezone.utc).isoformat() + + +def search_url(text, page): + return "https://www.leboncoin.fr/recherche?" + urlencode( + {"category": "2", "text": text, "page": page} + ) + + +def validate_ad(ad): + if not isinstance(ad, dict): + raise ValueError("Invalid listing object") + identifier = str(ad.get("list_id", "")) + if not identifier.isascii() or not identifier.isdigit() or int(identifier) < 1: + raise ValueError("Invalid listing ID") + if not isinstance(ad.get("location", {}), dict): + raise ValueError("Invalid location object") + attributes = ad.get("attributes", []) + if not isinstance(attributes, list) or any( + not isinstance(a, dict) or not isinstance(a.get("key"), str) for a in attributes + ): + raise ValueError("Invalid attributes") + + +def parse_search(text): + """Validate the source envelope, not the vehicle values.""" + try: + data = json.loads(text)["props"]["pageProps"]["searchData"] + ads, total, pages = data["ads"], data["total"], data["max_pages"] + if not isinstance(ads, list): + raise ValueError() + if type(total) is not int or type(pages) is not int or total < 0 or pages < 0: + raise ValueError() + if (total > 0 and (not ads or pages == 0)) or (total == 0 and ads): + raise ValueError() + for ad in ads: + validate_ad(ad) + return data + except (KeyError, TypeError, ValueError) as exc: + raise ValueError("Missing or changed Leboncoin search payload") from exc + + +def parse_detail(text, expected_id): + try: + ad = json.loads(text)["props"]["pageProps"]["ad"] + validate_ad(ad) + if str(ad["list_id"]) != str(expected_id): + raise ValueError() + return ad + except (KeyError, TypeError, ValueError) as exc: + raise ValueError("Missing, changed, or mismatched Leboncoin detail payload") from exc + + +def minimize(ad): + result = {key: value for key, value in ad.items() if key in FIELDS} + if "attributes" in ad: + result["attributes"] = [ + a for a in ad["attributes"] + if a["key"].lower() not in EXCLUDED_ATTRIBUTES + and not a["key"].lower().startswith(EXCLUDED_ATTRIBUTE_PREFIXES) + ] + if "location" in ad: + result["location"] = ad["location"] + return result + + +def write_json(path, value): + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def write_records(path, records): + temporary = path.with_suffix(path.suffix + ".tmp") + with gzip.open(temporary, "wt", encoding="utf-8") as stream: + for record in records: + stream.write(json.dumps(record, ensure_ascii=False) + "\n") + temporary.replace(path) + + +def collect(fetch, searches, output, max_pages=10, delay=5, search_only=False, max_details=None): + run_id = str(uuid.uuid4()) + directory = output / datetime.now(timezone.utc).strftime("%Y-%m-%d") / run_id + directory.mkdir(mode=0o700, parents=True, exist_ok=False) + manifest = { + "schema_version": 2, "source": "leboncoin", "run_id": run_id, + "started_at": now(), "status": "running", "max_pages": max_pages, + "delay_seconds": delay, "searches": [], + "privacy_policy": "v2-text-location-filtered-attributes", + "planned_searches": searches, + "details": { + "enabled": not search_only, "max_details": max_details, + "status": "disabled" if search_only else "running", "listings": {}, + }, + } + path = directory / "manifest.json" + write_json(path, manifest) + requests = 0 + detail_attempts = 0 + target = None + + def fetch_page(url, item=None): + nonlocal requests, detail_attempts + if requests: + time.sleep(delay) + if item is not None: + detail_attempts += 1 + item.update(status="running", attempted_at=now()) + write_json(path, manifest) + requests += 1 + return fetch(url) + + try: + for index, search in enumerate(searches): + scope = {**search, "status": "running", "pages": [], "records": 0} + manifest["searches"].append(scope) + write_json(path, manifest) + seen_pages = set() + for number in range(1, max_pages + 1): + target = scope + url = search_url(search["text"], number) + data = parse_search(fetch_page(url)) + observed_at = now() + signature = frozenset(str(ad["list_id"]) for ad in data["ads"]) + if signature and signature in seen_pages: + raise ValueError("Repeated result page; pagination did not advance") + seen_pages.add(signature) + filename = f"search-{index + 1:03d}-page-{number:04d}.jsonl.gz" + write_records(directory / filename, ({ + "schema_version": 2, "source": "leboncoin", "run_id": run_id, + "record_type": "search", "observed_at": observed_at, + "family": search["family"], "search_text": search["text"], + "search_url": url, "page": number, "payload": minimize(ad), + } for ad in data["ads"])) + scope["pages"].append({ + "page": number, "file": filename, "observed_at": observed_at, + "reported_total": data["total"], "reported_max_pages": data["max_pages"], + "records": len(data["ads"]), + }) + scope["records"] += len(data["ads"]) + if number >= data["max_pages"]: + scope["status"] = "complete" + elif number == max_pages: + scope["status"] = "partial" + scope["reason"] = "page_limit" + + details = manifest["details"]["listings"] + if not search_only: + for ad in data["ads"]: + details.setdefault(str(ad["list_id"]), {"status": "pending"}) + # Finalize discovery before visiting details so enrichment cannot lose it. + write_json(path, manifest) + if not search_only: + for ad in data["ads"]: + identifier = str(ad["list_id"]) + item = details[identifier] + if item["status"] != "pending": + continue + if max_details is not None and detail_attempts >= max_details: + item.update(status="not_attempted", reason="detail_limit") + continue + target = item + # Construct a same-source URL rather than navigate untrusted payload URLs. + detail_url = f"https://www.leboncoin.fr/ad/voitures/{identifier}" + item["url"] = detail_url + try: + detail = parse_detail(fetch_page(detail_url, item), identifier) + except SourceAccessError as exc: + if exc.status not in (404, 410): + raise + item.update(status="unavailable", http_status=exc.status) + write_json(path, manifest) + continue + detail_time = now() + detail_file = f"detail-{identifier}.jsonl.gz" + write_records(directory / detail_file, [{ + "schema_version": 2, "source": "leboncoin", "run_id": run_id, + "record_type": "detail", "observed_at": detail_time, + "listing_url": detail_url, "payload": minimize(detail), + }]) + item.update(status="complete", file=detail_file, observed_at=detail_time) + write_json(path, manifest) + if scope["status"] == "complete": + break + write_json(path, manifest) + except (Exception, KeyboardInterrupt) as exc: + manifest["error_type"] = type(exc).__name__ + if target is not None: + if target["status"] == "pending": + target["status"] = "not_attempted" + else: + target["status"] = "partial" if target.get("pages") else "failed" + # Exception bodies may contain page contents or personal information. + target["reason"] = type(exc).__name__ + if isinstance(exc, SourceAccessError): + target["http_status"] = exc.status + finally: + for scope in manifest["searches"]: + if scope["status"] == "running": + scope["status"] = "partial" if scope["pages"] else "failed" + details = manifest["details"] + if details["enabled"]: + for item in details["listings"].values(): + if item["status"] == "pending": + item.update(status="not_attempted", reason="run_stopped") + details["status"] = "complete" if all( + item["status"] == "complete" for item in details["listings"].values() + ) else "partial" + if not details["listings"] and "error_type" in manifest: + details["status"] = "not_attempted" + complete = ( + "error_type" not in manifest + and len(manifest["searches"]) == len(searches) + and all(s["status"] == "complete" for s in manifest["searches"]) + and details["status"] in ("complete", "disabled") + ) + manifest["status"] = "complete" if complete else ( + "partial" if any(s["pages"] for s in manifest["searches"]) else "failed" + ) + details["attempted"] = detail_attempts + manifest["finished_at"] = now() + write_json(path, manifest) + return path, manifest["status"] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=Path("searches.toml")) + parser.add_argument("--output", type=Path, default=Path("data/raw/leboncoin")) + parser.add_argument("--family", help="Collect only this configured family") + parser.add_argument("--max-pages", type=int, default=10) + parser.add_argument("--delay", type=float, default=5) + parser.add_argument("--headed", action="store_true") + parser.add_argument("--search-only", action="store_true", help="Skip individual listing pages") + parser.add_argument("--max-details", type=int, help="Maximum unique detail requests per run") + args = parser.parse_args() + if args.max_pages < 1 or not math.isfinite(args.delay) or args.delay < 1: + parser.error("max-pages must be positive and delay must be finite and >= 1 second") + if args.max_details is not None and args.max_details < 1: + parser.error("max-details must be positive") + with args.config.open("rb") as stream: + searches = tomllib.load(stream).get("searches", []) + if not isinstance(searches, list) or not searches or any( + not isinstance(s, dict) or set(s) != {"family", "text"} + or any(not isinstance(v, str) or not v.strip() for v in s.values()) + for s in searches + ): + parser.error("config needs searches with nonempty family and text strings") + if args.family: + searches = [s for s in searches if s["family"] == args.family] + if not searches: + parser.error("no matching searches") + + from playwright.sync_api import sync_playwright + + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=not args.headed) + try: + context = browser.new_context(locale="fr-FR") + page = context.new_page() + + def fetch(url): + response = page.goto(url, wait_until="domcontentloaded", timeout=60_000) + if response is None or response.status != 200: + raise SourceAccessError(response.status if response else None) + return page.locator("script#__NEXT_DATA__").text_content(timeout=15_000) + + path, status = collect( + fetch, searches, args.output, args.max_pages, args.delay, + args.search_only, args.max_details, + ) + finally: + browser.close() + print(f"{status}: {path}") + return 0 if status == "complete" else 1 diff --git a/src/car_traker/database.py b/src/car_traker/database.py new file mode 100644 index 0000000..be4b933 --- /dev/null +++ b/src/car_traker/database.py @@ -0,0 +1,407 @@ +"""Migrate PostgreSQL and replay finalized raw runs into private silver tables.""" + +import argparse +import gzip +import hashlib +from importlib.resources import files +from decimal import Decimal +import os +from pathlib import Path +import sys +from datetime import datetime +from uuid import UUID + +import psycopg +import simplejson as json +from psycopg import sql +from psycopg.types.json import Jsonb + +from .transform import clean_observation + + +TRANSFORM_VERSION = 1 + + +class InputError(ValueError): + """An input contract or integrity error, with no source data in its message.""" + + +def timestamp(value): + if not isinstance(value, str): + raise InputError("invalid_timestamp") + try: + parsed = datetime.fromisoformat(value) + if parsed.utcoffset() is None: + raise ValueError() + return parsed + except ValueError as exc: + raise InputError("invalid_timestamp") from exc + + +def load_json(text): + try: + value = json.loads(text, use_decimal=True) + # PostgreSQL JSONB cannot represent NUL, unpaired surrogates, or infinity. + def validate(item): + if isinstance(item, str): + item.encode("utf-8") + if "\0" in item: + raise ValueError() + elif isinstance(item, Decimal): + if not item.is_finite() or item.adjusted() >= 131072 or item.as_tuple().exponent < -16383: + raise ValueError() + elif isinstance(item, dict): + for key, val in item.items(): + validate(key) + validate(val) + elif isinstance(item, list): + for val in item: + validate(val) + validate(value) + return value + except (ValueError, UnicodeError, RecursionError) as exc: + raise InputError("invalid_json") from exc + + +def nonnegative(value): + if type(value) is not int or value < 0: + raise InputError("invalid_count") + return value + + +def listing_id(value): + text = str(value) + if not text.isascii() or not text.isdigit() or int(text) < 1: + raise InputError("invalid_listing_id") + return text + + +def load_run(path): + """Resolve only manifest-listed files and verify their physical row counts.""" + manifest = load_json(path.read_text(encoding="utf-8")) + try: + if type(manifest["schema_version"]) is not int or manifest["schema_version"] not in (1, 2): + raise InputError("unsupported_raw_schema") + if manifest["source"] != "leboncoin": + raise InputError("unsupported_source") + UUID(manifest["run_id"]) + if manifest["status"] not in ("complete", "partial", "failed"): + raise InputError("run_not_finalized") + started, finished = timestamp(manifest["started_at"]), timestamp(manifest["finished_at"]) + if finished < started: + raise InputError("invalid_run_time_range") + planned, scopes = manifest["planned_searches"], manifest["searches"] + if not isinstance(planned, list) or not planned or not isinstance(scopes, list) or len(scopes) > len(planned): + raise InputError("invalid_search_coverage") + inputs, coverage = [], [] + for index, search in enumerate(planned): + if any(not isinstance(search[k], str) or not search[k].strip() for k in ("family", "text")): + raise InputError("invalid_search_definition") + scope = scopes[index] if index < len(scopes) else { + **search, "status": "not_attempted", "pages": [], "records": 0, + } + if any(scope[k] != search[k] for k in ("family", "text")): + raise InputError("search_scope_mismatch") + if scope["status"] not in ("complete", "partial", "failed", "not_attempted"): + raise InputError("invalid_search_status") + pages = scope["pages"] + if not isinstance(pages, list): + raise InputError("invalid_pages") + count = 0 + for number, page in enumerate(pages, 1): + if type(page["page"]) is not int or page["page"] != number: + raise InputError("nonsequential_pages") + count += nonnegative(page["records"]) + nonnegative(page["reported_total"]) + nonnegative(page["reported_max_pages"]) + inputs.append({ + "raw_file": page["file"], "record_type": "search", "count": page["records"], + "observed_at": timestamp(page["observed_at"]), "page": number, + "family": search["family"], "search_text": search["text"], + }) + if count != nonnegative(scope["records"]): + raise InputError("search_count_mismatch") + if scope["status"] == "complete" and ( + not pages or len(pages) < pages[-1]["reported_max_pages"] + ): + raise InputError("inconsistent_complete_search") + coverage.append({ + "search_index": index + 1, "family": search["family"], "search_text": search["text"], + "status": scope["status"], "reason": scope.get("reason"), + "pages_observed": len(pages), "records_reported": count, + "reported_total": pages[-1]["reported_total"] if pages else None, + "reported_max_pages": pages[-1]["reported_max_pages"] if pages else None, + }) + details = manifest["details"] if manifest["schema_version"] == 2 else { + "enabled": False, "status": "disabled", "listings": {}, + } + if type(details["enabled"]) is not bool or not isinstance(details["listings"], dict): + raise InputError("invalid_detail_coverage") + if details["status"] not in ("complete", "partial", "not_attempted", "disabled"): + raise InputError("invalid_detail_status") + if not details["enabled"] and (details["listings"] or details["status"] != "disabled"): + raise InputError("invalid_disabled_details") + if details["enabled"] and details["status"] == "disabled": + raise InputError("invalid_disabled_details") + for identifier, detail in details["listings"].items(): + listing_id(identifier) + if detail["status"] not in ("complete", "unavailable", "failed", "not_attempted"): + raise InputError("invalid_listing_detail_status") + if detail["status"] == "complete": + inputs.append({ + "raw_file": detail["file"], "record_type": "detail", "count": 1, + "observed_at": timestamp(detail["observed_at"]), "listing_id": identifier, + }) + elif "file" in detail: + raise InputError("unfinished_detail_file") + if details["status"] == "complete" and any( + d["status"] != "complete" for d in details["listings"].values() + ): + raise InputError("inconsistent_complete_details") + if manifest["status"] == "complete" and ( + any(c["status"] != "complete" for c in coverage) + or details["status"] not in ("complete", "disabled") + ): + raise InputError("inconsistent_complete_run") + names = set() + for item in inputs: + name = item["raw_file"] + if (not isinstance(name, str) or Path(name).name != name + or not name.endswith(".jsonl.gz") or name in names): + raise InputError("invalid_raw_file_reference") + names.add(name) + snapshot = (path.parent / name).resolve() + if snapshot.parent != path.parent.resolve(): + raise InputError("raw_file_outside_run") + if not started <= item["observed_at"] <= finished: + raise InputError("observation_outside_run") + compressed = snapshot.read_bytes() + item["sha256"] = hashlib.sha256(compressed).hexdigest() + content = gzip.decompress(compressed).decode("utf-8") + # JSONL uses LF, not Unicode line separators that may occur in descriptions. + item["lines"] = content.split("\n") if content else [] + if item["lines"] and item["lines"][-1] == "": + item["lines"].pop() + if len(item["lines"]) != item["count"]: + raise InputError("raw_file_count_mismatch") + if details["enabled"]: + discovered = set() + for item in inputs: + if item["record_type"] != "search": + continue + for line in item["lines"]: + try: + discovered.add(listing_id(load_json(line)["payload"]["list_id"])) + except (InputError, KeyError, TypeError): + # Invalid records are accounted for separately during import. + continue + if not discovered.issubset(details["listings"]): + raise InputError("missing_detail_coverage") + digest = hashlib.sha256(json.dumps(manifest, sort_keys=True).encode()).hexdigest() + return manifest, digest, coverage, inputs + except InputError: + raise + except (KeyError, TypeError, ValueError, AttributeError) as exc: + raise InputError("invalid_manifest") from exc + + +def prepare_record(line, item, manifest): + record = load_json(line) + try: + if type(record["schema_version"]) is not int or any( + record[k] != manifest[k] for k in ("run_id", "source", "schema_version") + ): + raise InputError("record_run_mismatch") + kind = record.get("record_type", "search") if manifest["schema_version"] == 1 else record["record_type"] + if kind != item["record_type"]: + raise InputError("record_type_mismatch") + observed = timestamp(record["observed_at"]) + if observed != item["observed_at"]: + raise InputError("record_time_mismatch") + payload = record["payload"] + if not isinstance(payload, dict): + raise InputError("invalid_payload") + identifier = listing_id(payload["list_id"]) + if kind == "search": + if type(record["page"]) is not int or any( + record[k] != item[k] for k in ("family", "search_text", "page") + ): + raise InputError("record_search_mismatch") + if not isinstance(record.get("search_url"), str): + raise InputError("invalid_search_url") + elif identifier != item["listing_id"]: + raise InputError("record_listing_mismatch") + clean, issues = clean_observation(payload, observed) + return { + "source": manifest["source"], "source_listing_id": identifier, + "record_type": kind, "observed_at": observed, + "discovery_family": record.get("family") if kind == "search" else None, + "search_text": record.get("search_text") if kind == "search" else None, + "search_url": record.get("search_url") if kind == "search" else None, + "listing_url": f"https://www.leboncoin.fr/ad/voitures/{identifier}", + **clean, "raw_payload": Jsonb(payload, dumps=json.dumps), "issues": Jsonb(issues), + } + except InputError: + raise + except (KeyError, TypeError, ValueError, AttributeError) as exc: + raise InputError("invalid_record") from exc + + +def insert(conn, table, row): + conn.execute(sql.SQL("INSERT INTO silver.{} ({}) VALUES ({})").format( + sql.Identifier(table), sql.SQL(", ").join(map(sql.Identifier, row)), + sql.SQL(", ").join(sql.Placeholder() for _ in row), + ), list(row.values())) + + +def migrate(conn): + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(739240116)") + conn.execute("CREATE SCHEMA IF NOT EXISTS silver") + conn.execute("REVOKE ALL ON SCHEMA silver FROM PUBLIC") + conn.execute("""CREATE TABLE IF NOT EXISTS silver.schema_migrations ( + name text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now() + )""") + applied = dict(conn.execute("SELECT name, sha256 FROM silver.schema_migrations").fetchall()) + migrations = sorted(files("car_traker").joinpath("migrations").iterdir(), key=lambda p: p.name) + known = {m.name for m in migrations if m.name.endswith(".sql")} + if set(applied) - known: + raise InputError("database_has_unknown_migrations") + for migration in migrations: + if not migration.name.endswith(".sql"): + continue + content = migration.read_text(encoding="utf-8") + digest = hashlib.sha256(content.encode()).hexdigest() + if migration.name in applied: + if applied[migration.name] != digest: + raise InputError("applied_migration_changed") + continue + conn.execute(content) + insert(conn, "schema_migrations", {"name": migration.name, "sha256": digest}) + + +def import_run(conn, path): + manifest, digest, coverage, inputs = load_run(Path(path)) + run_id = manifest["run_id"] + accepted, rejected, field_issues = 0, 0, 0 + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (run_id,)) + previous = conn.execute( + "SELECT manifest_sha256 FROM silver.collection_runs WHERE run_id = %s", (run_id,) + ).fetchone() + if previous: + hashes = dict(conn.execute( + "SELECT raw_file, sha256 FROM silver.raw_files WHERE run_id = %s", (run_id,) + ).fetchall()) + if previous[0] != digest or hashes != {i["raw_file"]: i["sha256"] for i in inputs}: + raise InputError("imported_run_changed") + rejected = conn.execute( + "SELECT count(*) FROM silver.rejected_records WHERE run_id = %s", (run_id,) + ).fetchone()[0] + return {"run_id": run_id, "status": "already_imported", "rejected": rejected} + insert(conn, "collection_runs", { + "run_id": run_id, "source": manifest["source"], "raw_schema_version": manifest["schema_version"], + "source_status": manifest["status"], "started_at": timestamp(manifest["started_at"]), + "finished_at": timestamp(manifest["finished_at"]), "manifest": Jsonb(manifest, dumps=json.dumps), + "manifest_sha256": digest, "transform_version": TRANSFORM_VERSION, + }) + for scope in coverage: + insert(conn, "search_coverage", {"run_id": run_id, **scope}) + for identifier, detail in manifest.get("details", {}).get("listings", {}).items(): + insert(conn, "detail_coverage", { + "run_id": run_id, "source_listing_id": identifier, "status": detail["status"], + "reason": detail.get("reason"), "http_status": detail.get("http_status"), + }) + for item in inputs: + rows, rejects = [], [] + for number, line in enumerate(item["lines"], 1): + provenance = {"run_id": run_id, "raw_file": item["raw_file"], "raw_line": number} + try: + rows.append({**provenance, **prepare_record(line, item, manifest)}) + except InputError as exc: + rejects.append({**provenance, "code": str(exc), + "input_sha256": hashlib.sha256(line.encode()).hexdigest()}) + insert(conn, "raw_files", { + "run_id": run_id, "raw_file": item["raw_file"], "record_type": item["record_type"], + "sha256": item["sha256"], "expected_records": item["count"], + "accepted_records": len(rows), "rejected_records": len(rejects), + }) + for row in rows: + conn.execute("""INSERT INTO silver.listings (source, source_listing_id) + VALUES (%s, %s) ON CONFLICT DO NOTHING""", (row["source"], row["source_listing_id"])) + insert(conn, "observations", row) + field_issues += len(row["issues"].obj) + for row in rejects: + insert(conn, "rejected_records", row) + accepted += len(rows) + rejected += len(rejects) + return {"run_id": run_id, "status": "imported", "source_status": manifest["status"], + "accepted": accepted, "rejected": rejected, "field_issues": field_issues} + + +def database_error_hint(exc): + """Classify errors without logging server messages, SQL, or connection secrets.""" + message = str(exc).lower() + if "no pg_hba.conf entry" in message: + return ( + "PostgreSQL is reachable but pg_hba.conf does not allow this client/database/role. " + "Add a narrowly scoped rule on the homelab for the client IP seen by PostgreSQL, " + "then reload its configuration. See docs/POSTGRESQL.md#connection-troubleshooting." + ) + if "no password supplied" in message: + return "No matching password was supplied. Check PGPASSFILE, its 0600 permissions, and its host/port/database/user entry." + if "password authentication failed" in message or exc.sqlstate == "28P01": + return "PostgreSQL rejected authentication. Check the configured role and password file; do not share the password." + if "could not translate host name" in message or "name or service not known" in message: + return "The database hostname could not be resolved. Check PGHOST and your DNS/VPN connection." + if "connection refused" in message: + return "The database connection was refused. Check the host/port, PostgreSQL service, listen_addresses, and firewall." + if any(marker in message for marker in ("timeout expired", "connection timed out", "no route to host", "network is unreachable")): + return "The database could not be reached. Check the laptop-to-homelab route/VPN, port, and firewall." + if any(marker in message for marker in ("certificate verify failed", "root certificate file", "does not match host name")): + return "TLS certificate verification failed. Check PGSSLROOTCERT and the hostname against the server certificate; do not disable verification." + if "server does not support ssl" in message: + return "The requested TLS connection is unavailable on this server. Configure server TLS or an appropriate secure tunnel." + if exc.sqlstate == "3D000" or ("database" in message and "does not exist" in message): + return "The configured database does not exist. Verify PGDATABASE and provision it on the homelab if needed." + if exc.sqlstate == "42501": + return "The database role lacks required privileges. Use the dedicated database owner for migrations/imports, not a superuser." + if exc.sqlstate in ("42P01", "3F000"): + return "A required database table/schema is missing. Run car-traker-db migrate against the same database first." + return "Check the connection configuration and PostgreSQL server logs. Raw error details are withheld to protect credentials and source data." + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("check", help="Verify the connection with SELECT 1; no migrations or writes") + commands.add_parser("migrate", help="Apply checksum-verified SQL migrations") + importer = commands.add_parser("import", help="Import one finalized manifest atomically") + importer.add_argument("manifest", type=Path) + args = parser.parse_args() + if not any(os.environ.get(key) for key in ("DATABASE_URL", "PGSERVICE", "PGHOST", "PGHOSTADDR")): + parser.error("Set PGHOST for your homelab, PGSERVICE, or DATABASE_URL; no local database is assumed") + # Empty conninfo uses libpq PG* variables, service files and password files. + # Never accept a password in a CLI argument or print connection exceptions. + try: + with psycopg.connect(os.environ.get("DATABASE_URL", ""), connect_timeout=10, autocommit=True) as conn: + if args.command == "check": + conn.execute("SELECT 1") + print("Connection succeeded; no migrations or imports executed.") + elif args.command == "migrate": + migrate(conn) + print("Migrations applied.") + else: + result = import_run(conn, args.manifest) + print(json.dumps(result)) + return 2 if result.get("rejected", 0) else 0 + except InputError as exc: + print(f"Input error: {exc}", file=sys.stderr) + return 1 + except psycopg.Error as exc: + print(f"Database error ({type(exc).__name__}): {database_error_hint(exc)}", file=sys.stderr) + return 1 + except (OSError, EOFError, UnicodeError) as exc: + print(f"Database/import failed ({type(exc).__name__}); check configuration and input files.", file=sys.stderr) + return 1 + return 0 diff --git a/src/car_traker/migrations/001_silver.sql b/src/car_traker/migrations/001_silver.sql new file mode 100644 index 0000000..5af568d --- /dev/null +++ b/src/car_traker/migrations/001_silver.sql @@ -0,0 +1,163 @@ +CREATE TABLE silver.collection_runs ( + run_id uuid PRIMARY KEY, + source text NOT NULL, + raw_schema_version smallint NOT NULL CHECK (raw_schema_version IN (1, 2)), + source_status text NOT NULL CHECK (source_status IN ('complete', 'partial', 'failed')), + started_at timestamptz NOT NULL, + finished_at timestamptz NOT NULL CHECK (finished_at >= started_at), + manifest jsonb NOT NULL, + manifest_sha256 text NOT NULL, + transform_version integer NOT NULL, + imported_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE silver.raw_files ( + run_id uuid NOT NULL REFERENCES silver.collection_runs, + raw_file text NOT NULL, + record_type text NOT NULL CHECK (record_type IN ('search', 'detail')), + sha256 text NOT NULL, + expected_records integer NOT NULL CHECK (expected_records >= 0), + accepted_records integer NOT NULL CHECK (accepted_records >= 0), + rejected_records integer NOT NULL CHECK (rejected_records >= 0), + PRIMARY KEY (run_id, raw_file), + CHECK (accepted_records + rejected_records = expected_records) +); + +CREATE TABLE silver.search_coverage ( + run_id uuid NOT NULL REFERENCES silver.collection_runs, + search_index integer NOT NULL, + family text NOT NULL, + search_text text NOT NULL, + status text NOT NULL CHECK (status IN ('complete', 'partial', 'failed', 'not_attempted')), + reason text, + pages_observed integer NOT NULL, + records_reported integer NOT NULL, + reported_total integer, + reported_max_pages integer, + PRIMARY KEY (run_id, search_index) +); + +CREATE TABLE silver.detail_coverage ( + run_id uuid NOT NULL REFERENCES silver.collection_runs, + source_listing_id text NOT NULL, + status text NOT NULL CHECK (status IN ('complete', 'unavailable', 'failed', 'not_attempted')), + reason text, + http_status integer, + PRIMARY KEY (run_id, source_listing_id) +); + +CREATE TABLE silver.listings ( + source text NOT NULL, + source_listing_id text NOT NULL, + PRIMARY KEY (source, source_listing_id) +); + +CREATE TABLE silver.observations ( + observation_id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + run_id uuid NOT NULL, + raw_file text NOT NULL, + raw_line integer NOT NULL CHECK (raw_line > 0), + source text NOT NULL, + source_listing_id text NOT NULL, + record_type text NOT NULL CHECK (record_type IN ('search', 'detail')), + observed_at timestamptz NOT NULL, + discovery_family text, + search_text text, + search_url text, + listing_url text NOT NULL, + price_eur numeric(14, 2) CHECK (price_eur >= 0), + currency text CHECK (currency = 'EUR'), + make text, + model text, + model_year integer, + mileage_km integer CHECK (mileage_km >= 0), + fuel text, + gearbox text, + body_style text, + power_hp integer, + fiscal_power_cv integer, + source_published_at timestamptz, + source_updated_at timestamptz, + title text, + description text, + city text, + postcode text, + department_code text, + latitude double precision CHECK (latitude BETWEEN -90 AND 90), + longitude double precision CHECK (longitude BETWEEN -180 AND 180), + location_precision text, + raw_payload jsonb NOT NULL, + issues jsonb NOT NULL CHECK (jsonb_typeof(issues) = 'array'), + UNIQUE (run_id, raw_file, raw_line), + FOREIGN KEY (run_id, raw_file) REFERENCES silver.raw_files, + FOREIGN KEY (source, source_listing_id) REFERENCES silver.listings, + CHECK ((latitude IS NULL) = (longitude IS NULL)), + CHECK ((price_eur IS NULL) = (currency IS NULL)) +); + +CREATE INDEX observations_listing_time + ON silver.observations (source, source_listing_id, observed_at DESC); + +CREATE TABLE silver.rejected_records ( + run_id uuid NOT NULL, + raw_file text NOT NULL, + raw_line integer NOT NULL, + code text NOT NULL, + input_sha256 text NOT NULL, + PRIMARY KEY (run_id, raw_file, raw_line), + FOREIGN KEY (run_id, raw_file) REFERENCES silver.raw_files +); + +-- Private analytical views: text and raw payloads are deliberately not projected. +CREATE VIEW silver.v_listing_observations AS +SELECT o.observation_id, o.run_id, o.source, o.source_listing_id, + o.record_type, o.observed_at, o.discovery_family, o.listing_url, + o.price_eur, o.currency, o.make, o.model, o.model_year, o.mileage_km, + o.fuel, o.gearbox, o.body_style, o.power_hp, o.fiscal_power_cv, + o.source_published_at, o.source_updated_at, + o.city, o.postcode, o.department_code, o.latitude, o.longitude, + o.location_precision, r.source_status AS collection_status, + jsonb_array_length(o.issues) AS quality_issue_count +FROM silver.observations o JOIN silver.collection_runs r USING (run_id); + +-- Pick one coherent observation; do not backfill it with older detail fields. +CREATE VIEW silver.v_latest_listings AS +SELECT DISTINCT ON (source, source_listing_id) * +FROM silver.v_listing_observations +ORDER BY source, source_listing_id, observed_at DESC, + (record_type = 'detail') DESC, observation_id DESC; + +CREATE VIEW silver.v_price_history AS +SELECT observation_id, run_id, source, source_listing_id, record_type, + observed_at, price_eur, collection_status, + lag(price_eur) OVER ( + PARTITION BY source, source_listing_id + ORDER BY observed_at, (record_type = 'detail'), observation_id + ) AS previous_observed_price_eur +FROM silver.v_listing_observations +WHERE price_eur IS NOT NULL; + +CREATE VIEW silver.v_collection_coverage AS +SELECT r.run_id, r.source, r.started_at, r.finished_at, r.source_status, + coalesce(r.manifest #>> '{details,status}', 'disabled') AS detail_status, + (SELECT count(*) FROM silver.search_coverage s WHERE s.run_id = r.run_id) AS searches_planned, + (SELECT count(*) FROM silver.search_coverage s WHERE s.run_id = r.run_id AND s.status = 'complete') AS searches_complete, + (SELECT count(*) FROM silver.detail_coverage d WHERE d.run_id = r.run_id) AS details_discovered, + (SELECT count(*) FROM silver.detail_coverage d WHERE d.run_id = r.run_id AND d.status = 'complete') AS details_complete, + (SELECT count(*) FROM silver.observations o WHERE o.run_id = r.run_id) AS accepted_records, + (SELECT count(*) FROM silver.rejected_records q WHERE q.run_id = r.run_id) AS rejected_records, + (SELECT coalesce(sum(jsonb_array_length(o.issues)), 0) FROM silver.observations o WHERE o.run_id = r.run_id) AS field_issues +FROM silver.collection_runs r; + +CREATE VIEW silver.v_validation_issues AS +SELECT o.run_id, o.raw_file, o.raw_line, o.source, o.source_listing_id, + o.observation_id, 'field'::text AS issue_type, + issue ->> 'field' AS field, issue ->> 'code' AS code +FROM silver.observations o CROSS JOIN LATERAL jsonb_array_elements(o.issues) issue +UNION ALL +SELECT q.run_id, q.raw_file, q.raw_line, r.source, NULL, NULL, + 'rejected_record', '$record', q.code +FROM silver.rejected_records q JOIN silver.collection_runs r USING (run_id); + +REVOKE ALL ON ALL TABLES IN SCHEMA silver FROM PUBLIC; +REVOKE ALL ON ALL SEQUENCES IN SCHEMA silver FROM PUBLIC; diff --git a/src/car_traker/transform.py b/src/car_traker/transform.py new file mode 100644 index 0000000..1f09773 --- /dev/null +++ b/src/car_traker/transform.py @@ -0,0 +1,236 @@ +"""Conservative, offline normalization of a Leboncoin observation. + +Missing values remain None. Supplied values that cannot be represented safely +produce field/code issues, never copies of raw data. The input is not mutated. +""" + +from datetime import datetime +from decimal import Decimal, InvalidOperation, localcontext +import re + + +_NUMBER = re.compile(r"[+-]?[0-9]+(?:\.[0-9]+)?\Z") +_MAX_PRICE = Decimal("999999999999.99") # PostgreSQL numeric(14, 2). +_ATTRIBUTE_FIELDS = { + "make": "brand", + "model": "model", + "model_year": "regdate", + "mileage_km": "mileage", + "fuel": "fuel", + "gearbox": "gearbox", + "body_style": "vehicle_type", + "power_hp": "horse_power_din", + "fiscal_power_cv": "horsepower", +} + + +def clean_observation(payload: dict, observed_at: datetime) -> tuple[dict, list[dict]]: + """Return the fixed silver field contract and sanitized validation issues. + + Euro prices must be singleton arrays; cents are scalar integers. Prices + allow zero and at most two fractional digits (no rounding). Numeric + strings use plain decimal notation, without units or grouping punctuation. + Years are 1886..observation year + 1, mileage 0..10,000,000 km, DIN power + 1..3,000 hp and fiscal power 1..1,000 CV. Coordinates use global bounds. + Publication/update dates must carry their own timezone; observed_at is not + a substitute timestamp. Explicit nulls and empty price arrays are missing. + """ + result = dict.fromkeys(( + "price_eur", "currency", *_ATTRIBUTE_FIELDS, + "source_published_at", "source_updated_at", "title", "description", + "city", "postcode", "department_code", "latitude", "longitude", + "location_precision", + )) + issues = [] + + def issue(field, code): + entry = {"field": field, "code": code} + if entry not in issues: + issues.append(entry) + + def number(value, field, minimum, maximum, places=0): + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (str, int, float, Decimal)): + issue(field, "invalid_value") + return None + if isinstance(value, str) and not _NUMBER.fullmatch(value.strip()): + issue(field, "invalid_value") + return None + try: + parsed = Decimal(str(value).strip()) + except (InvalidOperation, ValueError): + issue(field, "invalid_value") + return None + if not parsed.is_finite(): + issue(field, "invalid_value") + return None + if not minimum <= parsed <= maximum: + issue(field, "out_of_range") + return None + if places is not None: + # Inspect digits rather than round under the active Decimal context. + digits = parsed.as_tuple().digits + exponent = parsed.as_tuple().exponent + trailing = 0 + for digit in reversed(digits): + if digit != 0: + break + trailing += 1 + if parsed and exponent + trailing < -places: + issue(field, "invalid_precision") + return None + return parsed + + def text(value, field, nonempty=False): + if value is None: + return None + if not isinstance(value, str) or (nonempty and not value.strip()): + issue(field, "invalid_value") + return None + return value + + price = payload.get("price") + price_bad = False + if isinstance(price, list): + if len(price) > 1: + issue("price_eur", "ambiguous_value") + price_bad = True + price = None + else: + price = price[0] if price else None + elif price is not None: + issue("price_eur", "invalid_value") + price_bad = True + price = None + euros = number(price, "price_eur", 0, _MAX_PRICE, 2) + cents_raw = payload.get("price_cents") + cents = number(cents_raw, "price_eur", 0, 99999999999999) + price_bad |= (price is not None and euros is None) or (cents_raw is not None and cents is None) + with localcontext() as context: + context.prec = 32 + if cents is not None: + cents = cents / 100 + if euros is not None and cents is not None and euros != cents: + issue("price_eur", "conflicting_prices") + price_bad = True + if not price_bad: + chosen = euros if euros is not None else cents + if chosen is not None: + result["price_eur"] = chosen.quantize(Decimal("0.01")) + result["currency"] = "EUR" + + bounds = { + "model_year": (1886, observed_at.year + 1), + "mileage_km": (0, 10_000_000), + "power_hp": (1, 3_000), + "fiscal_power_cv": (1, 1_000), + } + attributes = payload.get("attributes") + if attributes is None: + attributes = [] + if not isinstance(attributes, list): + issue("attributes", "invalid_value") + attributes = [] + for attribute in attributes: + if not isinstance(attribute, dict) or not isinstance(attribute.get("key"), str): + issue("attributes", "invalid_value") + + for field, key in _ATTRIBUTE_FIELDS.items(): + candidates = [] + signatures = [] + for attribute in attributes: + if not isinstance(attribute, dict) or attribute.get("key") != key: + continue + value = attribute.get("value") + values = attribute.get("values") + invalid = False + # model_all is a search ancestor only when a real scalar model + # is corroborated by the array. Do not generalize to other *_all codes. + if (key == "model" and isinstance(value, str) and value.strip() + and value != "model_all" and isinstance(values, list) + and value in values): + values = [entry for entry in values if entry != "model_all"] + if values is not None: + if not isinstance(values, list): + issue(field, "invalid_value") + invalid = True + elif len(values) > 1: + issue(field, "ambiguous_value") + invalid = True + elif values: + if value is not None and str(value) != str(values[0]): + issue(field, "conflicting_attributes") + invalid = True + elif value is None: + value = values[0] + label = attribute.get("value_label") + signatures.append((str(value) if value is not None else None, label)) + if field in bounds: + parsed = number(value, field, *bounds[field]) + candidate = int(parsed) if parsed is not None else None + elif field in ("fuel", "gearbox", "body_style"): + candidate = text(label, field, nonempty=True) if label is not None else None + if label is None and value is not None: + if isinstance(value, (int, float, Decimal)) and not isinstance(value, bool): + issue(field, "unmapped_code") + elif isinstance(value, str) and _NUMBER.fullmatch(value.strip()): + issue(field, "unmapped_code") + else: + candidate = text(value, field, nonempty=True) + else: + candidate = text(value, field, nonempty=True) + candidates.append(None if invalid else candidate) + if signatures and any(signature != signatures[0] for signature in signatures[1:]): + issue(field, "conflicting_attributes") + elif candidates: + result[field] = candidates[0] if all(c == candidates[0] for c in candidates) else None + elif field == "make" and (payload.get("attributes") is None + or isinstance(payload["attributes"], list)): + result[field] = text(payload.get("brand"), field, nonempty=True) + + for field, key in (("source_published_at", "first_publication_date"), + ("source_updated_at", "index_date")): + value = payload.get(key) + if value is None: + continue + if not isinstance(value, str): + issue(field, "invalid_value") + continue + try: + parsed = datetime.fromisoformat(value) + except ValueError: + issue(field, "invalid_value") + continue + if parsed.utcoffset() is None: + issue(field, "timezone_missing") + else: + result[field] = parsed + + result["title"] = text(payload.get("subject"), "title") + result["description"] = text(payload.get("body"), "description") + location = payload.get("location") + if location is None: + location = {} + if not isinstance(location, dict): + issue("location", "invalid_value") + location = {} + for field, key in (("city", "city"), ("postcode", "zipcode"), + ("department_code", "department_id")): + result[field] = text(location.get(key), field, nonempty=True) + latitude = number(location.get("lat"), "latitude", -90, 90, places=None) + longitude = number(location.get("lng"), "longitude", -180, 180, places=None) + if ((location.get("lat") is None) != (location.get("lng") is None) + or (latitude is None) != (longitude is None)): + issue("latitude", "incomplete_coordinates") + issue("longitude", "incomplete_coordinates") + elif latitude is not None: + result["latitude"], result["longitude"] = latitude, longitude + precision = location.get("origin_type") + if precision in (None, ""): + precision = location.get("type") + if precision in (None, ""): + # Observed source metadata; "address" does not prove an exact map point. + precision = location.get("source") if location.get("source") in ("city", "address") else None + result["location_precision"] = text(precision, "location_precision", nonempty=True) + return result, issues diff --git a/tests/test_collect.py b/tests/test_collect.py new file mode 100644 index 0000000..5012e58 --- /dev/null +++ b/tests/test_collect.py @@ -0,0 +1,260 @@ +import gzip +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from car_traker.collect import SourceAccessError, collect, minimize, parse_detail, parse_search, search_url + + +def payload(ids=(123,), total=1, pages=1): + return json.dumps({"props": {"pageProps": {"searchData": { + "ads": [{"list_id": i, "price": ["12000"], "body": ""} for i in ids], + "total": total, "max_pages": pages, + }}}}) + + +def detail_payload(identifier=123): + return json.dumps({"props": {"pageProps": {"ad": { + "list_id": identifier, "subject": "Synthetic Saab 900 Turbo", + "body": "Synthetic description: serviced in 2025.", "price": ["12500"], + "attributes": [{"key": "vehicle_new_option", "value": "original"}], + "location": {"lat": 43.6, "lng": 1.4, "city": "Test", "zipcode": "31000"}, + "owner": {"name": "Synthetic Seller"}, + }}}}) + + +class CollectorTests(unittest.TestCase): + def test_minimization_preserves_values_without_seller_data(self): + attribute = {"key": "mileage", "value": "unknown", "value_label": "Unknown"} + result = minimize({ + "list_id": 123, "price": ["12000"], "subject": "free text", + "body": "Synthetic description", "owner": {"name": "Synthetic Seller"}, + "images": {"urls": ["image"]}, + "attributes": [attribute, {"key": "profile_picture_url", "value": "private"}], + "location": {"department_id": "31", "lat": 43, "city": "Test"}, + }) + self.assertEqual(result, { + "list_id": 123, "price": ["12000"], "attributes": [attribute], + "subject": "free text", "body": "Synthetic description", + "location": {"department_id": "31", "lat": 43, "city": "Test"}, + }) + + def test_new_attributes_and_location_metadata_are_retained(self): + ad = parse_detail(detail_payload(), 123) + ad["location"]["origin_type"] = "city" + result = minimize(ad) + self.assertEqual(result["attributes"], ad["attributes"]) + self.assertEqual(result["location"], ad["location"]) + self.assertNotIn("owner", result) + + def test_missing_fields_stay_missing(self): + self.assertEqual(minimize({"list_id": 123}), {"list_id": 123}) + self.assertEqual(minimize({"list_id": 123, "attributes": [], "location": {}}), + {"list_id": 123, "attributes": [], "location": {}}) + + def test_known_sensitive_attributes_are_excluded(self): + keys = ["profile_picture_url", "seller_name", "contact_phone", "email", + "rating_score", "VIN", "vehicle_history_report_public_url"] + result = minimize({"attributes": [{"key": k, "value": "private"} for k in keys]}) + self.assertEqual(result["attributes"], []) + + def test_detail_payload_must_match_requested_listing(self): + for text in ("blocked", payload(), detail_payload(456), detail_payload("../123")): + with self.subTest(text=text), self.assertRaises(ValueError): + parse_detail(text, 123) + + def test_invalid_payload_is_not_an_empty_market(self): + for text in ("blocked", "{}", payload([], total=10), payload(pages=-1)): + with self.subTest(text=text), self.assertRaises(ValueError): + parse_search(text) + + def test_search_is_nationwide_and_encoded(self): + self.assertEqual(search_url("saab 900", 2), + "https://www.leboncoin.fr/recherche?category=2&text=saab+900&page=2") + + def run_collection(self, responses, **kwargs): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + iterator = iter(responses) + + def fetch(url): + result = next(iterator) + if isinstance(result, BaseException): + raise result + return result + + with patch("car_traker.collect.time.sleep"): + kwargs.setdefault("search_only", True) + path, status = collect(fetch, [{"family": "synthetic", "text": "test"}], + Path(temporary.name), **kwargs) + manifest = json.loads(path.read_text()) + self.assertEqual(status, manifest["status"]) + return path, manifest + + def test_complete_pagination_and_replay(self): + path, manifest = self.run_collection([ + payload([123], total=2, pages=2), payload([456], total=2, pages=2), + ]) + self.assertEqual(manifest["status"], "complete") + self.assertEqual(manifest["searches"][0]["records"], 2) + for page in manifest["searches"][0]["pages"]: + with gzip.open(path.parent / page["file"], "rt") as stream: + record = json.loads(stream.readline()) + self.assertEqual(record["run_id"], manifest["run_id"]) + self.assertEqual(record["payload"]["price"], ["12000"]) + self.assertEqual(record["payload"]["body"], "") + self.assertEqual(record["record_type"], "search") + self.assertEqual(record["schema_version"], 2) + + def test_page_limit_is_partial(self): + _, manifest = self.run_collection([payload(pages=2)], max_pages=1) + self.assertEqual(manifest["status"], "partial") + self.assertEqual(manifest["searches"][0]["reason"], "page_limit") + + def test_failure_preserves_previous_page(self): + path, manifest = self.run_collection([payload(pages=2), RuntimeError("private body")]) + self.assertEqual(manifest["status"], "partial") + self.assertEqual(len(list(path.parent.glob("*.jsonl.gz"))), 1) + self.assertNotIn("private body", path.read_text()) + + def test_block_is_failed_not_complete(self): + _, manifest = self.run_collection([SourceAccessError(403)]) + self.assertEqual(manifest["status"], "failed") + self.assertEqual(manifest["searches"][0]["http_status"], 403) + + def test_discovery_failure_does_not_claim_detail_completion(self): + _, manifest = self.run_collection([SourceAccessError(403)], search_only=False) + self.assertEqual(manifest["details"]["status"], "not_attempted") + + def test_discovered_empty_market_has_complete_detail_coverage(self): + _, manifest = self.run_collection([payload([], total=0, pages=0)], search_only=False) + self.assertEqual(manifest["status"], "complete") + self.assertEqual(manifest["details"]["status"], "complete") + + def test_failure_stops_remaining_searches(self): + with tempfile.TemporaryDirectory() as directory: + searches = [{"family": "synthetic", "text": "test"}, + {"family": "synthetic-2", "text": "test-2"}] + path, status = collect(lambda url: "blocked", searches, Path(directory)) + manifest = json.loads(path.read_text()) + self.assertEqual(status, "failed") + self.assertEqual(manifest["planned_searches"], searches) + self.assertEqual(len(manifest["searches"]), 1) + + def test_repeated_page_is_partial(self): + _, manifest = self.run_collection([payload(pages=2), payload(pages=2)]) + self.assertEqual(manifest["status"], "partial") + self.assertEqual(manifest["searches"][0]["records"], 1) + + def test_empty_search_is_complete(self): + _, manifest = self.run_collection([payload([], total=0, pages=0)]) + self.assertEqual(manifest["status"], "complete") + self.assertEqual(manifest["searches"][0]["records"], 0) + + def test_interrupt_is_recorded(self): + _, manifest = self.run_collection([KeyboardInterrupt()]) + self.assertEqual(manifest["status"], "failed") + self.assertEqual(manifest["searches"][0]["status"], "failed") + + def test_detail_observation_does_not_overwrite_search(self): + path, manifest = self.run_collection([payload(), detail_payload()], search_only=False) + self.assertEqual(manifest["status"], "complete") + self.assertEqual(manifest["details"]["status"], "complete") + item = manifest["details"]["listings"]["123"] + with gzip.open(path.parent / item["file"], "rt") as stream: + detail = json.loads(stream.readline()) + with gzip.open(path.parent / manifest["searches"][0]["pages"][0]["file"], "rt") as stream: + search = json.loads(stream.readline()) + self.assertEqual(detail["record_type"], "detail") + self.assertEqual(detail["payload"]["body"], "Synthetic description: serviced in 2025.") + self.assertEqual(detail["payload"]["location"]["lng"], 1.4) + self.assertEqual(detail["payload"]["price"], ["12500"]) + self.assertEqual(search["payload"]["price"], ["12000"]) + self.assertEqual(search["payload"]["body"], "") + self.assertNotIn("owner", detail["payload"]) + self.assertGreaterEqual(detail["observed_at"], search["observed_at"]) + + def test_detail_block_preserves_discovery_and_stops(self): + path, manifest = self.run_collection([ + payload([123, 456], total=2), SourceAccessError(403), + ], search_only=False) + self.assertEqual(manifest["status"], "partial") + self.assertEqual(manifest["searches"][0]["status"], "complete") + self.assertEqual(len(list(path.parent.glob("*.jsonl.gz"))), 1) + self.assertEqual(manifest["details"]["listings"]["123"]["http_status"], 403) + self.assertEqual(manifest["details"]["listings"]["456"]["status"], "not_attempted") + self.assertEqual(manifest["details"]["attempted"], 1) + + def test_unavailable_detail_does_not_stop_other_details(self): + for status in (404, 410): + with self.subTest(status=status): + _, manifest = self.run_collection([ + payload([123, 456], total=2), SourceAccessError(status), detail_payload(456), + ], search_only=False) + self.assertEqual(manifest["status"], "partial") + items = manifest["details"]["listings"] + self.assertEqual(items["123"]["status"], "unavailable") + self.assertEqual(items["456"]["status"], "complete") + + def test_malformed_detail_is_not_success(self): + _, manifest = self.run_collection([payload(), detail_payload(456)], search_only=False) + self.assertEqual(manifest["status"], "partial") + self.assertEqual(manifest["details"]["listings"]["123"]["reason"], "ValueError") + + def test_detail_limit_does_not_limit_search_pagination(self): + _, manifest = self.run_collection([ + payload([123], total=2, pages=2), detail_payload(), + payload([456], total=2, pages=2), + ], search_only=False, max_details=1) + self.assertEqual(manifest["status"], "partial") + self.assertEqual(manifest["searches"][0]["status"], "complete") + self.assertEqual(manifest["details"]["attempted"], 1) + self.assertEqual(manifest["details"]["listings"]["456"]["reason"], "detail_limit") + + def test_detail_interrupt_preserves_search(self): + _, manifest = self.run_collection([payload(), KeyboardInterrupt()], search_only=False) + self.assertEqual(manifest["status"], "partial") + self.assertEqual(manifest["searches"][0]["status"], "complete") + self.assertEqual(manifest["details"]["listings"]["123"]["status"], "failed") + + def test_interrupt_during_delay_does_not_count_as_detail_attempt(self): + with tempfile.TemporaryDirectory() as directory: + with patch("car_traker.collect.time.sleep", side_effect=KeyboardInterrupt()): + path, status = collect(lambda url: payload(), + [{"family": "synthetic", "text": "test"}], Path(directory)) + manifest = json.loads(path.read_text()) + self.assertEqual(status, "partial") + self.assertEqual(manifest["details"]["attempted"], 0) + self.assertEqual(manifest["details"]["listings"]["123"]["status"], "not_attempted") + + def test_overlapping_searches_fetch_detail_once_and_pace_all_requests(self): + with tempfile.TemporaryDirectory() as directory: + searches = [{"family": "synthetic", "text": "test"}, + {"family": "synthetic", "text": "test-2"}] + with patch("car_traker.collect.time.sleep") as sleep: + responses = iter([payload(), detail_payload(), payload()]) + urls = [] + + def fetch(url): + urls.append(url) + return next(responses) + + path, status = collect(fetch, searches, Path(directory), delay=7) + manifest = json.loads(path.read_text()) + self.assertEqual(status, "complete") + self.assertEqual(manifest["details"]["attempted"], 1) + self.assertEqual(len(urls), 3) + self.assertEqual(urls[1], "https://www.leboncoin.fr/ad/voitures/123") + self.assertEqual(sleep.call_count, 2) + sleep.assert_called_with(7) + + def test_search_only_explicitly_disables_detail_coverage(self): + _, manifest = self.run_collection([payload()]) + self.assertEqual(manifest["details"]["status"], "disabled") + self.assertEqual(manifest["details"]["attempted"], 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_database_cli.py b/tests/test_database_cli.py new file mode 100644 index 0000000..32beb7c --- /dev/null +++ b/tests/test_database_cli.py @@ -0,0 +1,69 @@ +import io +import unittest +from unittest.mock import MagicMock, patch + +import psycopg + +from car_traker.database import database_error_hint, main + + +class DatabaseCliTests(unittest.TestCase): + def test_safe_error_classification(self): + cases = ( + (psycopg.OperationalError('no pg_hba.conf entry for host "private-host"'), "pg_hba.conf"), + (psycopg.OperationalError("fe_sendauth: no password supplied"), "PGPASSFILE"), + (psycopg.OperationalError("password authentication failed"), "rejected authentication"), + (psycopg.OperationalError("could not translate host name"), "DNS/VPN"), + (psycopg.OperationalError("Connection refused"), "listen_addresses"), + (psycopg.OperationalError("timeout expired"), "route/VPN"), + (psycopg.OperationalError("certificate verify failed"), "PGSSLROOTCERT"), + (psycopg.OperationalError("server does not support SSL"), "server TLS"), + (psycopg.errors.InvalidCatalogName("private-db"), "PGDATABASE"), + (psycopg.errors.InsufficientPrivilege("private-table"), "privileges"), + (psycopg.errors.UndefinedTable("private-table"), "migrate"), + ) + for exc, expected in cases: + with self.subTest(error=type(exc).__name__, expected=expected): + hint = database_error_hint(exc) + self.assertIn(expected, hint) + self.assertNotIn("private-", hint) + + def test_unknown_error_does_not_expose_details(self): + hint = database_error_hint(psycopg.OperationalError("password=synthetic-secret source=private-text")) + self.assertNotIn("synthetic-secret", hint) + self.assertNotIn("private-text", hint) + self.assertIn("server logs", hint) + + def test_check_only_runs_select(self): + connection = MagicMock() + with patch.dict("os.environ", {"PGHOST": "synthetic-host"}, clear=True), patch( + "sys.argv", ["car-traker-db", "check"] + ), patch("car_traker.database.psycopg.connect") as connect, patch( + "sys.stdout", new_callable=io.StringIO + ) as output, patch("car_traker.database.migrate") as migrate: + connect.return_value.__enter__.return_value = connection + self.assertEqual(main(), 0) + connection.execute.assert_called_once_with("SELECT 1") + migrate.assert_not_called() + self.assertIn("no migrations or imports", output.getvalue()) + + def test_hba_failure_is_actionable_without_echoing_credentials(self): + exc = psycopg.OperationalError('no pg_hba.conf entry for host "private-host"; password=synthetic-secret') + with patch.dict("os.environ", {"PGHOST": "synthetic-host"}, clear=True), patch( + "sys.argv", ["car-traker-db", "check"] + ), patch("car_traker.database.psycopg.connect", side_effect=exc), patch( + "sys.stderr", new_callable=io.StringIO + ) as output: + self.assertEqual(main(), 1) + self.assertIn("pg_hba.conf", output.getvalue()) + self.assertNotIn("synthetic-secret", output.getvalue()) + self.assertNotIn("private-host", output.getvalue()) + + def test_no_implicit_local_connection(self): + with patch.dict("os.environ", {}, clear=True), patch( + "sys.argv", ["car-traker-db", "check"] + ), patch("sys.stderr", new_callable=io.StringIO), patch( + "car_traker.database.psycopg.connect" + ) as connect, self.assertRaises(SystemExit): + main() + connect.assert_not_called() diff --git a/tests/test_database_input.py b/tests/test_database_input.py new file mode 100644 index 0000000..31a14c7 --- /dev/null +++ b/tests/test_database_input.py @@ -0,0 +1,161 @@ +import gzip +import json +from decimal import Decimal +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest +from unittest.mock import patch + +from car_traker.collect import collect, write_json +from car_traker.database import InputError, load_json, load_run, prepare_record + + +class RawInputTests(unittest.TestCase): + def setUp(self): + temporary = TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + + def fixture(self): + source = json.dumps({"props": {"pageProps": {"searchData": { + "total": 1, "max_pages": 1, "ads": [{ + "list_id": 123, "price": [1000], "body": "Synthetic\u2028text\u2029only", + }], + }}}}) + with patch("car_traker.collect.time.sleep"): + path, _ = collect(lambda url: source, [{"family": "synthetic", "text": "test"}], + self.root, search_only=True) + return path, json.loads(path.read_text()) + + def test_collector_output_and_unicode_text_are_replayable(self): + path, _ = self.fixture() + manifest, digest, coverage, inputs = load_run(path) + self.assertEqual(len(digest), 64) + self.assertEqual(coverage[0]["records_reported"], 1) + result = prepare_record(inputs[0]["lines"][0], inputs[0], manifest) + self.assertEqual(result["description"], "Synthetic\u2028text\u2029only") + self.assertEqual(result["source_listing_id"], "123") + + def test_rejects_nonfinal_run(self): + path, manifest = self.fixture() + manifest["status"] = "running" + write_json(path, manifest) + with self.assertRaisesRegex(InputError, "run_not_finalized"): + load_run(path) + + def test_v2_requires_detail_coverage(self): + path, manifest = self.fixture() + del manifest["details"] + write_json(path, manifest) + with self.assertRaises(InputError): + load_run(path) + + def test_complete_search_must_cover_reported_pages(self): + path, manifest = self.fixture() + manifest["searches"][0]["pages"][0]["reported_max_pages"] = 3 + write_json(path, manifest) + with self.assertRaisesRegex(InputError, "inconsistent_complete_search"): + load_run(path) + + def test_discovered_ids_cannot_be_omitted_from_detail_coverage(self): + path, manifest = self.fixture() + for listings in ({}, {"456": {"status": "not_attempted"}}): + with self.subTest(listings=listings): + manifest["status"] = "partial" + manifest["details"] = {"enabled": True, "status": "partial", "listings": listings} + write_json(path, manifest) + with self.assertRaisesRegex(InputError, "missing_detail_coverage"): + load_run(path) + + def test_unknown_schema_is_rejected(self): + path, manifest = self.fixture() + for version in (True, 3, "2"): + with self.subTest(version=version): + manifest["schema_version"] = version + write_json(path, manifest) + with self.assertRaisesRegex(InputError, "unsupported_raw_schema"): + load_run(path) + + def test_rejects_traversal_and_unfinalized_filenames(self): + path, manifest = self.fixture() + for name in ("../snapshot.jsonl.gz", "/tmp/snapshot.jsonl.gz", "snapshot.jsonl.gz.tmp"): + with self.subTest(name=name): + manifest["searches"][0]["pages"][0]["file"] = name + write_json(path, manifest) + with self.assertRaisesRegex(InputError, "invalid_raw_file_reference"): + load_run(path) + + def test_rejects_symlink_escape(self): + path, manifest = self.fixture() + name = manifest["searches"][0]["pages"][0]["file"] + outside = self.root / "outside.jsonl.gz" + (path.parent / name).replace(outside) + (path.parent / name).symlink_to(outside) + with self.assertRaisesRegex(InputError, "raw_file_outside_run"): + load_run(path) + + def test_unreferenced_files_are_ignored(self): + path, _ = self.fixture() + (path.parent / "unreferenced.jsonl.gz").write_bytes(b"not gzip") + _, _, _, inputs = load_run(path) + self.assertEqual(len(inputs), 1) + + def test_rejects_missing_and_truncated_snapshots(self): + path, manifest = self.fixture() + snapshot = path.parent / manifest["searches"][0]["pages"][0]["file"] + snapshot.unlink() + with self.assertRaises(FileNotFoundError): + load_run(path) + snapshot.write_bytes(gzip.compress(b"")) + with self.assertRaisesRegex(InputError, "raw_file_count_mismatch"): + load_run(path) + + def test_provenance_mismatch_is_rejected(self): + path, _ = self.fixture() + manifest, _, _, inputs = load_run(path) + original = load_json(inputs[0]["lines"][0]) + for field, value in (("run_id", "wrong"), ("record_type", "detail"), + ("page", 2), ("family", "wrong"), + ("observed_at", "2020-01-01T00:00:00Z")): + with self.subTest(field=field), self.assertRaises(InputError): + prepare_record(json.dumps({**original, field: value}), inputs[0], manifest) + + def test_rejects_json_values_postgresql_cannot_store(self): + for text in ('{"x": NaN}', '{"x": 1e999999}', '{"x": "\\u0000"}', + '{"x": "\\ud800"}', '{"x": Infinity}', "not json"): + with self.subTest(text=text), self.assertRaisesRegex(InputError, "invalid_json"): + load_json(text) + + def test_numeric_json_tokens_keep_decimal_precision(self): + values = load_json('[1000.00000000000001, 1e-400]') + self.assertEqual(values, [Decimal('1000.00000000000001'), Decimal('1e-400')]) + path, _ = self.fixture() + manifest, _, _, inputs = load_run(path) + for token in ('1000.00000000000001', '1e-400'): + with self.subTest(token=token): + line = inputs[0]["lines"][0].replace('"price": [1000]', f'"price": [{token}]') + result = prepare_record(line, inputs[0], manifest) + self.assertIsNone(result["price_eur"]) + self.assertIn({"field": "price_eur", "code": "invalid_precision"}, result["issues"].obj) + self.assertEqual(result["raw_payload"].obj["price"][0], Decimal(token)) + + def test_missing_listing_identity_is_rejected_not_invented(self): + path, _ = self.fixture() + manifest, _, _, inputs = load_run(path) + record = load_json(inputs[0]["lines"][0]) + record["payload"] = {"subject": "Synthetic car"} + with self.assertRaisesRegex(InputError, "invalid_record"): + prepare_record(json.dumps(record), inputs[0], manifest) + + def test_invalid_vehicle_fields_are_issues_not_record_rejections(self): + path, _ = self.fixture() + manifest, _, _, inputs = load_run(path) + record = load_json(inputs[0]["lines"][0]) + record["payload"]["price"] = ["unknown"] + result = prepare_record(json.dumps(record), inputs[0], manifest) + self.assertIsNone(result["price_eur"]) + self.assertTrue(result["issues"].obj) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_database_integration.py b/tests/test_database_integration.py new file mode 100644 index 0000000..343e206 --- /dev/null +++ b/tests/test_database_integration.py @@ -0,0 +1,390 @@ +"""Opt-in tests against a disposable PostgreSQL database named *_test.""" + +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +from decimal import Decimal +import gzip +import hashlib +import io +import json +import os +from pathlib import Path +from tempfile import TemporaryDirectory +from threading import Barrier +import unittest +from unittest.mock import patch +from uuid import uuid4 + +import psycopg +from psycopg import sql +from psycopg.conninfo import conninfo_to_dict + +from car_traker import database +from car_traker.collect import search_url, write_json, write_records + + +@unittest.skipUnless(os.environ.get("TEST_DATABASE_URL"), "TEST_DATABASE_URL is not set") +class DatabaseIntegrationTests(unittest.TestCase): + tables = ( + "collection_runs", "raw_files", "search_coverage", "detail_coverage", + "listings", "observations", "rejected_records", + ) + + @classmethod + def connect(cls): + url = os.environ["TEST_DATABASE_URL"] + # Do not infer a target from libpq defaults or service configuration. + name = conninfo_to_dict(url).get("dbname", "") + if not name.endswith("_test"): + raise AssertionError("TEST_DATABASE_URL must explicitly name a database ending '_test'") + conn = psycopg.connect(url, autocommit=True, connect_timeout=10) + try: + if not conn.execute("SELECT current_database()").fetchone()[0].endswith("_test"): + raise AssertionError("Connected database name must end '_test'") + conn.execute("SET statement_timeout = '15s'") + conn.execute("SET lock_timeout = '10s'") + except BaseException: + conn.close() + raise + return conn + + @classmethod + def setUpClass(cls): + cls.conn = cls.connect() + cls.addClassCleanup(cls.conn.close) + database.migrate(cls.conn) + + def setUp(self): + self.conn.execute(sql.SQL("TRUNCATE {} RESTART IDENTITY CASCADE").format( + sql.SQL(", ").join(sql.Identifier("silver", table) for table in self.tables) + )) + temporary = TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.directory = Path(temporary.name) + + def fixture(self, *, version=2, pages=None, detail=None, day=1): + """Write raw envelopes directly; no collector or network client is run.""" + if pages is None: + pages = [[{"list_id": 123, "price": ["12000"]}]] + run_id = str(uuid4()) + directory = self.directory / run_id + directory.mkdir() + start = datetime(2026, 1, day, tzinfo=timezone.utc) + search = {"family": "synthetic", "text": "synthetic car"} + scope = {**search, "status": "complete", "pages": [], "records": 0} + manifest = { + "schema_version": version, "source": "leboncoin", "run_id": run_id, + "started_at": start.isoformat(), + "finished_at": (start + timedelta(hours=1)).isoformat(), + "status": "complete", "planned_searches": [search], "searches": [scope], + } + for number, payloads in enumerate(pages, 1): + observed = (start + timedelta(minutes=number)).isoformat() + filename = f"search-001-page-{number:04d}.jsonl.gz" + records = [{ + "schema_version": version, "source": "leboncoin", "run_id": run_id, + "observed_at": observed, "family": search["family"], + "search_text": search["text"], "search_url": search_url(search["text"], number), + "page": number, "payload": payload, + **({"record_type": "search"} if version == 2 else {}), + } for payload in payloads] + write_records(directory / filename, records) + scope["pages"].append({ + "page": number, "file": filename, "observed_at": observed, + "records": len(records), "reported_total": sum(map(len, pages)), + "reported_max_pages": len(pages), + }) + scope["records"] += len(records) + if version == 2: + manifest["details"] = {"enabled": False, "status": "disabled", "listings": {}} + if detail is not None: + identifier = str(detail["list_id"]) + observed = (start + timedelta(minutes=30)).isoformat() + filename = f"detail-{identifier}.jsonl.gz" + write_records(directory / filename, [{ + "schema_version": version, "source": "leboncoin", "run_id": run_id, + "record_type": "detail", "observed_at": observed, "payload": detail, + "listing_url": f"https://www.leboncoin.fr/ad/voitures/{identifier}", + }]) + manifest["details"] = { + "enabled": True, "status": "complete", "listings": { + identifier: {"status": "complete", "file": filename, "observed_at": observed}, + }, + } + path = directory / "manifest.json" + write_json(path, manifest) + return path, manifest + + def state(self): + # Compare complete persisted rows, not just counts, when asserting no mutation. + return {table: self.conn.execute(sql.SQL( + "SELECT to_jsonb(t) FROM silver.{} t ORDER BY to_jsonb(t)::text" + ).format(sql.Identifier(table))).fetchall() for table in (*self.tables, "schema_migrations")} + + def test_migrations_are_repeatable(self): + before = self.state() + self.assertTrue(before["schema_migrations"]) + database.migrate(self.conn) + database.migrate(self.conn) + self.assertEqual(self.state(), before) + + def test_applied_migration_checksum_mismatch_rolls_back(self): + before = self.state() + # The outer transaction also rolls back the deliberate checksum corruption. + with self.assertRaisesRegex(database.InputError, "^applied_migration_changed$"): + with self.conn.transaction(): + self.conn.execute("UPDATE silver.schema_migrations SET sha256 = %s", ("0" * 64,)) + database.migrate(self.conn) + self.assertEqual(self.state(), before) + database.migrate(self.conn) + + def test_import_and_replay_preserve_counts_and_rows(self): + path, manifest = self.fixture(detail={"list_id": 123, "price": ["12500"]}) + self.assertEqual(database.import_run(self.conn, path), { + "run_id": manifest["run_id"], "status": "imported", "source_status": "complete", + "accepted": 2, "rejected": 0, "field_issues": 0, + }) + before = self.state() + self.assertEqual({table: len(before[table]) for table in self.tables}, { + "collection_runs": 1, "raw_files": 2, "search_coverage": 1, + "detail_coverage": 1, "listings": 1, "observations": 2, "rejected_records": 0, + }) + self.assertEqual(database.import_run(self.conn, path), { + "run_id": manifest["run_id"], "status": "already_imported", "rejected": 0, + }) + self.assertEqual(self.state(), before) + self.assertEqual(self.conn.execute( + "SELECT expected_records, accepted_records, rejected_records FROM silver.raw_files" + ).fetchall(), [(1, 1, 0), (1, 1, 0)]) + + def test_v1_search_without_record_type_or_details(self): + path, _ = self.fixture(version=1) + result = database.import_run(self.conn, path) + self.assertEqual((result["accepted"], result["rejected"]), (1, 0)) + self.assertEqual(self.conn.execute( + "SELECT raw_schema_version, record_type, price_eur FROM silver.collection_runs " + "JOIN silver.observations USING (run_id)" + ).fetchone(), (1, "search", Decimal("12000"))) + self.assertEqual(self.conn.execute( + "SELECT detail_status FROM silver.v_collection_coverage" + ).fetchone(), ("disabled",)) + + def test_later_run_preserves_past_and_updates_views(self): + first, _ = self.fixture(detail={"list_id": 123, "price": ["12500"], "brand": "Synthetic"}) + database.import_run(self.conn, first) + original = self.state()["observations"] + later, manifest = self.fixture(day=2, pages=[[{"list_id": 123, "price": ["11000"]}]]) + database.import_run(self.conn, later) + for row in original: + self.assertIn(row, self.state()["observations"]) + self.assertEqual(self.conn.execute( + "SELECT run_id::text, price_eur, make FROM silver.v_latest_listings" + ).fetchall(), [(manifest["run_id"], Decimal("11000"), None)]) + self.assertEqual(self.conn.execute( + "SELECT price_eur, previous_observed_price_eur FROM silver.v_price_history " + "ORDER BY observed_at" + ).fetchall(), [(Decimal("12000"), None), (Decimal("12500"), Decimal("12000")), + (Decimal("11000"), Decimal("12500"))]) + self.assertEqual(len(self.state()["listings"]), 1) + + def test_search_and_detail_keep_distinct_prices_and_times(self): + path, manifest = self.fixture(detail={"list_id": 123, "price": ["12500"]}) + database.import_run(self.conn, path) + search = manifest["searches"][0]["pages"][0] + detail = manifest["details"]["listings"]["123"] + self.assertEqual(self.conn.execute( + "SELECT record_type, price_eur, observed_at, discovery_family " + "FROM silver.v_listing_observations ORDER BY observed_at" + ).fetchall(), [ + ("search", Decimal("12000"), datetime.fromisoformat(search["observed_at"]), "synthetic"), + ("detail", Decimal("12500"), datetime.fromisoformat(detail["observed_at"]), None), + ]) + self.assertEqual(self.conn.execute( + "SELECT record_type, price_eur, observed_at FROM silver.v_latest_listings" + ).fetchone(), ("detail", Decimal("12500"), datetime.fromisoformat(detail["observed_at"]))) + + def test_invalid_fields_are_null_with_sanitized_issues(self): + path, _ = self.fixture(pages=[[{ + "list_id": 123, "price": ["private-invalid-price"], + "attributes": [{"key": "mileage", "value": "-1"}], + "first_publication_date": "2026-01-01T00:00:00", + }]]) + result = database.import_run(self.conn, path) + self.assertEqual((result["accepted"], result["rejected"], result["field_issues"]), (1, 0, 3)) + self.assertEqual(self.conn.execute( + "SELECT price_eur, currency, mileage_km, source_published_at FROM silver.observations" + ).fetchone(), (None, None, None, None)) + self.assertEqual(set(self.conn.execute( + "SELECT field, code FROM silver.v_validation_issues" + ).fetchall()), {("price_eur", "invalid_value"), ("mileage_km", "out_of_range"), + ("source_published_at", "timezone_missing")}) + self.assertEqual(self.conn.execute( + "SELECT accepted_records, rejected_records, field_issues FROM silver.v_collection_coverage" + ).fetchone(), (1, 0, 3)) + + def test_malformed_record_is_rejected_with_private_safe_provenance(self): + secret = "synthetic-private-message-do-not-log" + path, manifest = self.fixture(pages=[[ + {"list_id": 123, "price": ["12000"]}, {"list_id": secret, "body": secret}, + ]]) + raw_file = manifest["searches"][0]["pages"][0]["file"] + line = gzip.decompress((path.parent / raw_file).read_bytes()).decode().splitlines()[1] + result = database.import_run(self.conn, path) + self.assertEqual((result["accepted"], result["rejected"]), (1, 1)) + self.assertEqual(self.conn.execute( + "SELECT run_id::text, raw_file, raw_line, code, input_sha256 FROM silver.rejected_records" + ).fetchone(), (manifest["run_id"], raw_file, 2, "invalid_listing_id", + hashlib.sha256(line.encode()).hexdigest())) + self.assertEqual(self.conn.execute( + "SELECT raw_file, raw_line, issue_type, field, code FROM silver.v_validation_issues" + ).fetchall(), [(raw_file, 2, "rejected_record", "$record", "invalid_listing_id")]) + self.assertNotIn(secret, repr(result)) + self.assertNotIn(secret, repr(self.state())) + self.assertEqual(self.conn.execute( + "SELECT expected_records, accepted_records, rejected_records FROM silver.raw_files" + ).fetchone(), (2, 1, 1)) + + def test_rejected_record_replay_keeps_nonzero_cli_exit(self): + path, _ = self.fixture(pages=[[{"list_id": "invalid"}]]) + with patch.dict(os.environ, {"DATABASE_URL": os.environ["TEST_DATABASE_URL"]}), patch( + "sys.argv", ["car-traker-db", "import", str(path)] + ), patch("sys.stdout", new_callable=io.StringIO) as output: + self.assertEqual(database.main(), 2) + self.assertEqual(database.main(), 2) + results = [json.loads(line) for line in output.getvalue().splitlines()] + self.assertEqual([r["rejected"] for r in results], [1, 1]) + self.assertEqual(results[1]["status"], "already_imported") + + def test_json_number_precision_survives_database_storage(self): + path, manifest = self.fixture(pages=[[ + {"list_id": 123, "price": ["EXCESS_PRECISION"]}, + {"list_id": 456, "price": ["UNDERFLOW"]}, + ]]) + snapshot = path.parent / manifest["searches"][0]["pages"][0]["file"] + content = gzip.decompress(snapshot.read_bytes()).decode() + content = content.replace('"EXCESS_PRECISION"', '1000.00000000000001').replace('"UNDERFLOW"', '1e-400') + snapshot.write_bytes(gzip.compress(content.encode())) + result = database.import_run(self.conn, path) + self.assertEqual((result["accepted"], result["rejected"], result["field_issues"]), (2, 0, 2)) + self.assertEqual(self.conn.execute( + "SELECT price_eur, (raw_payload #>> '{price,0}')::numeric " + "FROM silver.observations ORDER BY source_listing_id" + ).fetchall(), [(None, Decimal('1000.00000000000001')), (None, Decimal('1e-400'))]) + + def test_complete_run_cannot_omit_detail_coverage(self): + path, manifest = self.fixture() + manifest["details"] = {"enabled": True, "status": "complete", "listings": {}} + write_json(path, manifest) + before = self.state() + with self.assertRaisesRegex(database.InputError, "missing_detail_coverage"): + database.import_run(self.conn, path) + self.assertEqual(self.state(), before) + + def test_partial_and_failed_runs_preserve_unattempted_coverage(self): + for status in ("partial", "failed"): + with self.subTest(status=status): + path, manifest = self.fixture(pages=None if status == "partial" else []) + manifest["status"] = manifest["searches"][0]["status"] = status + manifest["planned_searches"].append({"family": "later", "text": "never attempted"}) + manifest["details"] = { + "enabled": True, "status": "not_attempted", + "listings": {"123": {"status": "not_attempted", "reason": "stopped"}}, + } + write_json(path, manifest) + result = database.import_run(self.conn, path) + self.assertEqual(result["source_status"], status) + self.assertEqual(self.conn.execute( + "SELECT status, pages_observed, records_reported FROM silver.search_coverage " + "WHERE run_id = %s ORDER BY search_index", (manifest["run_id"],) + ).fetchall(), [(status, int(status == "partial"), int(status == "partial")), + ("not_attempted", 0, 0)]) + self.assertEqual(self.conn.execute( + "SELECT status FROM silver.detail_coverage WHERE run_id = %s", (manifest["run_id"],) + ).fetchall(), [("not_attempted",)]) + self.assertEqual(self.conn.execute( + "SELECT source_status, searches_planned, searches_complete, details_complete " + "FROM silver.v_collection_coverage WHERE run_id = %s", (manifest["run_id"],) + ).fetchone(), (status, 2, 0, 0)) + + def test_changed_raw_file_replay_is_refused_without_mutation(self): + path, manifest = self.fixture() + database.import_run(self.conn, path) + before = self.state() + raw = path.parent / manifest["searches"][0]["pages"][0]["file"] + records = [json.loads(line) for line in gzip.decompress(raw.read_bytes()).decode().splitlines()] + records[0]["payload"]["price"] = ["9999"] + write_records(raw, records) + with self.assertRaisesRegex(database.InputError, "^imported_run_changed$"): + database.import_run(self.conn, path) + self.assertEqual(self.state(), before) + + def test_broken_later_file_aborts_entire_run(self): + baseline, _ = self.fixture() + database.import_run(self.conn, baseline) + before = self.state() + path, manifest = self.fixture(pages=[[{"list_id": 456}], [{"list_id": 789}]]) + raw = path.parent / manifest["searches"][0]["pages"][1]["file"] + write_json(raw, {"not": "gzip"}) + with self.assertRaises(gzip.BadGzipFile): + database.import_run(self.conn, path) + self.assertEqual(self.state(), before) + + def test_bad_later_file_count_aborts_entire_run(self): + before = self.state() + path, manifest = self.fixture(pages=[[{"list_id": 456}], [{"list_id": 789}]]) + write_records(path.parent / manifest["searches"][0]["pages"][1]["file"], []) + with self.assertRaisesRegex(database.InputError, "^raw_file_count_mismatch$"): + database.import_run(self.conn, path) + self.assertEqual(self.state(), before) + + def test_sql_failure_mid_import_rolls_back_all_writes(self): + baseline, _ = self.fixture() + database.import_run(self.conn, baseline) + before = self.state() + path, _ = self.fixture(pages=[[{"list_id": 456}], [{"list_id": 789}]]) + real_insert = database.insert + observations = 0 + + def failing_insert(conn, table, row): + nonlocal observations + if table == "observations": + observations += 1 + if observations == 2: + self.assertEqual(conn.execute( + "SELECT count(*) FROM silver.observations WHERE source_listing_id = '456'" + ).fetchone(), (1,)) + # Execute an invalid INSERT, putting PostgreSQL itself into failed state. + row = {**row, "raw_line": 0} + return real_insert(conn, table, row) + + with patch.object(database, "insert", side_effect=failing_insert): + with self.assertRaises(psycopg.errors.CheckViolation): + database.import_run(self.conn, path) + self.assertEqual(observations, 2) + self.assertEqual(self.state(), before) + self.assertEqual(database.import_run(self.conn, path)["accepted"], 2) + + def test_concurrent_same_run_import_is_idempotent(self): + path, manifest = self.fixture() + barrier = Barrier(2, timeout=10) + + def import_once(): + with self.connect() as conn: + barrier.wait() + return database.import_run(conn, path) + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(import_once) for _ in range(2)] + results = [future.result(timeout=30) for future in futures] + self.assertCountEqual([result["status"] for result in results], ["imported", "already_imported"]) + self.assertTrue(all(result["run_id"] == manifest["run_id"] for result in results)) + imported = next(result for result in results if result["status"] == "imported") + self.assertEqual((imported["accepted"], imported["rejected"]), (1, 0)) + self.assertEqual({table: len(rows) for table, rows in self.state().items() if table in self.tables}, { + "collection_runs": 1, "raw_files": 1, "search_coverage": 1, + "detail_coverage": 0, "listings": 1, "observations": 1, "rejected_records": 0, + }) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_transform.py b/tests/test_transform.py new file mode 100644 index 0000000..910eacc --- /dev/null +++ b/tests/test_transform.py @@ -0,0 +1,390 @@ +import copy +from datetime import datetime, timezone +from decimal import Decimal, localcontext +import unittest + +from car_traker.transform import clean_observation + + +OBSERVED_AT = datetime(2026, 9, 8, tzinfo=timezone.utc) +FIELDS = { + "price_eur", "currency", "make", "model", "model_year", "mileage_km", + "fuel", "gearbox", "body_style", "power_hp", "fiscal_power_cv", + "source_published_at", "source_updated_at", "title", "description", + "city", "postcode", "department_code", "latitude", "longitude", + "location_precision", +} + + +class CleanObservationTests(unittest.TestCase): + def clean(self, payload): + before = copy.deepcopy(payload) + result, issues = clean_observation(payload, OBSERVED_AT) + self.assertEqual(set(result), FIELDS) + self.assertEqual(payload, before) + for issue in issues: + self.assertEqual(set(issue), {"field", "code"}) + self.assertIsInstance(issue["field"], str) + self.assertIsInstance(issue["code"], str) + return result, issues + + def assert_issue(self, issues, field, code): + self.assertIn({"field": field, "code": code}, issues) + + def test_missing_values(self): + for payload in ({}, {"list_id": 123}, {"price": []}, + {"price": None, "price_cents": None, "attributes": None, + "location": None, "subject": None, "body": None}): + with self.subTest(payload=payload): + result, issues = self.clean(payload) + self.assertTrue(all(value is None for value in result.values())) + self.assertEqual(issues, []) + + def test_price_units(self): + for payload, expected in ( + ({"price": [1234]}, "1234.00"), + ({"price": ["1234.50"]}, "1234.50"), + ({"price_cents": 123450}, "1234.50"), + ({"price": [], "price_cents": "1"}, "0.01"), + ({"price": [12.5], "price_cents": "1250"}, "12.50"), + ({"price": [0], "price_cents": 0}, "0.00"), + ({"price": ["999999999999.99"], "price_cents": 99999999999999}, + "999999999999.99"), + ({"price": [Decimal("12.34000")]}, "12.34"), + ): + with self.subTest(payload=payload): + result, issues = self.clean(payload) + self.assertEqual(result["price_eur"], Decimal(expected)) + self.assertIsInstance(result["price_eur"], Decimal) + self.assertEqual(result["price_eur"].as_tuple().exponent, -2) + self.assertEqual(result["currency"], "EUR") + self.assertEqual(issues, []) + + def test_bad_price_does_not_fall_back_to_other_unit(self): + cases = [ + ({"price": [10], "price_cents": 10}, "conflicting_prices"), + ({"price": [10, 10], "price_cents": 1000}, "ambiguous_value"), + ({"price": [10], "price_cents": True}, "invalid_value"), + ({"price": ["10 euros"], "price_cents": 1000}, "invalid_value"), + ({"price": "10", "price_cents": 1000}, "invalid_value"), + ({"price": 10}, "invalid_value"), + ] + for payload, code in cases: + with self.subTest(payload=payload): + result, issues = self.clean(payload) + self.assertIsNone(result["price_eur"]) + self.assertIsNone(result["currency"]) + self.assert_issue(issues, "price_eur", code) + + def test_invalid_prices(self): + for value in (True, False, float("inf"), float("nan"), Decimal("NaN"), + Decimal("-Infinity"), -1, "-0.01", "1000000000000", + "1.001", "1.000000000000000000000000000001", "1,50", + "1 000", "1.000,00", "100 EUR", "1e3", "", {}, [[1]]): + with self.subTest(value=value): + result, issues = self.clean({"price": [value]}) + self.assertIsNone(result["price_eur"]) + self.assertIsNone(result["currency"]) + self.assertTrue(issues) + for value in (1.5, "1.01", -1, 100000000000000, True, "NaN", []): + with self.subTest(cents=value): + result, issues = self.clean({"price_cents": value}) + self.assertIsNone(result["price_eur"]) + self.assertTrue(issues) + + def test_decimal_context_does_not_round_prices(self): + with localcontext() as context: + context.prec = 4 + result, issues = self.clean({"price_cents": 99999999999999}) + self.assertEqual(result["price_eur"], Decimal("999999999999.99")) + self.assertEqual(issues, []) + result, issues = self.clean({"price": [Decimal("1.000000000000000000000001")]}) + self.assertIsNone(result["price_eur"]) + self.assert_issue(issues, "price_eur", "invalid_precision") + + def test_source_attributes_and_labels(self): + attributes = [ + {"key": "brand", "value": "BMW", "value_label": "Not the source brand"}, + {"key": "model", "values": ["Serie 3"], "value_label": "E30 325i"}, + {"key": "regdate", "value": "1990"}, + {"key": "mileage", "value": "150000"}, + {"key": "horse_power_din", "value": "170"}, + {"key": "horsepower", "value": "10"}, + {"key": "fuel", "value": "1", "value_label": "Essence"}, + {"key": "gearbox", "value": "2", "value_label": "Automatique"}, + {"key": "vehicle_type", "values": ["3"], "value_label": "Coupe"}, + {"key": "search_label", "value": "E30 generation sport variant"}, + ] + result, issues = self.clean({"attributes": attributes, "brand": "ignored"}) + expected = {"make": "BMW", "model": "Serie 3", "model_year": 1990, + "mileage_km": 150000, "power_hp": 170, "fiscal_power_cv": 10, + "fuel": "Essence", "gearbox": "Automatique", "body_style": "Coupe"} + for field, value in expected.items(): + self.assertEqual(result[field], value) + self.assertIs(type(result[field]), type(value)) + self.assertEqual(issues, []) + + def test_observed_model_taxonomy_and_fiscal_power(self): + for values in (["900", "model_all"], ["model_all", "900"]): + with self.subTest(values=values): + result, issues = self.clean({"brand": "SAAB", "attributes": [ + {"key": "model", "value": "900", "values": values}, + {"key": "u_car_model", "value": "SAAB_900", + "values": ["SAAB_900", "SAAB_model_all"]}, + {"key": "horsepower", "value": "8"}, + {"key": "horse_power_din", "value": "145"}, + ]}) + self.assertEqual(result["make"], "SAAB") + self.assertEqual(result["model"], "900") + self.assertEqual(result["fiscal_power_cv"], 8) + self.assertEqual(result["power_hp"], 145) + self.assertEqual(issues, []) + result, issues = self.clean({"attributes": [ + {"key": "horse_power", "value": "8"}, + {"key": "u_car_model", "value": "SAAB_900", + "values": ["SAAB_900", "SAAB_model_all"]}, + ]}) + self.assertIsNone(result["fiscal_power_cv"]) + self.assertIsNone(result["model"]) + self.assertEqual(issues, []) + + def test_model_taxonomy_does_not_hide_conflicts(self): + for value, values in ( + ("900", ["900", "9000", "model_all"]), + ("900", ["9000", "model_all"]), + ("900", ["900", "unrelated_all"]), + ("900", ["900", "SAAB_model_all"]), + ("900", ["model_all"]), + (None, ["900", "model_all"]), + ("", ["900", "model_all"]), + (" ", ["900", "model_all"]), + ("900", ["900", {}, "model_all"]), + ): + with self.subTest(value=value, values=values): + result, issues = self.clean({"attributes": [ + {"key": "model", "value": value, "values": values}]}) + self.assertIsNone(result["model"]) + self.assertTrue(issues) + for second, expected in (("900", "900"), ("9000", None)): + result, issues = self.clean({"attributes": [ + {"key": "model", "value": "900", "values": ["900", "model_all"]}, + {"key": "model", "value": second}, + ]}) + self.assertEqual(result["model"], expected) + if expected is None: + self.assert_issue(issues, "model", "conflicting_attributes") + else: + self.assertEqual(issues, []) + result, issues = self.clean({"attributes": [ + {"key": "brand", "value": "SAAB", "values": ["SAAB", "model_all"]}]}) + self.assertIsNone(result["make"]) + self.assert_issue(issues, "make", "ambiguous_value") + + def test_top_level_brand_fallback_only_without_attribute(self): + for payload in ({"brand": "SAAB"}, {"brand": "SAAB", "attributes": None}, + {"brand": "SAAB", "attributes": []}, + {"brand": "SAAB", "attributes": [{"key": "model", "value": "900"}]}): + result, issues = self.clean(payload) + self.assertEqual(result["make"], "SAAB") + self.assertEqual(issues, []) + for attributes, expected in ( + ([{"key": "brand", "value": "BMW"}], "BMW"), + ([{"key": "brand", "value": "BMW"}, {"key": "brand", "value": "Audi"}], None), + ([{"key": "brand", "value": None}], None), + ([{"key": "brand", "value": ""}], None), + ([{"key": "brand", "value": True}], None), + ([{"key": "brand", "values": ["BMW", "Audi"]}], None), + ({"brand": "BMW"}, None), + ): + with self.subTest(attributes=attributes): + result, issues = self.clean({"brand": "SAAB", "attributes": attributes}) + self.assertEqual(result["make"], expected) + for value in (True, 123, {}, [], ""): + result, issues = self.clean({"brand": value}) + self.assertIsNone(result["make"]) + self.assert_issue(issues, "make", "invalid_value") + + def test_unknown_codes_and_text_categories(self): + for field, key in (("fuel", "fuel"), ("gearbox", "gearbox"), + ("body_style", "vehicle_type")): + for code in ("999", 999, "1.0"): + with self.subTest(field=field, code=code): + result, issues = self.clean({"attributes": [{"key": key, "value": code}]}) + self.assertIsNone(result[field]) + self.assert_issue(issues, field, "unmapped_code") + for attribute in ({"key": key, "value": "Source category"}, + {"key": key, "value_label": "Source category"}): + result, issues = self.clean({"attributes": [attribute]}) + self.assertEqual(result[field], "Source category") + self.assertEqual(issues, []) + + def test_numeric_attributes_bounds(self): + for field, key, low, high in ( + ("model_year", "regdate", 1886, 2027), + ("mileage_km", "mileage", 0, 10_000_000), + ("power_hp", "horse_power_din", 1, 3000), + ("fiscal_power_cv", "horsepower", 1, 1000), + ): + for value in (low, str(high), f"{low}.0"): + with self.subTest(field=field, value=value): + result, issues = self.clean({"attributes": [{"key": key, "value": value}]}) + self.assertEqual(result[field], int(Decimal(value))) + self.assertIs(type(result[field]), int) + self.assertEqual(issues, []) + for value in (low - 1, high + 1, True, False, 1.5, "1,000", "100 km", + float("inf"), "NaN", [], {}, "", "2026-01-01"): + with self.subTest(field=field, value=value): + result, issues = self.clean({"attributes": [{"key": key, "value": value}]}) + self.assertIsNone(result[field]) + self.assertTrue(issues) + + def test_duplicate_attributes(self): + for key, field, first, second in ( + ("brand", "make", "BMW", "Audi"), + ("model", "model", "Serie 3", "Serie 5"), + ("mileage", "mileage_km", "100", "200"), + ("fuel", "fuel", "1", "2"), + ): + with self.subTest(key=key): + result, issues = self.clean({"attributes": [ + {"key": key, "value": first}, {"key": key, "value": second}]}) + self.assertIsNone(result[field]) + self.assert_issue(issues, field, "conflicting_attributes") + result, issues = self.clean({"attributes": [ + {"key": "fuel", "value": "1", "value_label": "Essence"}, + {"key": "fuel", "value": "1", "value_label": "Diesel"}]}) + self.assertIsNone(result["fuel"]) + self.assert_issue(issues, "fuel", "conflicting_attributes") + result, issues = self.clean({"attributes": [ + {"key": "mileage", "value": "100"}, + {"key": "mileage", "values": ["100"]}]}) + self.assertEqual(result["mileage_km"], 100) + self.assertEqual(issues, []) + + def test_attribute_representations(self): + for attribute, code in ( + ({"key": "model", "values": ["a", "b"]}, "ambiguous_value"), + ({"key": "model", "value": "a", "values": ["b"]}, "conflicting_attributes"), + ({"key": "model", "values": "a"}, "invalid_value"), + ({"key": "model", "value": {}}, "invalid_value"), + ({"key": "model", "value": True}, "invalid_value"), + ({"key": "model", "value": ""}, "invalid_value"), + ): + with self.subTest(attribute=attribute): + result, issues = self.clean({"attributes": [attribute]}) + self.assertIsNone(result["model"]) + self.assert_issue(issues, "model", code) + result, issues = self.clean({"attributes": [{"key": "model", "values": []}]}) + self.assertIsNone(result["model"]) + self.assertEqual(issues, []) + + def test_text_is_unmodified_and_not_inferred(self): + title = " BMW E30 325i\n\t" + description = "First line\r\nContact private@example.test +33 123456789 " + result, issues = self.clean({"subject": title, "body": description}) + self.assertEqual(result["title"], title) + self.assertEqual(result["description"], description) + self.assertIsNone(result["make"]) + self.assertIsNone(result["model"]) + self.assertEqual(issues, []) + result, issues = self.clean({"subject": "", "body": ""}) + self.assertEqual(result["title"], "") + self.assertEqual(result["description"], "") + self.assertEqual(issues, []) + for value in (True, 123, [description], {"private": description}): + result, issues = self.clean({"subject": value, "body": value}) + self.assertIsNone(result["title"]) + self.assertIsNone(result["description"]) + self.assertNotIn(description, repr(issues)) + self.assert_issue(issues, "title", "invalid_value") + + def test_source_timestamps(self): + for field, key in (("source_published_at", "first_publication_date"), + ("source_updated_at", "index_date")): + for value in ("2026-08-01T10:20:30Z", "2026-08-01 10:20:30+02:00", + "2026-08-01T10:20:30.123456-04:00"): + result, issues = self.clean({key: value}) + self.assertEqual(result[field], datetime.fromisoformat(value)) + self.assertIsNotNone(result[field].utcoffset()) + self.assertEqual(issues, []) + for value in ("2026-08-01T10:20:30", "2026-08-01 10:20:30", "2026-08-01"): + result, issues = self.clean({key: value}) + self.assertIsNone(result[field]) + self.assert_issue(issues, field, "timezone_missing") + for value in ("", "2026-02-30T12:00:00Z", "yesterday", True, 123, []): + result, issues = self.clean({key: value}) + self.assertIsNone(result[field]) + self.assert_issue(issues, field, "invalid_value") + + def test_global_coordinates_and_source_precision(self): + for lat, lng in (("-21.1151", "55.5364"), ("4.9", "-52.3"), + (-90, -180), (90, 180), (0, 0)): + result, issues = self.clean({"location": { + "lat": lat, "lng": lng, "city": "Source city", "zipcode": "00123", + "department_id": "2A", "origin_type": "city", "type": "point", + "source": "provider"}}) + self.assertEqual(result["latitude"], Decimal(str(lat))) + self.assertEqual(result["longitude"], Decimal(str(lng))) + self.assertEqual(result["postcode"], "00123") + self.assertEqual(result["department_code"], "2A") + self.assertEqual(result["city"], "Source city") + self.assertEqual(result["location_precision"], "city") + self.assertEqual(issues, []) + for location, expected in (({"lat": 48, "lng": 2}, None), + ({"source": "provider"}, None), + ({"type": "approximate"}, "approximate"), + ({"origin_type": "department"}, "department")): + result, issues = self.clean({"location": location}) + self.assertEqual(result["location_precision"], expected) + self.assertEqual(issues, []) + + def test_invalid_and_incomplete_coordinates(self): + for key, field, valid, bad in (("lat", "latitude", 48, 91), + ("lng", "longitude", 2, 181)): + for value in (bad, -bad, True, float("inf"), "NaN", "48 N", {}, []): + location = {"lat": 48, "lng": 2, key: value} + with self.subTest(location=location): + result, issues = self.clean({"location": location}) + self.assertIsNone(result["latitude"]) + self.assertIsNone(result["longitude"]) + self.assertTrue(any(i["field"] == field for i in issues)) + for value in (valid, bad): + result, issues = self.clean({"location": {key: value}}) + self.assertIsNone(result["latitude"]) + self.assertIsNone(result["longitude"]) + self.assert_issue(issues, "latitude", "incomplete_coordinates") + self.assert_issue(issues, "longitude", "incomplete_coordinates") + + def test_observed_location_source_with_empty_type_fields(self): + for source in ("city", "address"): + result, issues = self.clean({"location": { + "origin_type": "", "type": "", "source": source, + }}) + self.assertEqual(result["location_precision"], source) + self.assertEqual(issues, []) + result, issues = self.clean({"location": {"origin_type": "", "type": ""}}) + self.assertIsNone(result["location_precision"]) + self.assertEqual(issues, []) + + def test_malformed_containers_and_location_text(self): + for value in (True, "private@example.test", 123, {}): + result, issues = self.clean({"attributes": value}) + self.assert_issue(issues, "attributes", "invalid_value") + for value in (True, "private@example.test", 123, []): + result, issues = self.clean({"location": value}) + self.assert_issue(issues, "location", "invalid_value") + result, issues = self.clean({"attributes": [None, True, {}, {"key": []}]}) + self.assert_issue(issues, "attributes", "invalid_value") + for key, field in (("city", "city"), ("zipcode", "postcode"), + ("department_id", "department_code"), + ("origin_type", "location_precision")): + for value in (True, 123, [], {}, ""): + if key == "origin_type" and value == "": + continue + result, issues = self.clean({"location": {key: value}}) + self.assertIsNone(result[field]) + self.assert_issue(issues, field, "invalid_value") + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..bb80ca6 --- /dev/null +++ b/uv.lock @@ -0,0 +1,228 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" + +[[package]] +name = "car-traker" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "playwright" }, + { name = "psycopg", extra = ["binary"] }, + { name = "simplejson" }, +] + +[package.dev-dependencies] +dev = [ + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "playwright", specifier = ">=1.58,<2" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, + { name = "simplejson", specifier = ">=3.20,<4" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "ruff", specifier = ">=0.14,<0.15" }] + +[[package]] +name = "greenlet" +version = "3.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/d8/7cc97c142388aef03f622e001c572c4f84e9252a439549d483f555771970/greenlet-3.5.5.tar.gz", hash = "sha256:adb4bae02e91a8e863e48b177e4014bdcac8a6b5e047ea1df687a61534b85e6c", size = 207585, upload-time = "2026-08-10T15:09:36.136Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/3d/8cef5f724ec0d4add2af8961d504535ec60c3cca9e464f6d03bdba29d85b/greenlet-3.5.5-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b79fd2a5bc099b5e744f34c4c9a58954a5f4cb7529fb4b6e8446057d61b6edaa", size = 294730, upload-time = "2026-08-10T13:27:51.206Z" }, + { url = "https://files.pythonhosted.org/packages/88/4b/8e7aa3f514273aecff30a16ab1bac09ff54cfc7e6860fdd8058c37ff2499/greenlet-3.5.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:634cf15a233a949136879dd388e25d3296e16f3f1e217d2456797b8579ebc6ed", size = 614536, upload-time = "2026-08-10T14:14:36.589Z" }, + { url = "https://files.pythonhosted.org/packages/85/48/4e95e9dd5a8a397dc6a6345dd7f1935113d0fca4f85e89d3976da9cd988d/greenlet-3.5.5-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:499adea519f748407fc6806d20eedabac2884fd73b9f38d81236e190ba20dfef", size = 626924, upload-time = "2026-08-10T14:27:27.048Z" }, + { url = "https://files.pythonhosted.org/packages/0e/84/eaa476d6bf3816828d0d70e80dcc36bf30a058233bd889e707e693f6e860/greenlet-3.5.5-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7278591501941bb2456af102bb9cd59aab48c6cfd6e2dd68fa1290bb0c49a42", size = 632726, upload-time = "2026-08-10T14:30:09.874Z" }, + { url = "https://files.pythonhosted.org/packages/89/5d/398a1c71fa7a277deeb376c999979de6786f08fc2d5747a0b9d6e11738dd/greenlet-3.5.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2eabb980975cba5b93a95f6f69287d05fc05ac955bfd6a320a7c083eeb52c0b0", size = 623906, upload-time = "2026-08-10T13:40:50.501Z" }, + { url = "https://files.pythonhosted.org/packages/d0/f2/0cc2849ede68579291e9c59b3ab6ec1958f98681cca5b14d8fc75bf674a4/greenlet-3.5.5-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:4dfc7c4470354e7b09184d1a3a985761053a2fd694ddb5b5c80242afc2c8c90b", size = 434966, upload-time = "2026-08-10T14:30:03.729Z" }, + { url = "https://files.pythonhosted.org/packages/04/1b/745450fc5ea9e0cb17d840d248f284db3363de736d362c7d2d883e3eadba/greenlet-3.5.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:03115c2e0a371999bf8ae616aa8d653f96641d4705c457aebaa187276e9f7537", size = 1581430, upload-time = "2026-08-10T14:15:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/d4/29/d51b296e3191bb15d3d81ec375af1909e4466c0f395d744ed475801798a9/greenlet-3.5.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4441153ffba21b90d3ca89fe3d31f5c093ae6c0bf0cfdfc98f54cde22f95b62e", size = 1645684, upload-time = "2026-08-10T13:40:32.133Z" }, + { url = "https://files.pythonhosted.org/packages/12/63/369f1a1625e64e9e31df3963c6044056e3fdfa3fa3fdba3c54ffefa6e987/greenlet-3.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:95c5b1f4b3a193f8a0c2de4bfdcb48d119f7f1063941f1de1f2168051b3e52dd", size = 324075, upload-time = "2026-08-10T13:26:58.974Z" }, + { url = "https://files.pythonhosted.org/packages/45/78/649cb5c09d4d81f6dd1444e75474a7206784743283a21d24171562ac4899/greenlet-3.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:1af90aa4bc129883b340cdd6957a3bc74f60528a4993bbd1f53aaebe1d9981cc", size = 308260, upload-time = "2026-08-10T13:27:50.795Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8c/080e881fa2be95ff1ddbd6994b2bab3b1a78df3b3fcab39306011764fcc7/greenlet-3.5.5-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d4a389a852e392a6366058651a20fa5ba40d979865aa81bea2ccbdc44805070d", size = 295309, upload-time = "2026-08-10T13:26:03.032Z" }, + { url = "https://files.pythonhosted.org/packages/25/cc/0ac614e6586c0e42d4cc281a5819150f4f43685744a4c5ff77139286409d/greenlet-3.5.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70b157cd319873e8b544ddc2de158f55bbd0a9b0218c8ce9332039801518e328", size = 661185, upload-time = "2026-08-10T14:14:37.867Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b9/6808725354be8ad305dfe5172377664fc9642d4fc043be246b3314cf4482/greenlet-3.5.5-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8bdfd1424abcf26832961e766570cae79efdb9599d709088c9cb6ef82b194926", size = 673419, upload-time = "2026-08-10T14:27:28.652Z" }, + { url = "https://files.pythonhosted.org/packages/eb/52/f005d579acde46c3d1cc3cab1c9f3d5708c8a3006a4120e8cf5da801afe9/greenlet-3.5.5-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d98ef6f92e67c6dbf299dbfd8facc1b0d2d9cedf91e325e73b3d0373fe4309d8", size = 677863, upload-time = "2026-08-10T14:30:11.663Z" }, + { url = "https://files.pythonhosted.org/packages/42/2e/40c509967da7f254680826a2fa0dd22138ec79946c70b97542d74cde8b43/greenlet-3.5.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:182de51c6b572a705f2fafaab2e783bcf7d2760940229dfe73086cbae037af3e", size = 670822, upload-time = "2026-08-10T13:40:51.833Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8a/a75f8a2bdcef3c358a3147cdc9db3aa83755f0a038f766ab0bedb66f512c/greenlet-3.5.5-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:159df1942d88e8f784cbb38d6f18bdb365cd11319cfbb3e89623de2b97892d53", size = 480554, upload-time = "2026-08-10T14:30:05.171Z" }, + { url = "https://files.pythonhosted.org/packages/2d/22/c3c2eee4a8fe191d6d1d183086c56133d646024e3d70bfd414829f64560b/greenlet-3.5.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8fec3f165dfe332e490c3247c0f6c23b0bfc45f06496ad7f00ddb00e3d35e4dc", size = 1628469, upload-time = "2026-08-10T14:15:08.11Z" }, + { url = "https://files.pythonhosted.org/packages/f7/87/25babd09b94cb1f03e71db815fde463f0262e40cfbd953d58a8d77311351/greenlet-3.5.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c6ce25fee6cabc8bf22cb8b52e642cbb821be5b9aec8094d07ff03378141b8e9", size = 1691952, upload-time = "2026-08-10T13:40:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3d/5cc9701117ea4dc0eb7bf1f4f9b7888a6e2e5277ddfae095805ace50f2b6/greenlet-3.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:7dffc5c859fe6059974df1e37d7923d654a83e2ae18fdd616994270e001115e1", size = 327458, upload-time = "2026-08-10T13:27:02.868Z" }, + { url = "https://files.pythonhosted.org/packages/a7/6b/594fa2de7fae7629168a404a4305d7d7e31a5742c50a801b1839543cb93d/greenlet-3.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:5e2afcfc4d4305dd715809b03da5cbe437c8984f61d8917751eb5fe4aefa3e07", size = 311146, upload-time = "2026-08-10T13:27:25.046Z" }, + { url = "https://files.pythonhosted.org/packages/24/e0/50cd600b469e5734c72709b6b1838b6bc63f307b573c772c3132d6ecfe92/greenlet-3.5.5-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:0e5a7de979d764aea1f5b6e95cf92b5b37741b9823702041f34b126e7f690277", size = 305471, upload-time = "2026-08-10T13:26:20.568Z" }, + { url = "https://files.pythonhosted.org/packages/75/a3/77acd66dfc6387b5219b2080806c0cabb73c10eb1bb44b413c40a62015ba/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fef01bd457f11fc158b130ca0027a3c365693280e8e231b65bdaf57999f39f5b", size = 672470, upload-time = "2026-08-10T14:14:39.058Z" }, + { url = "https://files.pythonhosted.org/packages/b9/71/0d178142dca3ec19f46fb2212ae73d30ad53b9d548dc64804086033a7089/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5173a72310725a74afc82c164f0e52cb8ad0de62f2bb623f24f6c0cc07d80272", size = 679973, upload-time = "2026-08-10T14:27:30.072Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ac/0d7887aa4bbfc9eba075cc428244dfc96f623478454d5ec81180d0d6bd5a/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5e9ec2e7c98e895fcea0c5cc57b2606cf86ece6d0a56578f3eb225e2af4f0387", size = 681587, upload-time = "2026-08-10T14:30:13.519Z" }, + { url = "https://files.pythonhosted.org/packages/6e/31/46eb8567302eaf787abf88d09df014e14ae3baf460af1b8b0efdbd3efcd5/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44f08341873200ba8a60a8bc14ace3d91f1754f7fa7bc66157714a8cd420a476", size = 676634, upload-time = "2026-08-10T13:40:53.004Z" }, + { url = "https://files.pythonhosted.org/packages/4f/18/8d58ba1c429b0383e3219a3d0e0bba241d0444d8ed05b73349953c7d7c7b/greenlet-3.5.5-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:102817506f6090b5176c746a82603341a549b40e5c3d5b72a4c672228a918c41", size = 510175, upload-time = "2026-08-10T14:30:07.047Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e9/b88bbf5b29970cb84172dc2c32aa3e5e579ceb94c808e81c826454138850/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d246c0db9a2513cd45f019ba178ea4d4d4705bd210ee465e2c15d76a1ab13874", size = 1637320, upload-time = "2026-08-10T14:15:09.317Z" }, + { url = "https://files.pythonhosted.org/packages/6d/8c/7631ed29cc6f0392f11830076e172ce4885e70b0bc2c1bce1731176d4b4e/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:72507285b5caa1d17904a3f7c322ca780823a54170a0e04ec3f37bcc60d4db71", size = 1697412, upload-time = "2026-08-10T13:40:34.924Z" }, + { url = "https://files.pythonhosted.org/packages/da/0f/f7dd935f9c4cb1be49098770587f54d8a78518e55c89bce86c4fb4109057/greenlet-3.5.5-cp314-cp314t-win_amd64.whl", hash = "sha256:7805655781fb8f28a55d05fe57ed61f5f10f1892fb587673e3bb5264f28041f0", size = 331514, upload-time = "2026-08-10T13:29:20.611Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e5/681b01f8fbc1b55232822f99e8f8afeb78a55a7c76a7bf9dbdc7ccb03a6d/greenlet-3.5.5-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:c0db80fcd5b8aece93f66c64f78a786bbb6b96c5fe63ef5a5a4581ecf8bab206", size = 295975, upload-time = "2026-08-10T13:28:45.985Z" }, + { url = "https://files.pythonhosted.org/packages/11/f2/69b488cd9e7267bf4b0fe8cdebf25d8d6df680d21bdf41150d23e23d6652/greenlet-3.5.5-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b241c32f912ada659808d68e308c568baf577eebf757d15471472de0c18cfad", size = 666823, upload-time = "2026-08-10T14:14:40.222Z" }, + { url = "https://files.pythonhosted.org/packages/84/d4/d5bc2fdebbdda0c94555925ba79948b8395d75a7f6a36cc85dce5bab9f11/greenlet-3.5.5-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ef6a08349401d8eaf3cb12688ac8557de95788556b8631ef17555a4a173022c0", size = 677613, upload-time = "2026-08-10T14:27:31.543Z" }, + { url = "https://files.pythonhosted.org/packages/65/53/4e13642efc4d7ad6554ecb2242a5be42666b2e1a067323e88dfc0124a04b/greenlet-3.5.5-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:37faa97daccb6d9f4c2141ce3118d023c3c5506864a7d8bdf726f665018c1f76", size = 681436, upload-time = "2026-08-10T14:30:14.839Z" }, + { url = "https://files.pythonhosted.org/packages/bd/93/542d8a3a90f3b35c6ad8bf7e56a03010287f2cafa289a5b7985b5207db39/greenlet-3.5.5-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2e3d061b8e13aec2f0441689b3c71b244a20e5d274a52cb0f7e31bd1d139552", size = 675930, upload-time = "2026-08-10T13:40:54.205Z" }, + { url = "https://files.pythonhosted.org/packages/cd/32/188447c9a468d6977d2989397226b0c6b65ab6f4cf943f931643328512fc/greenlet-3.5.5-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:b18007dc2473a7942fd157366b55f01da6fed7ce85318591005b419e0a439474", size = 487404, upload-time = "2026-08-10T14:30:08.903Z" }, + { url = "https://files.pythonhosted.org/packages/52/b5/89c9f2e8460d71101037d47a1feed11928615a5edd42370be290e0657eeb/greenlet-3.5.5-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9ab5f5b93655e77fe0d6c2dfd22b5eac751bb1f876d8ec21761b7c1fb9266007", size = 1633878, upload-time = "2026-08-10T14:15:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/b8/60/297de93f3b02ac78a5e04d32bb8bbe3080f4a73d8ed95016561463b70618/greenlet-3.5.5-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f0e5a21bd4452a88cf032fc43c4a5b307ab1380eacb63b5988f9c0317885e773", size = 1696597, upload-time = "2026-08-10T13:40:36.252Z" }, + { url = "https://files.pythonhosted.org/packages/18/25/54c6eaff4f337fb670215e89eb2d00d9499487b658e709d4b477be4a342e/greenlet-3.5.5-cp315-cp315-win_amd64.whl", hash = "sha256:469dbb0a78625642f4a626cfd0c6e8bccc0385b5e49189b6308bbe849ec88a8e", size = 327700, upload-time = "2026-08-10T13:28:06.752Z" }, + { url = "https://files.pythonhosted.org/packages/67/67/857e88a36301caa0e029870132c2478bd55d896630321432afab03a3115f/greenlet-3.5.5-cp315-cp315-win_arm64.whl", hash = "sha256:2d57406c3efd32d7a81e17a674314e8bd00792cdab49ea3228a49aa1bfb2e769", size = 311750, upload-time = "2026-08-10T13:34:08.815Z" }, + { url = "https://files.pythonhosted.org/packages/10/e2/3144c0a116067ac1e30457b0139a94d60d1d36a86e015de68e9ac87cb3bc/greenlet-3.5.5-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:68184dfcf50ccaa8e864770fe0633a7e27250ea9329f8192ef47ee9ecfd78e1c", size = 306387, upload-time = "2026-08-10T13:27:00.897Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a1/cb4223a7e9b9f43b8807e8eb212358bfe2dfaa174a9ea2889eb1714dcba2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ec0dc0e59dc9c61af5c47348365ccbbd7addfafe0a93b00336ff3da2907bdc6", size = 676472, upload-time = "2026-08-10T14:14:41.417Z" }, + { url = "https://files.pythonhosted.org/packages/9e/cd/a154b4498e5d8f12ada291cfb3b8d596eadde2177f5bf09a9be699d2a446/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e604f58e35833fc46ef20302bcb314dddbfd3fcf33a4f936216d51dd678d63ae", size = 684238, upload-time = "2026-08-10T14:27:32.946Z" }, + { url = "https://files.pythonhosted.org/packages/ce/f4/e450a68a152f819491d8c7df6a8254e761d87e6a78759268961f8c5bd4dd/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2888a3a38bc5ee5bb6c438372197152e815837e4fab7ed7a1f86ef18ffd58ad1", size = 686022, upload-time = "2026-08-10T14:30:15.96Z" }, + { url = "https://files.pythonhosted.org/packages/bf/bb/b0031d260c2968a3c87deebc51d80c64e499377f993aafe06ee3b7488cc2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:40239b5384f96da3963585cc6d7eaa9b56f8ae67e8d92cc82dd9e202fc847de3", size = 681246, upload-time = "2026-08-10T13:40:55.402Z" }, + { url = "https://files.pythonhosted.org/packages/18/23/17e63d6bf3b9c9b9dbea981b7f643a71f79603bdfb4f1c3a9cf353e22aed/greenlet-3.5.5-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:1e8d9391fe77f15649589a907cef972dbbd6352ef7ff7dc0492f658c0c26495f", size = 516951, upload-time = "2026-08-10T14:30:10.907Z" }, + { url = "https://files.pythonhosted.org/packages/9a/07/da554b71ab88e649da146e1065d86a48a5c5d92e50ab74ef41b504aa7f56/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:a1eaccf5c3a1d3e46dead602c72e6836731e8e245c9de6a27764567b6b62d4c0", size = 1642735, upload-time = "2026-08-10T14:15:11.92Z" }, + { url = "https://files.pythonhosted.org/packages/78/76/26a3782a051677668af9d92beaa47cd87ba9dd5072f762961144a03dd4c6/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:19e4e026fe20691f333b8eb1a3bc9625eceba8c3f9d62ec5a6f8581afbc6b5a5", size = 1700925, upload-time = "2026-08-10T13:40:37.656Z" }, + { url = "https://files.pythonhosted.org/packages/28/d9/fe7baf4190c2ae71f267efb9de21b3172bb35bc0ed1ef53dd6027d658e33/greenlet-3.5.5-cp315-cp315t-win_amd64.whl", hash = "sha256:712aee154f648bde84634654bb38bb78c69ac640c37a45c9effed800735049d8", size = 331829, upload-time = "2026-08-10T13:26:48.851Z" }, + { url = "https://files.pythonhosted.org/packages/df/af/419a4e383bd600858a9b67e9b280a60fdc383ee3f2fe5b6c0c1ef04e74d1/greenlet-3.5.5-cp315-cp315t-win_arm64.whl", hash = "sha256:7f049911ee81a16a03c33d5450d8d5867d27f596ca5fb201b86f4524e874468b", size = 315093, upload-time = "2026-08-10T13:29:34.949Z" }, +] + +[[package]] +name = "playwright" +version = "1.62.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/5b/ca2abcf3aa69f9fb510215e3064f30b57fe57657c8d04ede45bb966d5606/playwright-1.62.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:d8da938f3748841a8754f2e1f0216902c1c8f8ae3720de8b32ccf8e6913a7c4f", size = 43732091, upload-time = "2026-07-31T17:00:44.178Z" }, + { url = "https://files.pythonhosted.org/packages/af/1a/0bfbe9904350961f4dbb713f04342e40d548c5fc26c8157bd13617c81492/playwright-1.62.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:db755ab27db21a04186f1fe8169888e42356086e439b1059b923ef417f0b6034", size = 42510842, upload-time = "2026-07-31T17:00:48.596Z" }, + { url = "https://files.pythonhosted.org/packages/66/dc/c0486b407ad0699a250f6bbe3066fca95344009a99ca66e88ca175c69dc1/playwright-1.62.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:5108bd5b3e87169ddf269feee097da5893af7f8aea4634dfc840518d64c1f1da", size = 43732093, upload-time = "2026-07-31T17:00:52.218Z" }, + { url = "https://files.pythonhosted.org/packages/43/6b/b24aebc2b04bffcb342bccf96e287c78b363e1615bed5cea97500cc0393a/playwright-1.62.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:ba33bae6a13b3d9d354c751cb618af357d20fe1d57767cbcce52079bbef17ad3", size = 47748926, upload-time = "2026-07-31T17:00:56.438Z" }, + { url = "https://files.pythonhosted.org/packages/36/43/b4b18bdc87e1949568fffdcde3ff9a0456266b2d0c6d4432cc34d89ea6eb/playwright-1.62.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db2d76613a57ad844362ce42f7d0c2fa26b19a4f7a46d4f76b891c631e6e5aff", size = 47441423, upload-time = "2026-07-31T17:01:00.404Z" }, + { url = "https://files.pythonhosted.org/packages/81/22/af5d926fc2c32a339eec00a443644bc40ab9db1dd2dd9017873c59773c0c/playwright-1.62.0-py3-none-win32.whl", hash = "sha256:e5614fa89355d7081457680324bb219f79f69c423c5cb6fa250e30b0d8aebf1c", size = 38164450, upload-time = "2026-07-31T17:01:04.187Z" }, + { url = "https://files.pythonhosted.org/packages/2b/a9/4160c1033c07af98bf841ad079457dd78408a5ee0dd56cbfe50b8b6a1c22/playwright-1.62.0-py3-none-win_amd64.whl", hash = "sha256:92c0d98ed04eb35af557b709875edba415b1f548bdb22ddb5bb3e1e6c835c2f1", size = 38164458, upload-time = "2026-07-31T17:01:08.459Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ec/06b55d619a7082a766aa04f2c6bb31435c87f02930087d8a0517119408fa/playwright-1.62.0-py3-none-win_arm64.whl", hash = "sha256:ea8d3055aa9d5a9f1832ac82517bd8b42c78fac7ebcbebb0107116735c8cb6a1", size = 34208868, upload-time = "2026-07-31T17:01:11.818Z" }, +] + +[[package]] +name = "psycopg" +version = "3.3.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/73/8fb739d0f6bba247b9b93c9840c402a4f88545be5f1d4b02b23366371c00/psycopg-3.3.5.tar.gz", hash = "sha256:d0a3d9ccf5788af054cbd745278cb02401b5c312aeaafbf2c6144460aec47da4", size = 166508, upload-time = "2026-08-31T22:45:43.151Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/2e/d0a645bcaadde68bd6d93c43f02f14b0191bdda367ce3f7722abe3da744a/psycopg-3.3.5-py3-none-any.whl", hash = "sha256:ce5aa5cdb4f9379f00f487590e5890bfa7df9a164648c969ffa628505e21af4e", size = 213598, upload-time = "2026-08-31T22:39:02.184Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/1c/e718752cc63cf4e99e4a10fd36e3a3364dabdd0819484a24c0d79fbb9685/psycopg_binary-3.3.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6e85d50b87257fb117675a19ee59daa7bf9a57f6431500adf7059df799232ef4", size = 4704421, upload-time = "2026-08-31T22:42:59.564Z" }, + { url = "https://files.pythonhosted.org/packages/af/cf/a0e748e27c09b92738e4460582d121ba1908be3e36791e150f435e54b332/psycopg_binary-3.3.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e5becd311f9af8d180bad372f51fb2252fd02cb2073056e2b170c9274f95fe7f", size = 4765054, upload-time = "2026-08-31T22:43:05.421Z" }, + { url = "https://files.pythonhosted.org/packages/39/62/0cbac0266d56c94dd1f702d9af2b5d54bf80b6e658048f4f2c5bd63dd7e3/psycopg_binary-3.3.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:19e5bf9872dbd164c220567fd385ba2309c7d9df1541f78343510c6b0f36a1b7", size = 5547137, upload-time = "2026-08-31T22:43:11.768Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3a/73c6f8871f38fc07a9c0b4cbc9467beb116a2783cecf87dfa53900a396dc/psycopg_binary-3.3.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cb3b3bffebfe07110730626e76238161124f35ac87b748d663316a28d22f58b0", size = 5227577, upload-time = "2026-08-31T22:43:20.767Z" }, + { url = "https://files.pythonhosted.org/packages/59/7b/9f17b9f4d297b774dc574199c4dcd02dadc32a00c4056265918de5c70635/psycopg_binary-3.3.5-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2111f880add40fb03c60556069ad68e884a0908a74d2debafc603caf93b73552", size = 6824606, upload-time = "2026-08-31T22:43:33.698Z" }, + { url = "https://files.pythonhosted.org/packages/94/86/d84dadd94a004dbbb43ce0579f1f766fdc6b8cbef745090e24bea77b5283/psycopg_binary-3.3.5-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:40505676b1526b9ea387dace034040a8c8b0bcf984cd6bd4720a2ab15e813586", size = 5060854, upload-time = "2026-08-31T22:43:42.258Z" }, + { url = "https://files.pythonhosted.org/packages/30/d0/e5078be2c7d2490c0d6cd4cb7b3601aec44be2fa8b3fe927e9419b06004f/psycopg_binary-3.3.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5816472e3bb05615f33a741e0835043d1f4bf9709ff30d2f4aed71815cfc6b5e", size = 4589511, upload-time = "2026-08-31T22:43:50.012Z" }, + { url = "https://files.pythonhosted.org/packages/f1/01/08dfb5b18fa482e025864fd91a022310d0782c42e4cf5dda5d0e010b6790/psycopg_binary-3.3.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:358748fc4c8ccdc0e2bdf55420494930e19c3ade586ea9c3a6de3dad1f897311", size = 4268144, upload-time = "2026-08-31T22:43:56.993Z" }, + { url = "https://files.pythonhosted.org/packages/88/b9/cb01dc1d63f3241b49b2ca7fb9f98d0f5c76127f0dbb9440635a6ad0233e/psycopg_binary-3.3.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1ef2e498be47800f6202b9a2304c22646325ca6d54001b7c785bcfdb24a1e8ab", size = 4001036, upload-time = "2026-08-31T22:44:04.429Z" }, + { url = "https://files.pythonhosted.org/packages/95/49/7c17dd832c05b380562ff2ff5f6ab2bcaeca8b7fff2c2b355854368a5bdb/psycopg_binary-3.3.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:88e01aa2e938a45655a8a5213fc3a44ba78cb4cab8a569b3e0bcb3d1d0eaba16", size = 4313112, upload-time = "2026-08-31T22:44:12.446Z" }, + { url = "https://files.pythonhosted.org/packages/7c/2c/b0b2f887185d6a2ec0b3bef948cc07656d2d1a5d96fa7f2bb03f6ef06ca4/psycopg_binary-3.3.5-cp313-cp313-win_amd64.whl", hash = "sha256:ba466011569297114449df9d523438e1adeedf3e4f31ffb78e897ec3fef3076b", size = 3647313, upload-time = "2026-08-31T22:44:19.139Z" }, + { url = "https://files.pythonhosted.org/packages/46/7f/4e2395da194558533bd9c31f35e4dc58ecbbae6a7176b0d2f72d629e8a51/psycopg_binary-3.3.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:40f8b132c7243ef5f503f0b6f986bf16d38a51b0df1c6ba2577743f128be03e3", size = 4712612, upload-time = "2026-08-31T22:44:25.723Z" }, + { url = "https://files.pythonhosted.org/packages/ae/94/fdb2093c8ccd7048449156db526ad746740cf34fe9c01e5dc1b7a7a8b257/psycopg_binary-3.3.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c0cac998b9b1e82dec853d2e53b3d34d56a525cf231f9441a636cfd5992929a9", size = 4775139, upload-time = "2026-08-31T22:44:36.719Z" }, + { url = "https://files.pythonhosted.org/packages/31/52/5195e87960715f7be2005761b72d56fce4e7757d5333fd40384f071c2de1/psycopg_binary-3.3.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:479b96fd78149cfa10369dc53fbfb89ee729be13146b584a23dbc7e164c0cf1e", size = 5556807, upload-time = "2026-08-31T22:44:42.668Z" }, + { url = "https://files.pythonhosted.org/packages/f7/42/2d616210a91e1327516ed5ae71961aaa31bb740e4a61d7190f2221685a40/psycopg_binary-3.3.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f45d77e398542ce0937d9fa3cd9d84e9c5fc6b34c50a66404ae840bada312750", size = 5236206, upload-time = "2026-08-31T22:44:47.943Z" }, + { url = "https://files.pythonhosted.org/packages/08/2e/e54b0d4cc263b3526e3728bb50a69660d5e79e52255ccd2a1a71e40e6f9a/psycopg_binary-3.3.5-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98a388509306e5e08a4203253ac52846bc1b034e5cbd0ae6da1211593cc28594", size = 6838066, upload-time = "2026-08-31T22:44:55.701Z" }, + { url = "https://files.pythonhosted.org/packages/77/80/ec22a110f81a44c411965982097efeee86006bdd5a1f51f628318106c84c/psycopg_binary-3.3.5-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39e2794b95af61a2ff69e33e5ab6ac5df36e9ffea9a3b18e38b2aaca8c5ad5", size = 5072036, upload-time = "2026-08-31T22:45:02.838Z" }, + { url = "https://files.pythonhosted.org/packages/93/55/7bc3c3ac769ab4fe0c619f2179b4aab3ae043f4d478283dd8279d24c0be4/psycopg_binary-3.3.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9c071bf78e5c2e6efa40bc9089a954d7b41221347a72f35c6bf2d8c96e632f75", size = 4604058, upload-time = "2026-08-31T22:45:10.444Z" }, + { url = "https://files.pythonhosted.org/packages/a3/69/8e7414f7dc10b2959e664330cdcf393e412f67355975dafa876cef265264/psycopg_binary-3.3.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:14fdfd65a96ecbd8b586d14546105641f4a6ac7cbe335c786830ea4de94bbe60", size = 4284766, upload-time = "2026-08-31T22:45:17.881Z" }, + { url = "https://files.pythonhosted.org/packages/c1/72/33f293c1d3ee9114f47e9ef4880f31c9de864e84a9b09454c26e106c25ed/psycopg_binary-3.3.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:8dbd694f3741dd4ac5bc60b70e17f7841aefb3f0f38cef4d2756de270e03af43", size = 4011958, upload-time = "2026-08-31T22:45:24.792Z" }, + { url = "https://files.pythonhosted.org/packages/b7/c9/8e38840e5a7d006987bbc9acb29951912253fa50549a4db92b3aa535f089/psycopg_binary-3.3.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:14f432430fd9e1a9e7d9ab2fe14956c77f5d074ebdc556a1ad04e9a1bd3fca04", size = 4323273, upload-time = "2026-08-31T22:45:32.973Z" }, + { url = "https://files.pythonhosted.org/packages/fc/c7/b7ebf601c307f93e7c4c4ebac0edc9db3b2729ca038efe700a18f86b5517/psycopg_binary-3.3.5-cp314-cp314-win_amd64.whl", hash = "sha256:df209e64674a34b41662c67fdc8b4e0ffd77d2136393790691d086a09f9a6cab", size = 3745885, upload-time = "2026-08-31T22:45:40.537Z" }, +] + +[[package]] +name = "pyee" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, +] + +[[package]] +name = "ruff" +version = "0.14.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/06/f71e3a86b2df0dfa2d2f72195941cd09b44f87711cb7fa5193732cb9a5fc/ruff-0.14.14.tar.gz", hash = "sha256:2d0f819c9a90205f3a867dbbd0be083bee9912e170fd7d9704cc8ae45824896b", size = 4515732, upload-time = "2026-01-22T22:30:17.527Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/89/20a12e97bc6b9f9f68343952da08a8099c57237aef953a56b82711d55edd/ruff-0.14.14-py3-none-linux_armv6l.whl", hash = "sha256:7cfe36b56e8489dee8fbc777c61959f60ec0f1f11817e8f2415f429552846aed", size = 10467650, upload-time = "2026-01-22T22:30:08.578Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b1/c5de3fd2d5a831fcae21beda5e3589c0ba67eec8202e992388e4b17a6040/ruff-0.14.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6006a0082336e7920b9573ef8a7f52eec837add1265cc74e04ea8a4368cd704c", size = 10883245, upload-time = "2026-01-22T22:30:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/b8/7c/3c1db59a10e7490f8f6f8559d1db8636cbb13dccebf18686f4e3c9d7c772/ruff-0.14.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:026c1d25996818f0bf498636686199d9bd0d9d6341c9c2c3b62e2a0198b758de", size = 10231273, upload-time = "2026-01-22T22:30:34.642Z" }, + { url = "https://files.pythonhosted.org/packages/a1/6e/5e0e0d9674be0f8581d1f5e0f0a04761203affce3232c1a1189d0e3b4dad/ruff-0.14.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f666445819d31210b71e0a6d1c01e24447a20b85458eea25a25fe8142210ae0e", size = 10585753, upload-time = "2026-01-22T22:30:31.781Z" }, + { url = "https://files.pythonhosted.org/packages/23/09/754ab09f46ff1884d422dc26d59ba18b4e5d355be147721bb2518aa2a014/ruff-0.14.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c0f18b922c6d2ff9a5e6c3ee16259adc513ca775bcf82c67ebab7cbd9da5bc8", size = 10286052, upload-time = "2026-01-22T22:30:24.827Z" }, + { url = "https://files.pythonhosted.org/packages/c8/cc/e71f88dd2a12afb5f50733851729d6b571a7c3a35bfdb16c3035132675a0/ruff-0.14.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1629e67489c2dea43e8658c3dba659edbfd87361624b4040d1df04c9740ae906", size = 11043637, upload-time = "2026-01-22T22:30:13.239Z" }, + { url = "https://files.pythonhosted.org/packages/67/b2/397245026352494497dac935d7f00f1468c03a23a0c5db6ad8fc49ca3fb2/ruff-0.14.14-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:27493a2131ea0f899057d49d303e4292b2cae2bb57253c1ed1f256fbcd1da480", size = 12194761, upload-time = "2026-01-22T22:30:22.542Z" }, + { url = "https://files.pythonhosted.org/packages/5b/06/06ef271459f778323112c51b7587ce85230785cd64e91772034ddb88f200/ruff-0.14.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01ff589aab3f5b539e35db38425da31a57521efd1e4ad1ae08fc34dbe30bd7df", size = 12005701, upload-time = "2026-01-22T22:30:20.499Z" }, + { url = "https://files.pythonhosted.org/packages/41/d6/99364514541cf811ccc5ac44362f88df66373e9fec1b9d1c4cc830593fe7/ruff-0.14.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1cc12d74eef0f29f51775f5b755913eb523546b88e2d733e1d701fe65144e89b", size = 11282455, upload-time = "2026-01-22T22:29:59.679Z" }, + { url = "https://files.pythonhosted.org/packages/ca/71/37daa46f89475f8582b7762ecd2722492df26421714a33e72ccc9a84d7a5/ruff-0.14.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb8481604b7a9e75eff53772496201690ce2687067e038b3cc31aaf16aa0b974", size = 11215882, upload-time = "2026-01-22T22:29:57.032Z" }, + { url = "https://files.pythonhosted.org/packages/2c/10/a31f86169ec91c0705e618443ee74ede0bdd94da0a57b28e72db68b2dbac/ruff-0.14.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:14649acb1cf7b5d2d283ebd2f58d56b75836ed8c6f329664fa91cdea19e76e66", size = 11180549, upload-time = "2026-01-22T22:30:27.175Z" }, + { url = "https://files.pythonhosted.org/packages/fd/1e/c723f20536b5163adf79bdd10c5f093414293cdf567eed9bdb7b83940f3f/ruff-0.14.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e8058d2145566510790eab4e2fad186002e288dec5e0d343a92fe7b0bc1b3e13", size = 10543416, upload-time = "2026-01-22T22:30:01.964Z" }, + { url = "https://files.pythonhosted.org/packages/3e/34/8a84cea7e42c2d94ba5bde1d7a4fae164d6318f13f933d92da6d7c2041ff/ruff-0.14.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:e651e977a79e4c758eb807f0481d673a67ffe53cfa92209781dfa3a996cf8412", size = 10285491, upload-time = "2026-01-22T22:30:29.51Z" }, + { url = "https://files.pythonhosted.org/packages/55/ef/b7c5ea0be82518906c978e365e56a77f8de7678c8bb6651ccfbdc178c29f/ruff-0.14.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:cc8b22da8d9d6fdd844a68ae937e2a0adf9b16514e9a97cc60355e2d4b219fc3", size = 10733525, upload-time = "2026-01-22T22:30:06.499Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/aaf1dfbcc53a2811f6cc0a1759de24e4b03e02ba8762daabd9b6bd8c59e3/ruff-0.14.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:16bc890fb4cc9781bb05beb5ab4cd51be9e7cb376bf1dd3580512b24eb3fda2b", size = 11315626, upload-time = "2026-01-22T22:30:36.848Z" }, + { url = "https://files.pythonhosted.org/packages/2c/aa/9f89c719c467dfaf8ad799b9bae0df494513fb21d31a6059cb5870e57e74/ruff-0.14.14-py3-none-win32.whl", hash = "sha256:b530c191970b143375b6a68e6f743800b2b786bbcf03a7965b06c4bf04568167", size = 10502442, upload-time = "2026-01-22T22:30:38.93Z" }, + { url = "https://files.pythonhosted.org/packages/87/44/90fa543014c45560cae1fffc63ea059fb3575ee6e1cb654562197e5d16fb/ruff-0.14.14-py3-none-win_amd64.whl", hash = "sha256:3dde1435e6b6fe5b66506c1dff67a421d0b7f6488d466f651c07f4cab3bf20fd", size = 11630486, upload-time = "2026-01-22T22:30:10.852Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6a/40fee331a52339926a92e17ae748827270b288a35ef4a15c9c8f2ec54715/ruff-0.14.14-py3-none-win_arm64.whl", hash = "sha256:56e6981a98b13a32236a72a8da421d7839221fa308b223b9283312312e5ac76c", size = 10920448, upload-time = "2026-01-22T22:30:15.417Z" }, +] + +[[package]] +name = "simplejson" +version = "3.20.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f4/a1ac5ed32f7ed9a088d62a59d410d4c204b3b3815722e2ccfb491fa8251b/simplejson-3.20.2.tar.gz", hash = "sha256:5fe7a6ce14d1c300d80d08695b7f7e633de6cd72c80644021874d985b3393649", size = 85784, upload-time = "2025-09-26T16:29:36.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/9e/f326d43f6bf47f4e7704a4426c36e044c6bedfd24e072fb8e27589a373a5/simplejson-3.20.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:90d311ba8fcd733a3677e0be21804827226a57144130ba01c3c6a325e887dd86", size = 93530, upload-time = "2025-09-26T16:28:18.07Z" }, + { url = "https://files.pythonhosted.org/packages/35/28/5a4b8f3483fbfb68f3f460bc002cef3a5735ef30950e7c4adce9c8da15c7/simplejson-3.20.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:feed6806f614bdf7f5cb6d0123cb0c1c5f40407ef103aa935cffaa694e2e0c74", size = 75846, upload-time = "2025-09-26T16:28:19.12Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4d/30dfef83b9ac48afae1cf1ab19c2867e27b8d22b5d9f8ca7ce5a0a157d8c/simplejson-3.20.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6b1d8d7c3e1a205c49e1aee6ba907dcb8ccea83651e6c3e2cb2062f1e52b0726", size = 75661, upload-time = "2025-09-26T16:28:20.219Z" }, + { url = "https://files.pythonhosted.org/packages/09/1d/171009bd35c7099d72ef6afd4bb13527bab469965c968a17d69a203d62a6/simplejson-3.20.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:552f55745044a24c3cb7ec67e54234be56d5d6d0e054f2e4cf4fb3e297429be5", size = 150579, upload-time = "2025-09-26T16:28:21.337Z" }, + { url = "https://files.pythonhosted.org/packages/61/ae/229bbcf90a702adc6bfa476e9f0a37e21d8c58e1059043038797cbe75b8c/simplejson-3.20.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2da97ac65165d66b0570c9e545786f0ac7b5de5854d3711a16cacbcaa8c472d", size = 158797, upload-time = "2025-09-26T16:28:22.53Z" }, + { url = "https://files.pythonhosted.org/packages/90/c5/fefc0ac6b86b9108e302e0af1cf57518f46da0baedd60a12170791d56959/simplejson-3.20.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f59a12966daa356bf68927fca5a67bebac0033cd18b96de9c2d426cd11756cd0", size = 148851, upload-time = "2025-09-26T16:28:23.733Z" }, + { url = "https://files.pythonhosted.org/packages/43/f1/b392952200f3393bb06fbc4dd975fc63a6843261705839355560b7264eb2/simplejson-3.20.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133ae2098a8e162c71da97cdab1f383afdd91373b7ff5fe65169b04167da976b", size = 152598, upload-time = "2025-09-26T16:28:24.962Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b4/d6b7279e52a3e9c0fa8c032ce6164e593e8d9cf390698ee981ed0864291b/simplejson-3.20.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7977640af7b7d5e6a852d26622057d428706a550f7f5083e7c4dd010a84d941f", size = 150498, upload-time = "2025-09-26T16:28:26.114Z" }, + { url = "https://files.pythonhosted.org/packages/62/22/ec2490dd859224326d10c2fac1353e8ad5c84121be4837a6dd6638ba4345/simplejson-3.20.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b530ad6d55e71fa9e93e1109cf8182f427a6355848a4ffa09f69cc44e1512522", size = 152129, upload-time = "2025-09-26T16:28:27.552Z" }, + { url = "https://files.pythonhosted.org/packages/33/ce/b60214d013e93dd9e5a705dcb2b88b6c72bada442a97f79828332217f3eb/simplejson-3.20.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bd96a7d981bf64f0e42345584768da4435c05b24fd3c364663f5fbc8fabf82e3", size = 159359, upload-time = "2025-09-26T16:28:28.667Z" }, + { url = "https://files.pythonhosted.org/packages/99/21/603709455827cdf5b9d83abe726343f542491ca8dc6a2528eb08de0cf034/simplejson-3.20.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f28ee755fadb426ba2e464d6fcf25d3f152a05eb6b38e0b4f790352f5540c769", size = 154717, upload-time = "2025-09-26T16:28:30.288Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f9/dc7f7a4bac16cf7eb55a4df03ad93190e11826d2a8950052949d3dfc11e2/simplejson-3.20.2-cp313-cp313-win32.whl", hash = "sha256:472785b52e48e3eed9b78b95e26a256f59bb1ee38339be3075dad799e2e1e661", size = 74289, upload-time = "2025-09-26T16:28:31.809Z" }, + { url = "https://files.pythonhosted.org/packages/87/10/d42ad61230436735c68af1120622b28a782877146a83d714da7b6a2a1c4e/simplejson-3.20.2-cp313-cp313-win_amd64.whl", hash = "sha256:a1a85013eb33e4820286139540accbe2c98d2da894b2dcefd280209db508e608", size = 75972, upload-time = "2025-09-26T16:28:32.883Z" }, + { url = "https://files.pythonhosted.org/packages/05/5b/83e1ff87eb60ca706972f7e02e15c0b33396e7bdbd080069a5d1b53cf0d8/simplejson-3.20.2-py3-none-any.whl", hash = "sha256:3b6bb7fb96efd673eac2e4235200bfffdc2353ad12c54117e1e4e2fc485ac017", size = 57309, upload-time = "2025-09-26T16:29:35.312Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, +]