standardize car-tracker naming and verify container entrypoints
All checks were successful
CI / Lint, PostgreSQL tests, and package (push) Successful in 12s

This commit is contained in:
Antoine Vandeplanque 2026-09-30 16:29:53 +02:00
parent 5f42c24153
commit a353e1c683
29 changed files with 135 additions and 93 deletions

View File

@ -5,8 +5,10 @@
!uv.lock !uv.lock
!README.md !README.md
!src/ !src/
!src/car_traker/ !src/car_tracker/
!src/car_traker/** !src/car_tracker/**
src/**/__pycache__/ src/**/__pycache__/
src/**/*.pyc src/**/*.pyc
# Keep locally generated packaging metadata out of the build context.
src/**/*.egg-info/
!searches.toml !searches.toml

View File

@ -15,15 +15,15 @@ jobs:
env: env:
POSTGRES_USER: postgres POSTGRES_USER: postgres
POSTGRES_PASSWORD: ci-test-only POSTGRES_PASSWORD: ci-test-only
POSTGRES_DB: car_traker_test POSTGRES_DB: car_tracker_test
options: >- options: >-
--health-cmd "pg_isready -U postgres -d car_traker_test" --health-cmd "pg_isready -U postgres -d car_tracker_test"
--health-interval 5s --health-interval 5s
--health-timeout 5s --health-timeout 5s
--health-retries 12 --health-retries 12
env: env:
# Disposable service only. Never replace this with the homelab database URL. # Disposable service only. Never replace this with the homelab database URL.
TEST_DATABASE_URL: postgresql://postgres:ci-test-only@postgres:5432/car_traker_test TEST_DATABASE_URL: postgresql://postgres:ci-test-only@postgres:5432/car_tracker_test
UV_NO_PROGRESS: "1" UV_NO_PROGRESS: "1"
steps: steps:
- name: Check out repository - name: Check out repository
@ -55,6 +55,6 @@ jobs:
- name: Check installed wheel and packaged migrations - name: Check installed wheel and packaged migrations
run: | run: |
uv venv /tmp/car-traker-wheel uv venv /tmp/car-tracker-wheel
uv pip install --python /tmp/car-traker-wheel/bin/python --no-deps dist/*.whl uv pip install --python /tmp/car-tracker-wheel/bin/python --no-deps dist/*.whl
/tmp/car-traker-wheel/bin/python -c 'from importlib.resources import files; assert files("car_traker").joinpath("migrations", "001_silver.sql").is_file(), "SQL migration missing from wheel"' /tmp/car-tracker-wheel/bin/python -c 'from importlib.resources import files; assert files("car_tracker").joinpath("migrations", "001_silver.sql").is_file(), "SQL migration missing from wheel"'

View File

@ -22,9 +22,15 @@ RUN uv sync --locked --no-dev --no-install-project \
&& mkdir /data \ && mkdir /data \
&& chown app:app /data && chown app:app /data
COPY src/ ./src/ COPY src/ ./src/
RUN uv sync --locked --no-dev --no-editable && rm -rf /tmp/.cache # Fail the build early if an application executable or migration is missing.
RUN uv sync --locked --no-dev --no-editable \
&& rm -rf /tmp/.cache \
&& for script in car-tracker car-tracker-db car-tracker-pipeline; do \
test -x "/opt/venv/bin/${script}" || { echo "missing console script: ${script}" >&2; exit 1; }; \
done \
&& python -c 'from importlib.resources import files; assert files("car_tracker").joinpath("migrations", "001_silver.sql").is_file()'
COPY searches.toml ./searches.toml COPY searches.toml ./searches.toml
USER 1000:1000 USER 1000:1000
ENTRYPOINT ["python", "-m", "car_traker.container_entrypoint"] ENTRYPOINT ["python", "-m", "car_tracker.container_entrypoint"]
CMD ["car-traker-pipeline", "--headed", "--config", "/app/searches.toml", "--output", "/data", "--family", "saab-900", "--max-pages", "1", "--max-details", "1"] CMD ["car-tracker-pipeline", "--headed", "--config", "/app/searches.toml", "--output", "/data", "--family", "saab-900", "--max-pages", "1", "--max-details", "1"]

View File

@ -10,7 +10,7 @@ and precise location below. Seller/contact objects and photos remain excluded;
free text may still contain personal information and must remain private until free text may still contain personal information and must remain private until
reviewed/redacted for downstream publication. Source coordinates may be approximate. reviewed/redacted for downstream publication. Source coordinates may be approximate.
The collector now lives in `src/car_traker/`, storing separate search/detail The collector now lives in `src/car_tracker/`, storing separate search/detail
observations in versioned compressed JSON Lines with a coverage manifest. See observations in versioned compressed JSON Lines with a coverage manifest. See
`README.md` for current commands, capture policy, retention, and verified live `README.md` for current commands, capture policy, retention, and verified live
results. The original handoff below remains planning context, not a description results. The original handoff below remains planning context, not a description
@ -25,7 +25,7 @@ connection setup remain user actions documented in `docs/POSTGRESQL.md`; no home
database has been accessed or modified during implementation. database has been accessed or modified during implementation.
The manual homelab job is now packaged in `Dockerfile` and `ops/app/compose.yaml`. The manual homelab job is now packaged in `Dockerfile` and `ops/app/compose.yaml`.
`car-traker-pipeline` performs a database preflight, bounded collection, and an `car-tracker-pipeline` performs a database preflight, bounded collection, and an
idempotent import of the exact returned manifest, retaining partial coverage and idempotent import of the exact returned manifest, retaining partial coverage and
separate operational outcomes. There is no scheduler or automatic deployment. separate operational outcomes. There is no scheduler or automatic deployment.
See `docs/DEPLOYMENT.md`; container tests use isolated PostgreSQL 17 and synthetic See `docs/DEPLOYMENT.md`; container tests use isolated PostgreSQL 17 and synthetic
@ -65,10 +65,10 @@ identified.
- PostgreSQL already runs on a Debian VM; reuse it rather than require a second - PostgreSQL already runs on a Debian VM; reuse it rather than require a second
production database. production database.
- Repository hosted on the user's own Gitea instance. - Repository hosted on the user's own Gitea instance.
- Current workspace: `/home/tonio/Workspace/perso/car-traker`. - Current workspace: `/home/tonio/Workspace/perso/car-tracker`.
- At inspection, the repository contained `.gitignore`, `.python-version`, - At inspection, the repository contained `.gitignore`, `.python-version`,
`main.py`, `pyproject.toml`, and an empty `README.md`. `main.py`, `pyproject.toml`, and an empty `README.md`.
- `pyproject.toml` specifies Python >=3.13, project name `car-traker`, version - `pyproject.toml` specifies Python >=3.13, project name `car-tracker`, version
`0.1.0`, and no dependencies. `0.1.0`, and no dependencies.
- The worktree was clean before this handoff document was added. - The worktree was clean before this handoff document was added.
- No application implementation was changed during the planning discussion. - No application implementation was changed during the planning discussion.

View File

@ -14,14 +14,14 @@ Python 3.13+ and `uv`:
```sh ```sh
uv sync uv sync
uv run playwright install chromium uv run playwright install chromium
uv run car-traker --family saab-900 --max-pages 1 --max-details 1 --headed uv run car-tracker --family saab-900 --max-pages 1 --max-details 1 --headed
``` ```
The smoke command deliberately caps coverage and returns exit code 1 if further The smoke command deliberately caps coverage and returns exit code 1 if further
pages or details exist. Collect all configured searches and their individual ads with: pages or details exist. Collect all configured searches and their individual ads with:
```sh ```sh
uv run car-traker --headed uv run car-tracker --headed
``` ```
Options: `--config searches.toml`, `--output data/raw/leboncoin`, `--family`, Options: `--config searches.toml`, `--output data/raw/leboncoin`, `--family`,
@ -47,9 +47,9 @@ results are intentionally preserved with their query provenance.
## Project Structure ## Project Structure
```text ```text
src/car_traker/ # Installed Python package src/car_tracker/ # Installed Python package
__init__.py __init__.py
__main__.py # python -m car_traker entry point __main__.py # python -m car_tracker entry point
collect.py # Collector and CLI collect.py # Collector and CLI
database.py # PostgreSQL migrations and import CLI database.py # PostgreSQL migrations and import CLI
pipeline.py # Manual collect -> raw -> import job pipeline.py # Manual collect -> raw -> import job
@ -69,7 +69,7 @@ data/ # Local snapshots (ignored by Git)
`uv sync` installs the package in editable mode from `src/`. Run commands from the `uv sync` installs the package in editable mode from `src/`. Run commands from the
repository root so the default configuration and data paths resolve correctly. repository root so the default configuration and data paths resolve correctly.
`uv run python -m car_traker` also works as an alternative to `uv run car-traker`. `uv run python -m car_tracker` also works as an alternative to `uv run car-tracker`.
## Silver Import ## Silver Import
@ -81,8 +81,8 @@ role/database setup, network access, and laptop connection configuration.
Once the remote connection is configured: Once the remote connection is configured:
```sh ```sh
uv run car-traker-db migrate uv run car-tracker-db migrate
uv run car-traker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json uv run car-tracker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json
``` ```
Importing an unchanged run again is a no-op. Search and detail observations retain Importing an unchanged run again is a no-op. Search and detail observations retain
@ -92,7 +92,7 @@ and validation issues. Generation/trim inference, redaction, and scheduling are
## Manual Homelab Job ## Manual Homelab Job
`car-traker-pipeline` checks database connectivity/schema, collects raw observations, `car-tracker-pipeline` checks database connectivity/schema, collects raw observations,
and imports that exact finalized run. Partial/failed collections remain marked as and imports that exact finalized run. Partial/failed collections remain marked as
such even when their import succeeds. A lock prevents overlapping pipeline jobs on such even when their import succeeds. A lock prevents overlapping pipeline jobs on
the same output directory, and `pipeline.json` records the import outcome separately the same output directory, and `pipeline.json` records the import outcome separately
@ -101,7 +101,7 @@ from the immutable raw manifest.
For a laptop smoke run with the existing database connection configured: For a laptop smoke run with the existing database connection configured:
```sh ```sh
uv run car-traker-pipeline --headed --family saab-900 --max-pages 1 --max-details 1 uv run car-tracker-pipeline --headed --family saab-900 --max-pages 1 --max-details 1
``` ```
For Docker on the homelab, follow [manual deployment](docs/DEPLOYMENT.md) first to For Docker on the homelab, follow [manual deployment](docs/DEPLOYMENT.md) first to
@ -110,7 +110,7 @@ Then, from the homelab checkout:
```sh ```sh
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml build app docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml build app
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-traker-db check docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-tracker-db check
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app
``` ```

View File

@ -13,7 +13,7 @@ One Docker-backed job, selected by the runner label `python-ci`, performs:
2. Install uv 0.9.3 and Python 3.13, then `uv sync --locked --dev`. 2. Install uv 0.9.3 and Python 3.13, then `uv sync --locked --dev`.
3. Run Ruff's correctness-focused checks over `src` and `tests`. 3. Run Ruff's correctness-focused checks over `src` and `tests`.
4. Run the full unittest suite, including integration tests against a fresh 4. Run the full unittest suite, including integration tests against a fresh
PostgreSQL 17 service named `postgres` and database `car_traker_test`, matching PostgreSQL 17 service named `postgres` and database `car_tracker_test`, matching
the homelab's PostgreSQL major version. the homelab's PostgreSQL major version.
5. Build the source distribution and wheel, install that wheel in an independent 5. Build the source distribution and wheel, install that wheel in an independent
environment, and verify that its SQL migrations are packaged. environment, and verify that its SQL migrations are packaged.

View File

@ -110,8 +110,8 @@ necessary source/port through the firewall. Check `pg_hba_file_rules` for errors
and reload authentication changes with `SELECT pg_reload_conf();` as an admin. and reload authentication changes with `SELECT pg_reload_conf();` as an admin.
Changing `listen_addresses` requires a PostgreSQL restart: coordinate downtime for Changing `listen_addresses` requires a PostgreSQL restart: coordinate downtime for
the shared VM's clients rather than restarting casually. See the shared VM's clients rather than restarting casually. See
[PostgreSQL troubleshooting](POSTGRESQL.md#connection-troubleshooting) for details; [PostgreSQL troubleshooting](POSTGRESQL.md#connection-troubleshooting) for details.
that document's `car_traker` names are examples, not this VM's `car_tracker` names. Use the existing `car_tracker` database and role on this VM.
## Build And Smoke Run ## Build And Smoke Run
Validate without printing interpolated configuration, then build on the homelab: Validate without printing interpolated configuration, then build on the homelab:
@ -119,7 +119,7 @@ Validate without printing interpolated configuration, then build on the homelab:
```sh ```sh
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml config --quiet docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml config --quiet
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml build app docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml build app
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-traker-db check docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-tracker-db check
``` ```
`check` is read-only connectivity testing, not a schema or migration-privilege `check` is read-only connectivity testing, not a schema or migration-privilege
@ -128,7 +128,7 @@ opening the browser. No new migrations are required for this milestone. If the
existing packaged migrations have not yet been applied, apply them explicitly: existing packaged migrations have not yet been applied, apply them explicitly:
```sh ```sh
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-traker-db migrate docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-tracker-db migrate
``` ```
Migrations are not automatic. Once DB access/schema are ready, run the image's Migrations are not automatic. Once DB access/schema are ready, run the image's
@ -144,7 +144,7 @@ smoke run, this is the explicit larger manual command:
```sh ```sh
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app \ docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app \
car-traker-pipeline --headed --config /app/searches.toml --output /data \ car-tracker-pipeline --headed --config /app/searches.toml --output /data \
--family saab-900 --max-pages 3 --max-details 5 --family saab-900 --max-pages 3 --max-details 5
``` ```
@ -175,7 +175,7 @@ date/run ID printed by the pipeline:
```sh ```sh
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app \ docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app \
car-traker-db import '/data/<date>/<run-id>/manifest.json' car-tracker-db import '/data/<date>/<run-id>/manifest.json'
``` ```
An identical replay reports `already_imported` without duplicating observations. An identical replay reports `already_imported` without duplicating observations.

View File

@ -6,7 +6,7 @@ Development and CLI execution currently happen on your laptop. Raw files remain
on the laptop, and only the database connection crosses the network: on the laptop, and only the database connection crosses the network:
```text ```text
Laptop: raw JSONL -> Python silver importer -> Homelab VM: PostgreSQL/car_traker Laptop: raw JSONL -> Python silver importer -> Homelab VM: PostgreSQL/car_tracker
``` ```
You do not need to install PostgreSQL on the laptop or move the repository to the You do not need to install PostgreSQL on the laptop or move the repository to the
@ -22,18 +22,18 @@ Create a dedicated database and an application login. Run these commands as your
PostgreSQL administrator, for example with `sudo -u postgres psql` on the Debian VM: PostgreSQL administrator, for example with `sudo -u postgres psql` on the Debian VM:
```sql ```sql
CREATE ROLE car_traker LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; CREATE ROLE car_tracker LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;
SET password_encryption = 'scram-sha-256'; SET password_encryption = 'scram-sha-256';
\password car_traker \password car_tracker
CREATE DATABASE car_traker OWNER car_traker; CREATE DATABASE car_tracker OWNER car_tracker;
REVOKE ALL ON DATABASE car_traker FROM PUBLIC; REVOKE ALL ON DATABASE car_tracker FROM PUBLIC;
\connect car_traker \connect car_tracker
REVOKE CREATE ON SCHEMA public FROM PUBLIC; REVOKE CREATE ON SCHEMA public FROM PUBLIC;
``` ```
`\password` prompts locally without putting the password in SQL history. These are `\password` prompts locally without putting the password in SQL history. These are
one-time provisioning commands; inspect existing roles/databases rather than one-time provisioning commands; inspect existing roles/databases rather than
rerunning blindly if the names already exist. The importer connects as `car_traker`, rerunning blindly if the names already exist. The importer connects as `car_tracker`,
not `postgres`. Owning this dedicated database allows it to create its `silver` not `postgres`. Owning this dedicated database allows it to create its `silver`
schema and run migrations without cluster-wide administrative privileges. schema and run migrations without cluster-wide administrative privileges.
@ -47,7 +47,7 @@ Restrict the firewall to the importer machine's IP and PostgreSQL port, normally
scoped `pg_hba.conf` rule, replacing `<IMPORTER_IP>` with the actual client address: scoped `pg_hba.conf` rule, replacing `<IMPORTER_IP>` with the actual client address:
```text ```text
hostssl car_traker car_traker <IMPORTER_IP>/32 scram-sha-256 hostssl car_tracker car_tracker <IMPORTER_IP>/32 scram-sha-256
``` ```
Rules are first-match; review existing rules and their order. Reload authentication Rules are first-match; review existing rules and their order. Reload authentication
@ -69,8 +69,8 @@ outside the repository. Prefer a password file rather than a password in command
```sh ```sh
export PGHOST=postgres.your-lan.example export PGHOST=postgres.your-lan.example
export PGPORT=5432 export PGPORT=5432
export PGDATABASE=car_traker export PGDATABASE=car_tracker
export PGUSER=car_traker export PGUSER=car_tracker
export PGSSLMODE=verify-full export PGSSLMODE=verify-full
export PGSSLROOTCERT=/path/to/your/postgres-ca.crt export PGSSLROOTCERT=/path/to/your/postgres-ca.crt
export PGPASSFILE="$HOME/.pgpass" export PGPASSFILE="$HOME/.pgpass"
@ -79,7 +79,7 @@ export PGPASSFILE="$HOME/.pgpass"
The matching password-file entry has this form: The matching password-file entry has this form:
```text ```text
postgres.your-lan.example:5432:car_traker:car_traker:<your-password> postgres.your-lan.example:5432:car_tracker:car_tracker:<your-password>
``` ```
Protect it with `chmod 600 "$HOME/.pgpass"`. Escape `:` and `\` in passwords using Protect it with `chmod 600 "$HOME/.pgpass"`. Escape `:` and `\` in passwords using
@ -99,7 +99,7 @@ psql -c 'SELECT current_database(), current_user, version();'
Alternatively, no local `psql` installation is needed for the read-only CLI check: Alternatively, no local `psql` installation is needed for the read-only CLI check:
```sh ```sh
uv run car-traker-db check uv run car-tracker-db check
``` ```
It connects with the same settings as migrations/imports and executes only `SELECT 1`. It connects with the same settings as migrations/imports and executes only `SELECT 1`.
@ -114,7 +114,7 @@ contain credentials or private data. `no pg_hba.conf entry` means PostgreSQL was
reached, but its access rules do not allow the client/database/role combination. reached, but its access rules do not allow the client/database/role combination.
The rule must use the **client IP seen by PostgreSQL**, which can be a VPN or NAT The rule must use the **client IP seen by PostgreSQL**, which can be a VPN or NAT
address rather than the laptop's Wi-Fi/LAN address. Use the actual database and address rather than the laptop's Wi-Fi/LAN address. Use the actual database and
role names you created; `car_traker` in this document is an example, not a hardcoded role names you created; `car_tracker` in this document is an example, not a hardcoded
application requirement. application requirement.
On the homelab, locate the active configuration and inspect parsed rules as an On the homelab, locate the active configuration and inspect parsed rules as an
@ -130,7 +130,7 @@ Add the appropriate narrow `hostssl` rule in the file returned by `SHOW hba_file
then run `SELECT pg_reload_conf();`. Check the `error` column and server logs if it then run `SELECT pg_reload_conf();`. Check the `error` column and server logs if it
still fails; a successful reload request alone does not prove every rule is valid. still fails; a successful reload request alone does not prove every rule is valid.
Do not change passwords or use `trust` to work around a missing HBA rule. Once access Do not change passwords or use `trust` to work around a missing HBA rule. Once access
is permitted, rerun `car-traker-db check` from the laptop before migrating. is permitted, rerun `car-tracker-db check` from the laptop before migrating.
Other categories point to DNS/VPN, refused/timed-out connections, authentication, Other categories point to DNS/VPN, refused/timed-out connections, authentication,
TLS verification, missing databases, or insufficient privileges. This check does TLS verification, missing databases, or insufficient privileges. This check does
@ -142,8 +142,8 @@ From the repository root, with the connection configured:
```sh ```sh
uv sync uv sync
uv run car-traker-db migrate uv run car-tracker-db migrate
uv run car-traker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json uv run car-tracker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json
``` ```
The example uses the existing 35-search/one-detail live run. Choose any finalized The example uses the existing 35-search/one-detail live run. Choose any finalized
@ -298,13 +298,13 @@ Database integration tests are skipped unless `TEST_DATABASE_URL` is set. They
One local example (test password only; localhost-only port; ephemeral data): One local example (test password only; localhost-only port; ephemeral data):
```sh ```sh
docker run --detach --rm --name car-traker-silver-test \ docker run --detach --rm --name car-tracker-silver-test \
--env POSTGRES_PASSWORD=local-test-only --env POSTGRES_DB=car_traker_test \ --env POSTGRES_PASSWORD=local-test-only --env POSTGRES_DB=car_tracker_test \
--publish 127.0.0.1:55432:5432 --tmpfs /var/lib/postgresql/data postgres:17 --publish 127.0.0.1:55432:5432 --tmpfs /var/lib/postgresql/data postgres:17
docker exec car-traker-silver-test pg_isready -U postgres -d car_traker_test docker exec car-tracker-silver-test pg_isready -U postgres -d car_tracker_test
TEST_DATABASE_URL='postgresql://postgres:local-test-only@127.0.0.1:55432/car_traker_test' \ TEST_DATABASE_URL='postgresql://postgres:local-test-only@127.0.0.1:55432/car_tracker_test' \
uv run python -m unittest discover -s tests -v uv run python -m unittest discover -s tests -v
docker stop car-traker-silver-test docker stop car-tracker-silver-test
``` ```
Wait until `pg_isready` succeeds before running the tests. This temporary container Wait until `pg_isready` succeeds before running the tests. This temporary container

View File

@ -1,11 +1,13 @@
name: car-traker-app name: car-tracker-app
services: services:
app: app:
build: build:
context: ../.. context: ../..
dockerfile: Dockerfile dockerfile: Dockerfile
image: car-traker:local # The reconcile agent sets CAR_TRACKER_IMAGE to an immutable CI-built tag.
# The default keeps the documented local build-and-run path working.
image: "${CAR_TRACKER_IMAGE:-car-tracker:local}"
# One-shot manual job only. Never restart a blocked scraper automatically. # One-shot manual job only. Never restart a blocked scraper automatically.
restart: "no" restart: "no"
init: true init: true

View File

@ -1,5 +1,5 @@
[project] [project]
name = "car-traker" name = "car-tracker"
version = "0.1.0" version = "0.1.0"
description = "French youngtimer asking-price observatory" description = "French youngtimer asking-price observatory"
readme = "README.md" readme = "README.md"
@ -16,9 +16,9 @@ target-version = "py313"
select = ["E4", "E7", "E9", "F"] select = ["E4", "E7", "E9", "F"]
[project.scripts] [project.scripts]
car-traker = "car_traker.collect:main" car-tracker = "car_tracker.collect:main"
car-traker-db = "car_traker.database:main" car-tracker-db = "car_tracker.database:main"
car-traker-pipeline = "car_traker.pipeline:main" car-tracker-pipeline = "car_tracker.pipeline:main"
[build-system] [build-system]
requires = ["setuptools>=77"] requires = ["setuptools>=77"]
@ -28,4 +28,4 @@ build-backend = "setuptools.build_meta"
where = ["src"] where = ["src"]
[tool.setuptools.package-data] [tool.setuptools.package-data]
car_traker = ["migrations/*.sql"] car_tracker = ["migrations/*.sql"]

View File

@ -0,0 +1 @@
"""Car-tracker collection and database tools."""

View File

@ -41,8 +41,16 @@ def main():
child.send_signal(stopped) child.send_signal(stopped)
code = child.wait() code = child.wait()
return code if code >= 0 else 128 - code return code if code >= 0 else 128 - code
except FileNotFoundError:
# Do not echo caller-supplied paths or arguments.
message = ("Virtual display executable could not be found." if display is None
else "Container command could not be found; check the executable name.")
print(message, file=sys.stderr)
return 3
except (OSError, ValueError): except (OSError, ValueError):
print("Could not start the virtual display or container command.", file=sys.stderr) message = ("Could not start the virtual display." if display is None
else "Could not start or wait for the container command.")
print(message, file=sys.stderr)
return 3 return 3
finally: finally:
# Only clean up descendants after the command has finalized its files/transaction. # Only clean up descendants after the command has finalized its files/transaction.

View File

@ -263,7 +263,7 @@ def migrate(conn):
name text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now() name text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now()
)""") )""")
applied = dict(conn.execute("SELECT name, sha256 FROM silver.schema_migrations").fetchall()) applied = dict(conn.execute("SELECT name, sha256 FROM silver.schema_migrations").fetchall())
migrations = sorted(files("car_traker").joinpath("migrations").iterdir(), key=lambda p: p.name) migrations = sorted(files("car_tracker").joinpath("migrations").iterdir(), key=lambda p: p.name)
known = {m.name for m in migrations if m.name.endswith(".sql")} known = {m.name for m in migrations if m.name.endswith(".sql")}
if set(applied) - known: if set(applied) - known:
raise InputError("database_has_unknown_migrations") raise InputError("database_has_unknown_migrations")
@ -367,7 +367,7 @@ def database_error_hint(exc):
if exc.sqlstate == "42501": if exc.sqlstate == "42501":
return "The database role lacks required privileges. Use the dedicated database owner for migrations/imports, not a superuser." return "The database role lacks required privileges. Use the dedicated database owner for migrations/imports, not a superuser."
if exc.sqlstate in ("42P01", "3F000"): if exc.sqlstate in ("42P01", "3F000"):
return "A required database table/schema is missing. Run car-traker-db migrate against the same database first." return "A required database table/schema is missing. Run car-tracker-db migrate against the same database first."
return "Check the connection configuration and PostgreSQL server logs. Raw error details are withheld to protect credentials and source data." return "Check the connection configuration and PostgreSQL server logs. Raw error details are withheld to protect credentials and source data."

View File

@ -13,8 +13,8 @@ from unittest.mock import patch
from psycopg import sql from psycopg import sql
from car_traker import database, pipeline from car_tracker import database, pipeline
from car_traker.collect import search_url from car_tracker.collect import search_url
import test_database_integration as database_tests import test_database_integration as database_tests
@ -92,7 +92,7 @@ class BrowserPipelineTests(unittest.TestCase):
environment["DATABASE_URL"] = os.environ["TEST_DATABASE_URL"] environment["DATABASE_URL"] = os.environ["TEST_DATABASE_URL"]
with patch.dict(os.environ, environment, clear=True), patch.object( with patch.dict(os.environ, environment, clear=True), patch.object(
BrowserContext, "new_page", new_page, BrowserContext, "new_page", new_page,
), patch("car_traker.collect.time.sleep"), redirect_stdout(stdout), redirect_stderr(stderr): ), patch("car_tracker.collect.time.sleep"), redirect_stdout(stdout), redirect_stderr(stderr):
code = pipeline.main([ code = pipeline.main([
"--headed", "--config", str(self.config), "--output", str(self.output), "--headed", "--config", str(self.config), "--output", str(self.output),
"--delay", "1", "--max-pages", "1", "--max-details", "1", "--delay", "1", "--max-pages", "1", "--max-details", "1",

View File

@ -5,7 +5,7 @@ import unittest
from pathlib import Path from pathlib import Path
from unittest.mock import patch from unittest.mock import patch
from car_traker.collect import SourceAccessError, collect, minimize, parse_detail, parse_search, search_url from car_tracker.collect import SourceAccessError, collect, minimize, parse_detail, parse_search, search_url
def payload(ids=(123,), total=1, pages=1): def payload(ids=(123,), total=1, pages=1):
@ -85,7 +85,7 @@ class CollectorTests(unittest.TestCase):
raise result raise result
return result return result
with patch("car_traker.collect.time.sleep"): with patch("car_tracker.collect.time.sleep"):
kwargs.setdefault("search_only", True) kwargs.setdefault("search_only", True)
path, status = collect(fetch, [{"family": "synthetic", "text": "test"}], path, status = collect(fetch, [{"family": "synthetic", "text": "test"}],
Path(temporary.name), **kwargs) Path(temporary.name), **kwargs)
@ -221,7 +221,7 @@ class CollectorTests(unittest.TestCase):
def test_interrupt_during_delay_does_not_count_as_detail_attempt(self): def test_interrupt_during_delay_does_not_count_as_detail_attempt(self):
with tempfile.TemporaryDirectory() as directory: with tempfile.TemporaryDirectory() as directory:
with patch("car_traker.collect.time.sleep", side_effect=KeyboardInterrupt()): with patch("car_tracker.collect.time.sleep", side_effect=KeyboardInterrupt()):
path, status = collect(lambda url: payload(), path, status = collect(lambda url: payload(),
[{"family": "synthetic", "text": "test"}], Path(directory)) [{"family": "synthetic", "text": "test"}], Path(directory))
manifest = json.loads(path.read_text()) manifest = json.loads(path.read_text())
@ -233,7 +233,7 @@ class CollectorTests(unittest.TestCase):
with tempfile.TemporaryDirectory() as directory: with tempfile.TemporaryDirectory() as directory:
searches = [{"family": "synthetic", "text": "test"}, searches = [{"family": "synthetic", "text": "test"},
{"family": "synthetic", "text": "test-2"}] {"family": "synthetic", "text": "test-2"}]
with patch("car_traker.collect.time.sleep") as sleep: with patch("car_tracker.collect.time.sleep") as sleep:
responses = iter([payload(), detail_payload(), payload()]) responses = iter([payload(), detail_payload(), payload()])
urls = [] urls = []

View File

@ -5,7 +5,7 @@ from tempfile import TemporaryDirectory
import unittest import unittest
from unittest.mock import patch from unittest.mock import patch
from car_traker.collect import collect_browser from car_tracker.collect import collect_browser
class BrowserCleanupTests(unittest.TestCase): class BrowserCleanupTests(unittest.TestCase):
@ -18,7 +18,7 @@ class BrowserCleanupTests(unittest.TestCase):
max_pages=1, delay=1, search_only=True, max_details=None) max_pages=1, delay=1, search_only=True, max_details=None)
self.playwright = self.enterContext(patch("playwright.sync_api.sync_playwright")) self.playwright = self.enterContext(patch("playwright.sync_api.sync_playwright"))
self.browser = self.playwright.return_value.__enter__.return_value.chromium.launch.return_value self.browser = self.playwright.return_value.__enter__.return_value.chromium.launch.return_value
self.collect = self.enterContext(patch("car_traker.collect.collect", return_value=(self.path, "complete"))) self.collect = self.enterContext(patch("car_tracker.collect.collect", return_value=(self.path, "complete")))
self.stderr = self.enterContext(patch("sys.stderr", new_callable=io.StringIO)) self.stderr = self.enterContext(patch("sys.stderr", new_callable=io.StringIO))
def test_browser_close_failure_retains_finalized_result(self): def test_browser_close_failure_retains_finalized_result(self):

View File

@ -5,9 +5,9 @@ import signal
import subprocess import subprocess
import sys import sys
import unittest import unittest
from unittest.mock import patch from unittest.mock import Mock, patch
from car_traker import container_entrypoint from car_tracker import container_entrypoint
class ContainerEntrypointTests(unittest.TestCase): class ContainerEntrypointTests(unittest.TestCase):
@ -15,12 +15,35 @@ class ContainerEntrypointTests(unittest.TestCase):
with patch("sys.argv", ["entrypoint"]), patch("sys.stderr", new_callable=io.StringIO): with patch("sys.argv", ["entrypoint"]), patch("sys.stderr", new_callable=io.StringIO):
self.assertEqual(container_entrypoint.main(), 3) self.assertEqual(container_entrypoint.main(), 3)
def test_missing_display_has_distinct_safe_error(self):
error = FileNotFoundError(2, "missing", "/private/display")
with patch("sys.argv", ["entrypoint", "command"]), \
patch("sys.stderr", new_callable=io.StringIO) as stderr, \
patch.object(container_entrypoint.subprocess, "Popen", side_effect=error):
self.assertEqual(container_entrypoint.main(), 3)
self.assertIn("Virtual display executable could not be found", stderr.getvalue())
self.assertNotIn("/private/display", stderr.getvalue())
def test_missing_command_has_distinct_safe_error_and_cleans_display(self):
display = Mock()
error = FileNotFoundError(2, "missing", "/private/command")
with patch("sys.argv", ["entrypoint", "/private/command"]), \
patch("sys.stderr", new_callable=io.StringIO) as stderr, \
patch.object(container_entrypoint.subprocess, "Popen", side_effect=[display, error]), \
patch.object(container_entrypoint.select, "select", return_value=([3], [], [])), \
patch.object(container_entrypoint.os, "read", return_value=b"0\n"):
self.assertEqual(container_entrypoint.main(), 3)
self.assertIn("Container command could not be found", stderr.getvalue())
self.assertNotIn("/private/command", stderr.getvalue())
display.terminate.assert_called_once()
display.wait.assert_called_once()
@unittest.skipUnless(os.environ.get("RUN_BROWSER_TESTS") == "1", "Xvfb container tests are opt-in") @unittest.skipUnless(os.environ.get("RUN_BROWSER_TESTS") == "1", "Xvfb container tests are opt-in")
class VirtualDisplayTests(unittest.TestCase): class VirtualDisplayTests(unittest.TestCase):
def test_child_exit_code_is_preserved(self): def test_child_exit_code_is_preserved(self):
result = subprocess.run([ result = subprocess.run([
sys.executable, "-m", "car_traker.container_entrypoint", sys.executable, "-m", "car_tracker.container_entrypoint",
sys.executable, "-c", "raise SystemExit(7)", sys.executable, "-c", "raise SystemExit(7)",
], capture_output=True, text=True, timeout=20) ], capture_output=True, text=True, timeout=20)
self.assertEqual(result.returncode, 7, result.stderr) self.assertEqual(result.returncode, 7, result.stderr)
@ -39,7 +62,7 @@ while True:
signal.pause() signal.pause()
""" """
process = subprocess.Popen([ process = subprocess.Popen([
sys.executable, "-m", "car_traker.container_entrypoint", sys.executable, "-m", "car_tracker.container_entrypoint",
sys.executable, "-c", command, sys.executable, "-c", command,
], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
try: try:

View File

@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch
import psycopg import psycopg
from car_traker.database import database_error_hint, main from car_tracker.database import database_error_hint, main
class DatabaseCliTests(unittest.TestCase): class DatabaseCliTests(unittest.TestCase):
@ -37,10 +37,10 @@ class DatabaseCliTests(unittest.TestCase):
def test_check_only_runs_select(self): def test_check_only_runs_select(self):
connection = MagicMock() connection = MagicMock()
with patch.dict("os.environ", {"PGHOST": "synthetic-host"}, clear=True), patch( with patch.dict("os.environ", {"PGHOST": "synthetic-host"}, clear=True), patch(
"sys.argv", ["car-traker-db", "check"] "sys.argv", ["car-tracker-db", "check"]
), patch("car_traker.database.psycopg.connect") as connect, patch( ), patch("car_tracker.database.psycopg.connect") as connect, patch(
"sys.stdout", new_callable=io.StringIO "sys.stdout", new_callable=io.StringIO
) as output, patch("car_traker.database.migrate") as migrate: ) as output, patch("car_tracker.database.migrate") as migrate:
connect.return_value.__enter__.return_value = connection connect.return_value.__enter__.return_value = connection
self.assertEqual(main(), 0) self.assertEqual(main(), 0)
connection.execute.assert_called_once_with("SELECT 1") connection.execute.assert_called_once_with("SELECT 1")
@ -50,8 +50,8 @@ class DatabaseCliTests(unittest.TestCase):
def test_hba_failure_is_actionable_without_echoing_credentials(self): def test_hba_failure_is_actionable_without_echoing_credentials(self):
exc = psycopg.OperationalError('no pg_hba.conf entry for host "private-host"; password=synthetic-secret') exc = psycopg.OperationalError('no pg_hba.conf entry for host "private-host"; password=synthetic-secret')
with patch.dict("os.environ", {"PGHOST": "synthetic-host"}, clear=True), patch( with patch.dict("os.environ", {"PGHOST": "synthetic-host"}, clear=True), patch(
"sys.argv", ["car-traker-db", "check"] "sys.argv", ["car-tracker-db", "check"]
), patch("car_traker.database.psycopg.connect", side_effect=exc), patch( ), patch("car_tracker.database.psycopg.connect", side_effect=exc), patch(
"sys.stderr", new_callable=io.StringIO "sys.stderr", new_callable=io.StringIO
) as output: ) as output:
self.assertEqual(main(), 1) self.assertEqual(main(), 1)
@ -61,9 +61,9 @@ class DatabaseCliTests(unittest.TestCase):
def test_no_implicit_local_connection(self): def test_no_implicit_local_connection(self):
with patch.dict("os.environ", {}, clear=True), patch( with patch.dict("os.environ", {}, clear=True), patch(
"sys.argv", ["car-traker-db", "check"] "sys.argv", ["car-tracker-db", "check"]
), patch("sys.stderr", new_callable=io.StringIO), patch( ), patch("sys.stderr", new_callable=io.StringIO), patch(
"car_traker.database.psycopg.connect" "car_tracker.database.psycopg.connect"
) as connect, self.assertRaises(SystemExit): ) as connect, self.assertRaises(SystemExit):
main() main()
connect.assert_not_called() connect.assert_not_called()

View File

@ -6,8 +6,8 @@ from tempfile import TemporaryDirectory
import unittest import unittest
from unittest.mock import patch from unittest.mock import patch
from car_traker.collect import collect, write_json from car_tracker.collect import collect, write_json
from car_traker.database import InputError, load_json, load_run, prepare_record from car_tracker.database import InputError, load_json, load_run, prepare_record
class RawInputTests(unittest.TestCase): class RawInputTests(unittest.TestCase):
@ -22,7 +22,7 @@ class RawInputTests(unittest.TestCase):
"list_id": 123, "price": [1000], "body": "Synthetic\u2028text\u2029only", "list_id": 123, "price": [1000], "body": "Synthetic\u2028text\u2029only",
}], }],
}}}}) }}}})
with patch("car_traker.collect.time.sleep"): with patch("car_tracker.collect.time.sleep"):
path, _ = collect(lambda url: source, [{"family": "synthetic", "text": "test"}], path, _ = collect(lambda url: source, [{"family": "synthetic", "text": "test"}],
self.root, search_only=True) self.root, search_only=True)
return path, json.loads(path.read_text()) return path, json.loads(path.read_text())

View File

@ -19,8 +19,8 @@ import psycopg
from psycopg import sql from psycopg import sql
from psycopg.conninfo import conninfo_to_dict from psycopg.conninfo import conninfo_to_dict
from car_traker import database from car_tracker import database
from car_traker.collect import search_url, write_json, write_records from car_tracker.collect import search_url, write_json, write_records
@unittest.skipUnless(os.environ.get("TEST_DATABASE_URL"), "TEST_DATABASE_URL is not set") @unittest.skipUnless(os.environ.get("TEST_DATABASE_URL"), "TEST_DATABASE_URL is not set")
@ -247,7 +247,7 @@ class DatabaseIntegrationTests(unittest.TestCase):
def test_rejected_record_replay_keeps_nonzero_cli_exit(self): def test_rejected_record_replay_keeps_nonzero_cli_exit(self):
path, _ = self.fixture(pages=[[{"list_id": "invalid"}]]) path, _ = self.fixture(pages=[[{"list_id": "invalid"}]])
with patch.dict(os.environ, {"DATABASE_URL": os.environ["TEST_DATABASE_URL"]}), patch( with patch.dict(os.environ, {"DATABASE_URL": os.environ["TEST_DATABASE_URL"]}), patch(
"sys.argv", ["car-traker-db", "import", str(path)] "sys.argv", ["car-tracker-db", "import", str(path)]
), patch("sys.stdout", new_callable=io.StringIO) as output: ), patch("sys.stdout", new_callable=io.StringIO) as output:
self.assertEqual(database.main(), 2) self.assertEqual(database.main(), 2)
self.assertEqual(database.main(), 2) self.assertEqual(database.main(), 2)

View File

@ -10,7 +10,7 @@ import tempfile
import unittest import unittest
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
from car_traker import pipeline from car_tracker import pipeline
class PipelineTests(unittest.TestCase): class PipelineTests(unittest.TestCase):

View File

@ -3,7 +3,7 @@ from datetime import datetime, timezone
from decimal import Decimal, localcontext from decimal import Decimal, localcontext
import unittest import unittest
from car_traker.transform import clean_observation from car_tracker.transform import clean_observation
OBSERVED_AT = datetime(2026, 9, 8, tzinfo=timezone.utc) OBSERVED_AT = datetime(2026, 9, 8, tzinfo=timezone.utc)

2
uv.lock generated
View File

@ -3,7 +3,7 @@ revision = 3
requires-python = ">=3.13" requires-python = ">=3.13"
[[package]] [[package]]
name = "car-traker" name = "car-tracker"
version = "0.1.0" version = "0.1.0"
source = { editable = "." } source = { editable = "." }
dependencies = [ dependencies = [