standardize car-tracker naming and verify container entrypoints
All checks were successful
CI / Lint, PostgreSQL tests, and package (push) Successful in 12s
All checks were successful
CI / Lint, PostgreSQL tests, and package (push) Successful in 12s
This commit is contained in:
Antoine Vandeplanque
parent
5f42c24153
commit
a353e1c683
@ -5,8 +5,10 @@
|
||||
!uv.lock
|
||||
!README.md
|
||||
!src/
|
||||
!src/car_traker/
|
||||
!src/car_traker/**
|
||||
!src/car_tracker/
|
||||
!src/car_tracker/**
|
||||
src/**/__pycache__/
|
||||
src/**/*.pyc
|
||||
# Keep locally generated packaging metadata out of the build context.
|
||||
src/**/*.egg-info/
|
||||
!searches.toml
|
||||
|
||||
@ -15,15 +15,15 @@ jobs:
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: ci-test-only
|
||||
POSTGRES_DB: car_traker_test
|
||||
POSTGRES_DB: car_tracker_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres -d car_traker_test"
|
||||
--health-cmd "pg_isready -U postgres -d car_tracker_test"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 12
|
||||
env:
|
||||
# Disposable service only. Never replace this with the homelab database URL.
|
||||
TEST_DATABASE_URL: postgresql://postgres:ci-test-only@postgres:5432/car_traker_test
|
||||
TEST_DATABASE_URL: postgresql://postgres:ci-test-only@postgres:5432/car_tracker_test
|
||||
UV_NO_PROGRESS: "1"
|
||||
steps:
|
||||
- name: Check out repository
|
||||
@ -55,6 +55,6 @@ jobs:
|
||||
|
||||
- name: Check installed wheel and packaged migrations
|
||||
run: |
|
||||
uv venv /tmp/car-traker-wheel
|
||||
uv pip install --python /tmp/car-traker-wheel/bin/python --no-deps dist/*.whl
|
||||
/tmp/car-traker-wheel/bin/python -c 'from importlib.resources import files; assert files("car_traker").joinpath("migrations", "001_silver.sql").is_file(), "SQL migration missing from wheel"'
|
||||
uv venv /tmp/car-tracker-wheel
|
||||
uv pip install --python /tmp/car-tracker-wheel/bin/python --no-deps dist/*.whl
|
||||
/tmp/car-tracker-wheel/bin/python -c 'from importlib.resources import files; assert files("car_tracker").joinpath("migrations", "001_silver.sql").is_file(), "SQL migration missing from wheel"'
|
||||
|
||||
12
Dockerfile
12
Dockerfile
@ -22,9 +22,15 @@ RUN uv sync --locked --no-dev --no-install-project \
|
||||
&& mkdir /data \
|
||||
&& chown app:app /data
|
||||
COPY src/ ./src/
|
||||
RUN uv sync --locked --no-dev --no-editable && rm -rf /tmp/.cache
|
||||
# Fail the build early if an application executable or migration is missing.
|
||||
RUN uv sync --locked --no-dev --no-editable \
|
||||
&& rm -rf /tmp/.cache \
|
||||
&& for script in car-tracker car-tracker-db car-tracker-pipeline; do \
|
||||
test -x "/opt/venv/bin/${script}" || { echo "missing console script: ${script}" >&2; exit 1; }; \
|
||||
done \
|
||||
&& python -c 'from importlib.resources import files; assert files("car_tracker").joinpath("migrations", "001_silver.sql").is_file()'
|
||||
COPY searches.toml ./searches.toml
|
||||
|
||||
USER 1000:1000
|
||||
ENTRYPOINT ["python", "-m", "car_traker.container_entrypoint"]
|
||||
CMD ["car-traker-pipeline", "--headed", "--config", "/app/searches.toml", "--output", "/data", "--family", "saab-900", "--max-pages", "1", "--max-details", "1"]
|
||||
ENTRYPOINT ["python", "-m", "car_tracker.container_entrypoint"]
|
||||
CMD ["car-tracker-pipeline", "--headed", "--config", "/app/searches.toml", "--output", "/data", "--family", "saab-900", "--max-pages", "1", "--max-details", "1"]
|
||||
|
||||
@ -10,7 +10,7 @@ and precise location below. Seller/contact objects and photos remain excluded;
|
||||
free text may still contain personal information and must remain private until
|
||||
reviewed/redacted for downstream publication. Source coordinates may be approximate.
|
||||
|
||||
The collector now lives in `src/car_traker/`, storing separate search/detail
|
||||
The collector now lives in `src/car_tracker/`, storing separate search/detail
|
||||
observations in versioned compressed JSON Lines with a coverage manifest. See
|
||||
`README.md` for current commands, capture policy, retention, and verified live
|
||||
results. The original handoff below remains planning context, not a description
|
||||
@ -25,7 +25,7 @@ connection setup remain user actions documented in `docs/POSTGRESQL.md`; no home
|
||||
database has been accessed or modified during implementation.
|
||||
|
||||
The manual homelab job is now packaged in `Dockerfile` and `ops/app/compose.yaml`.
|
||||
`car-traker-pipeline` performs a database preflight, bounded collection, and an
|
||||
`car-tracker-pipeline` performs a database preflight, bounded collection, and an
|
||||
idempotent import of the exact returned manifest, retaining partial coverage and
|
||||
separate operational outcomes. There is no scheduler or automatic deployment.
|
||||
See `docs/DEPLOYMENT.md`; container tests use isolated PostgreSQL 17 and synthetic
|
||||
@ -65,10 +65,10 @@ identified.
|
||||
- PostgreSQL already runs on a Debian VM; reuse it rather than require a second
|
||||
production database.
|
||||
- Repository hosted on the user's own Gitea instance.
|
||||
- Current workspace: `/home/tonio/Workspace/perso/car-traker`.
|
||||
- Current workspace: `/home/tonio/Workspace/perso/car-tracker`.
|
||||
- At inspection, the repository contained `.gitignore`, `.python-version`,
|
||||
`main.py`, `pyproject.toml`, and an empty `README.md`.
|
||||
- `pyproject.toml` specifies Python >=3.13, project name `car-traker`, version
|
||||
- `pyproject.toml` specifies Python >=3.13, project name `car-tracker`, version
|
||||
`0.1.0`, and no dependencies.
|
||||
- The worktree was clean before this handoff document was added.
|
||||
- No application implementation was changed during the planning discussion.
|
||||
|
||||
20
README.md
20
README.md
@ -14,14 +14,14 @@ Python 3.13+ and `uv`:
|
||||
```sh
|
||||
uv sync
|
||||
uv run playwright install chromium
|
||||
uv run car-traker --family saab-900 --max-pages 1 --max-details 1 --headed
|
||||
uv run car-tracker --family saab-900 --max-pages 1 --max-details 1 --headed
|
||||
```
|
||||
|
||||
The smoke command deliberately caps coverage and returns exit code 1 if further
|
||||
pages or details exist. Collect all configured searches and their individual ads with:
|
||||
|
||||
```sh
|
||||
uv run car-traker --headed
|
||||
uv run car-tracker --headed
|
||||
```
|
||||
|
||||
Options: `--config searches.toml`, `--output data/raw/leboncoin`, `--family`,
|
||||
@ -47,9 +47,9 @@ results are intentionally preserved with their query provenance.
|
||||
## Project Structure
|
||||
|
||||
```text
|
||||
src/car_traker/ # Installed Python package
|
||||
src/car_tracker/ # Installed Python package
|
||||
__init__.py
|
||||
__main__.py # python -m car_traker entry point
|
||||
__main__.py # python -m car_tracker entry point
|
||||
collect.py # Collector and CLI
|
||||
database.py # PostgreSQL migrations and import CLI
|
||||
pipeline.py # Manual collect -> raw -> import job
|
||||
@ -69,7 +69,7 @@ data/ # Local snapshots (ignored by Git)
|
||||
|
||||
`uv sync` installs the package in editable mode from `src/`. Run commands from the
|
||||
repository root so the default configuration and data paths resolve correctly.
|
||||
`uv run python -m car_traker` also works as an alternative to `uv run car-traker`.
|
||||
`uv run python -m car_tracker` also works as an alternative to `uv run car-tracker`.
|
||||
|
||||
## Silver Import
|
||||
|
||||
@ -81,8 +81,8 @@ role/database setup, network access, and laptop connection configuration.
|
||||
Once the remote connection is configured:
|
||||
|
||||
```sh
|
||||
uv run car-traker-db migrate
|
||||
uv run car-traker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json
|
||||
uv run car-tracker-db migrate
|
||||
uv run car-tracker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json
|
||||
```
|
||||
|
||||
Importing an unchanged run again is a no-op. Search and detail observations retain
|
||||
@ -92,7 +92,7 @@ and validation issues. Generation/trim inference, redaction, and scheduling are
|
||||
|
||||
## Manual Homelab Job
|
||||
|
||||
`car-traker-pipeline` checks database connectivity/schema, collects raw observations,
|
||||
`car-tracker-pipeline` checks database connectivity/schema, collects raw observations,
|
||||
and imports that exact finalized run. Partial/failed collections remain marked as
|
||||
such even when their import succeeds. A lock prevents overlapping pipeline jobs on
|
||||
the same output directory, and `pipeline.json` records the import outcome separately
|
||||
@ -101,7 +101,7 @@ from the immutable raw manifest.
|
||||
For a laptop smoke run with the existing database connection configured:
|
||||
|
||||
```sh
|
||||
uv run car-traker-pipeline --headed --family saab-900 --max-pages 1 --max-details 1
|
||||
uv run car-tracker-pipeline --headed --family saab-900 --max-pages 1 --max-details 1
|
||||
```
|
||||
|
||||
For Docker on the homelab, follow [manual deployment](docs/DEPLOYMENT.md) first to
|
||||
@ -110,7 +110,7 @@ Then, from the homelab checkout:
|
||||
|
||||
```sh
|
||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml build app
|
||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-traker-db check
|
||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-tracker-db check
|
||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app
|
||||
```
|
||||
|
||||
|
||||
@ -13,7 +13,7 @@ One Docker-backed job, selected by the runner label `python-ci`, performs:
|
||||
2. Install uv 0.9.3 and Python 3.13, then `uv sync --locked --dev`.
|
||||
3. Run Ruff's correctness-focused checks over `src` and `tests`.
|
||||
4. Run the full unittest suite, including integration tests against a fresh
|
||||
PostgreSQL 17 service named `postgres` and database `car_traker_test`, matching
|
||||
PostgreSQL 17 service named `postgres` and database `car_tracker_test`, matching
|
||||
the homelab's PostgreSQL major version.
|
||||
5. Build the source distribution and wheel, install that wheel in an independent
|
||||
environment, and verify that its SQL migrations are packaged.
|
||||
|
||||
@ -110,8 +110,8 @@ necessary source/port through the firewall. Check `pg_hba_file_rules` for errors
|
||||
and reload authentication changes with `SELECT pg_reload_conf();` as an admin.
|
||||
Changing `listen_addresses` requires a PostgreSQL restart: coordinate downtime for
|
||||
the shared VM's clients rather than restarting casually. See
|
||||
[PostgreSQL troubleshooting](POSTGRESQL.md#connection-troubleshooting) for details;
|
||||
that document's `car_traker` names are examples, not this VM's `car_tracker` names.
|
||||
[PostgreSQL troubleshooting](POSTGRESQL.md#connection-troubleshooting) for details.
|
||||
Use the existing `car_tracker` database and role on this VM.
|
||||
|
||||
## Build And Smoke Run
|
||||
Validate without printing interpolated configuration, then build on the homelab:
|
||||
@ -119,7 +119,7 @@ Validate without printing interpolated configuration, then build on the homelab:
|
||||
```sh
|
||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml config --quiet
|
||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml build app
|
||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-traker-db check
|
||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-tracker-db check
|
||||
```
|
||||
|
||||
`check` is read-only connectivity testing, not a schema or migration-privilege
|
||||
@ -128,7 +128,7 @@ opening the browser. No new migrations are required for this milestone. If the
|
||||
existing packaged migrations have not yet been applied, apply them explicitly:
|
||||
|
||||
```sh
|
||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-traker-db migrate
|
||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-tracker-db migrate
|
||||
```
|
||||
|
||||
Migrations are not automatic. Once DB access/schema are ready, run the image's
|
||||
@ -144,7 +144,7 @@ smoke run, this is the explicit larger manual command:
|
||||
|
||||
```sh
|
||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app \
|
||||
car-traker-pipeline --headed --config /app/searches.toml --output /data \
|
||||
car-tracker-pipeline --headed --config /app/searches.toml --output /data \
|
||||
--family saab-900 --max-pages 3 --max-details 5
|
||||
```
|
||||
|
||||
@ -175,7 +175,7 @@ date/run ID printed by the pipeline:
|
||||
|
||||
```sh
|
||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app \
|
||||
car-traker-db import '/data/<date>/<run-id>/manifest.json'
|
||||
car-tracker-db import '/data/<date>/<run-id>/manifest.json'
|
||||
```
|
||||
|
||||
An identical replay reports `already_imported` without duplicating observations.
|
||||
|
||||
@ -6,7 +6,7 @@ Development and CLI execution currently happen on your laptop. Raw files remain
|
||||
on the laptop, and only the database connection crosses the network:
|
||||
|
||||
```text
|
||||
Laptop: raw JSONL -> Python silver importer -> Homelab VM: PostgreSQL/car_traker
|
||||
Laptop: raw JSONL -> Python silver importer -> Homelab VM: PostgreSQL/car_tracker
|
||||
```
|
||||
|
||||
You do not need to install PostgreSQL on the laptop or move the repository to the
|
||||
@ -22,18 +22,18 @@ Create a dedicated database and an application login. Run these commands as your
|
||||
PostgreSQL administrator, for example with `sudo -u postgres psql` on the Debian VM:
|
||||
|
||||
```sql
|
||||
CREATE ROLE car_traker LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;
|
||||
CREATE ROLE car_tracker LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;
|
||||
SET password_encryption = 'scram-sha-256';
|
||||
\password car_traker
|
||||
CREATE DATABASE car_traker OWNER car_traker;
|
||||
REVOKE ALL ON DATABASE car_traker FROM PUBLIC;
|
||||
\connect car_traker
|
||||
\password car_tracker
|
||||
CREATE DATABASE car_tracker OWNER car_tracker;
|
||||
REVOKE ALL ON DATABASE car_tracker FROM PUBLIC;
|
||||
\connect car_tracker
|
||||
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
|
||||
```
|
||||
|
||||
`\password` prompts locally without putting the password in SQL history. These are
|
||||
one-time provisioning commands; inspect existing roles/databases rather than
|
||||
rerunning blindly if the names already exist. The importer connects as `car_traker`,
|
||||
rerunning blindly if the names already exist. The importer connects as `car_tracker`,
|
||||
not `postgres`. Owning this dedicated database allows it to create its `silver`
|
||||
schema and run migrations without cluster-wide administrative privileges.
|
||||
|
||||
@ -47,7 +47,7 @@ Restrict the firewall to the importer machine's IP and PostgreSQL port, normally
|
||||
scoped `pg_hba.conf` rule, replacing `<IMPORTER_IP>` with the actual client address:
|
||||
|
||||
```text
|
||||
hostssl car_traker car_traker <IMPORTER_IP>/32 scram-sha-256
|
||||
hostssl car_tracker car_tracker <IMPORTER_IP>/32 scram-sha-256
|
||||
```
|
||||
|
||||
Rules are first-match; review existing rules and their order. Reload authentication
|
||||
@ -69,8 +69,8 @@ outside the repository. Prefer a password file rather than a password in command
|
||||
```sh
|
||||
export PGHOST=postgres.your-lan.example
|
||||
export PGPORT=5432
|
||||
export PGDATABASE=car_traker
|
||||
export PGUSER=car_traker
|
||||
export PGDATABASE=car_tracker
|
||||
export PGUSER=car_tracker
|
||||
export PGSSLMODE=verify-full
|
||||
export PGSSLROOTCERT=/path/to/your/postgres-ca.crt
|
||||
export PGPASSFILE="$HOME/.pgpass"
|
||||
@ -79,7 +79,7 @@ export PGPASSFILE="$HOME/.pgpass"
|
||||
The matching password-file entry has this form:
|
||||
|
||||
```text
|
||||
postgres.your-lan.example:5432:car_traker:car_traker:<your-password>
|
||||
postgres.your-lan.example:5432:car_tracker:car_tracker:<your-password>
|
||||
```
|
||||
|
||||
Protect it with `chmod 600 "$HOME/.pgpass"`. Escape `:` and `\` in passwords using
|
||||
@ -99,7 +99,7 @@ psql -c 'SELECT current_database(), current_user, version();'
|
||||
Alternatively, no local `psql` installation is needed for the read-only CLI check:
|
||||
|
||||
```sh
|
||||
uv run car-traker-db check
|
||||
uv run car-tracker-db check
|
||||
```
|
||||
|
||||
It connects with the same settings as migrations/imports and executes only `SELECT 1`.
|
||||
@ -114,7 +114,7 @@ contain credentials or private data. `no pg_hba.conf entry` means PostgreSQL was
|
||||
reached, but its access rules do not allow the client/database/role combination.
|
||||
The rule must use the **client IP seen by PostgreSQL**, which can be a VPN or NAT
|
||||
address rather than the laptop's Wi-Fi/LAN address. Use the actual database and
|
||||
role names you created; `car_traker` in this document is an example, not a hardcoded
|
||||
role names you created; `car_tracker` in this document is an example, not a hardcoded
|
||||
application requirement.
|
||||
|
||||
On the homelab, locate the active configuration and inspect parsed rules as an
|
||||
@ -130,7 +130,7 @@ Add the appropriate narrow `hostssl` rule in the file returned by `SHOW hba_file
|
||||
then run `SELECT pg_reload_conf();`. Check the `error` column and server logs if it
|
||||
still fails; a successful reload request alone does not prove every rule is valid.
|
||||
Do not change passwords or use `trust` to work around a missing HBA rule. Once access
|
||||
is permitted, rerun `car-traker-db check` from the laptop before migrating.
|
||||
is permitted, rerun `car-tracker-db check` from the laptop before migrating.
|
||||
|
||||
Other categories point to DNS/VPN, refused/timed-out connections, authentication,
|
||||
TLS verification, missing databases, or insufficient privileges. This check does
|
||||
@ -142,8 +142,8 @@ From the repository root, with the connection configured:
|
||||
|
||||
```sh
|
||||
uv sync
|
||||
uv run car-traker-db migrate
|
||||
uv run car-traker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json
|
||||
uv run car-tracker-db migrate
|
||||
uv run car-tracker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json
|
||||
```
|
||||
|
||||
The example uses the existing 35-search/one-detail live run. Choose any finalized
|
||||
@ -298,13 +298,13 @@ Database integration tests are skipped unless `TEST_DATABASE_URL` is set. They
|
||||
One local example (test password only; localhost-only port; ephemeral data):
|
||||
|
||||
```sh
|
||||
docker run --detach --rm --name car-traker-silver-test \
|
||||
--env POSTGRES_PASSWORD=local-test-only --env POSTGRES_DB=car_traker_test \
|
||||
docker run --detach --rm --name car-tracker-silver-test \
|
||||
--env POSTGRES_PASSWORD=local-test-only --env POSTGRES_DB=car_tracker_test \
|
||||
--publish 127.0.0.1:55432:5432 --tmpfs /var/lib/postgresql/data postgres:17
|
||||
docker exec car-traker-silver-test pg_isready -U postgres -d car_traker_test
|
||||
TEST_DATABASE_URL='postgresql://postgres:local-test-only@127.0.0.1:55432/car_traker_test' \
|
||||
docker exec car-tracker-silver-test pg_isready -U postgres -d car_tracker_test
|
||||
TEST_DATABASE_URL='postgresql://postgres:local-test-only@127.0.0.1:55432/car_tracker_test' \
|
||||
uv run python -m unittest discover -s tests -v
|
||||
docker stop car-traker-silver-test
|
||||
docker stop car-tracker-silver-test
|
||||
```
|
||||
|
||||
Wait until `pg_isready` succeeds before running the tests. This temporary container
|
||||
|
||||
@ -1,11 +1,13 @@
|
||||
name: car-traker-app
|
||||
name: car-tracker-app
|
||||
|
||||
services:
|
||||
app:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: Dockerfile
|
||||
image: car-traker:local
|
||||
# The reconcile agent sets CAR_TRACKER_IMAGE to an immutable CI-built tag.
|
||||
# The default keeps the documented local build-and-run path working.
|
||||
image: "${CAR_TRACKER_IMAGE:-car-tracker:local}"
|
||||
# One-shot manual job only. Never restart a blocked scraper automatically.
|
||||
restart: "no"
|
||||
init: true
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
[project]
|
||||
name = "car-traker"
|
||||
name = "car-tracker"
|
||||
version = "0.1.0"
|
||||
description = "French youngtimer asking-price observatory"
|
||||
readme = "README.md"
|
||||
@ -16,9 +16,9 @@ target-version = "py313"
|
||||
select = ["E4", "E7", "E9", "F"]
|
||||
|
||||
[project.scripts]
|
||||
car-traker = "car_traker.collect:main"
|
||||
car-traker-db = "car_traker.database:main"
|
||||
car-traker-pipeline = "car_traker.pipeline:main"
|
||||
car-tracker = "car_tracker.collect:main"
|
||||
car-tracker-db = "car_tracker.database:main"
|
||||
car-tracker-pipeline = "car_tracker.pipeline:main"
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=77"]
|
||||
@ -28,4 +28,4 @@ build-backend = "setuptools.build_meta"
|
||||
where = ["src"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
car_traker = ["migrations/*.sql"]
|
||||
car_tracker = ["migrations/*.sql"]
|
||||
|
||||
1
src/car_tracker/__init__.py
Normal file
1
src/car_tracker/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
"""Car-tracker collection and database tools."""
|
||||
@ -41,8 +41,16 @@ def main():
|
||||
child.send_signal(stopped)
|
||||
code = child.wait()
|
||||
return code if code >= 0 else 128 - code
|
||||
except FileNotFoundError:
|
||||
# Do not echo caller-supplied paths or arguments.
|
||||
message = ("Virtual display executable could not be found." if display is None
|
||||
else "Container command could not be found; check the executable name.")
|
||||
print(message, file=sys.stderr)
|
||||
return 3
|
||||
except (OSError, ValueError):
|
||||
print("Could not start the virtual display or container command.", file=sys.stderr)
|
||||
message = ("Could not start the virtual display." if display is None
|
||||
else "Could not start or wait for the container command.")
|
||||
print(message, file=sys.stderr)
|
||||
return 3
|
||||
finally:
|
||||
# Only clean up descendants after the command has finalized its files/transaction.
|
||||
@ -263,7 +263,7 @@ def migrate(conn):
|
||||
name text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now()
|
||||
)""")
|
||||
applied = dict(conn.execute("SELECT name, sha256 FROM silver.schema_migrations").fetchall())
|
||||
migrations = sorted(files("car_traker").joinpath("migrations").iterdir(), key=lambda p: p.name)
|
||||
migrations = sorted(files("car_tracker").joinpath("migrations").iterdir(), key=lambda p: p.name)
|
||||
known = {m.name for m in migrations if m.name.endswith(".sql")}
|
||||
if set(applied) - known:
|
||||
raise InputError("database_has_unknown_migrations")
|
||||
@ -367,7 +367,7 @@ def database_error_hint(exc):
|
||||
if exc.sqlstate == "42501":
|
||||
return "The database role lacks required privileges. Use the dedicated database owner for migrations/imports, not a superuser."
|
||||
if exc.sqlstate in ("42P01", "3F000"):
|
||||
return "A required database table/schema is missing. Run car-traker-db migrate against the same database first."
|
||||
return "A required database table/schema is missing. Run car-tracker-db migrate against the same database first."
|
||||
return "Check the connection configuration and PostgreSQL server logs. Raw error details are withheld to protect credentials and source data."
|
||||
|
||||
|
||||
@ -13,8 +13,8 @@ from unittest.mock import patch
|
||||
|
||||
from psycopg import sql
|
||||
|
||||
from car_traker import database, pipeline
|
||||
from car_traker.collect import search_url
|
||||
from car_tracker import database, pipeline
|
||||
from car_tracker.collect import search_url
|
||||
import test_database_integration as database_tests
|
||||
|
||||
|
||||
@ -92,7 +92,7 @@ class BrowserPipelineTests(unittest.TestCase):
|
||||
environment["DATABASE_URL"] = os.environ["TEST_DATABASE_URL"]
|
||||
with patch.dict(os.environ, environment, clear=True), patch.object(
|
||||
BrowserContext, "new_page", new_page,
|
||||
), patch("car_traker.collect.time.sleep"), redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
), patch("car_tracker.collect.time.sleep"), redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
code = pipeline.main([
|
||||
"--headed", "--config", str(self.config), "--output", str(self.output),
|
||||
"--delay", "1", "--max-pages", "1", "--max-details", "1",
|
||||
|
||||
@ -5,7 +5,7 @@ import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from car_traker.collect import SourceAccessError, collect, minimize, parse_detail, parse_search, search_url
|
||||
from car_tracker.collect import SourceAccessError, collect, minimize, parse_detail, parse_search, search_url
|
||||
|
||||
|
||||
def payload(ids=(123,), total=1, pages=1):
|
||||
@ -85,7 +85,7 @@ class CollectorTests(unittest.TestCase):
|
||||
raise result
|
||||
return result
|
||||
|
||||
with patch("car_traker.collect.time.sleep"):
|
||||
with patch("car_tracker.collect.time.sleep"):
|
||||
kwargs.setdefault("search_only", True)
|
||||
path, status = collect(fetch, [{"family": "synthetic", "text": "test"}],
|
||||
Path(temporary.name), **kwargs)
|
||||
@ -221,7 +221,7 @@ class CollectorTests(unittest.TestCase):
|
||||
|
||||
def test_interrupt_during_delay_does_not_count_as_detail_attempt(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
with patch("car_traker.collect.time.sleep", side_effect=KeyboardInterrupt()):
|
||||
with patch("car_tracker.collect.time.sleep", side_effect=KeyboardInterrupt()):
|
||||
path, status = collect(lambda url: payload(),
|
||||
[{"family": "synthetic", "text": "test"}], Path(directory))
|
||||
manifest = json.loads(path.read_text())
|
||||
@ -233,7 +233,7 @@ class CollectorTests(unittest.TestCase):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
searches = [{"family": "synthetic", "text": "test"},
|
||||
{"family": "synthetic", "text": "test-2"}]
|
||||
with patch("car_traker.collect.time.sleep") as sleep:
|
||||
with patch("car_tracker.collect.time.sleep") as sleep:
|
||||
responses = iter([payload(), detail_payload(), payload()])
|
||||
urls = []
|
||||
|
||||
|
||||
@ -5,7 +5,7 @@ from tempfile import TemporaryDirectory
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from car_traker.collect import collect_browser
|
||||
from car_tracker.collect import collect_browser
|
||||
|
||||
|
||||
class BrowserCleanupTests(unittest.TestCase):
|
||||
@ -18,7 +18,7 @@ class BrowserCleanupTests(unittest.TestCase):
|
||||
max_pages=1, delay=1, search_only=True, max_details=None)
|
||||
self.playwright = self.enterContext(patch("playwright.sync_api.sync_playwright"))
|
||||
self.browser = self.playwright.return_value.__enter__.return_value.chromium.launch.return_value
|
||||
self.collect = self.enterContext(patch("car_traker.collect.collect", return_value=(self.path, "complete")))
|
||||
self.collect = self.enterContext(patch("car_tracker.collect.collect", return_value=(self.path, "complete")))
|
||||
self.stderr = self.enterContext(patch("sys.stderr", new_callable=io.StringIO))
|
||||
|
||||
def test_browser_close_failure_retains_finalized_result(self):
|
||||
|
||||
@ -5,9 +5,9 @@ import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from car_traker import container_entrypoint
|
||||
from car_tracker import container_entrypoint
|
||||
|
||||
|
||||
class ContainerEntrypointTests(unittest.TestCase):
|
||||
@ -15,12 +15,35 @@ class ContainerEntrypointTests(unittest.TestCase):
|
||||
with patch("sys.argv", ["entrypoint"]), patch("sys.stderr", new_callable=io.StringIO):
|
||||
self.assertEqual(container_entrypoint.main(), 3)
|
||||
|
||||
def test_missing_display_has_distinct_safe_error(self):
|
||||
error = FileNotFoundError(2, "missing", "/private/display")
|
||||
with patch("sys.argv", ["entrypoint", "command"]), \
|
||||
patch("sys.stderr", new_callable=io.StringIO) as stderr, \
|
||||
patch.object(container_entrypoint.subprocess, "Popen", side_effect=error):
|
||||
self.assertEqual(container_entrypoint.main(), 3)
|
||||
self.assertIn("Virtual display executable could not be found", stderr.getvalue())
|
||||
self.assertNotIn("/private/display", stderr.getvalue())
|
||||
|
||||
def test_missing_command_has_distinct_safe_error_and_cleans_display(self):
|
||||
display = Mock()
|
||||
error = FileNotFoundError(2, "missing", "/private/command")
|
||||
with patch("sys.argv", ["entrypoint", "/private/command"]), \
|
||||
patch("sys.stderr", new_callable=io.StringIO) as stderr, \
|
||||
patch.object(container_entrypoint.subprocess, "Popen", side_effect=[display, error]), \
|
||||
patch.object(container_entrypoint.select, "select", return_value=([3], [], [])), \
|
||||
patch.object(container_entrypoint.os, "read", return_value=b"0\n"):
|
||||
self.assertEqual(container_entrypoint.main(), 3)
|
||||
self.assertIn("Container command could not be found", stderr.getvalue())
|
||||
self.assertNotIn("/private/command", stderr.getvalue())
|
||||
display.terminate.assert_called_once()
|
||||
display.wait.assert_called_once()
|
||||
|
||||
|
||||
@unittest.skipUnless(os.environ.get("RUN_BROWSER_TESTS") == "1", "Xvfb container tests are opt-in")
|
||||
class VirtualDisplayTests(unittest.TestCase):
|
||||
def test_child_exit_code_is_preserved(self):
|
||||
result = subprocess.run([
|
||||
sys.executable, "-m", "car_traker.container_entrypoint",
|
||||
sys.executable, "-m", "car_tracker.container_entrypoint",
|
||||
sys.executable, "-c", "raise SystemExit(7)",
|
||||
], capture_output=True, text=True, timeout=20)
|
||||
self.assertEqual(result.returncode, 7, result.stderr)
|
||||
@ -39,7 +62,7 @@ while True:
|
||||
signal.pause()
|
||||
"""
|
||||
process = subprocess.Popen([
|
||||
sys.executable, "-m", "car_traker.container_entrypoint",
|
||||
sys.executable, "-m", "car_tracker.container_entrypoint",
|
||||
sys.executable, "-c", command,
|
||||
], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
try:
|
||||
|
||||
@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import psycopg
|
||||
|
||||
from car_traker.database import database_error_hint, main
|
||||
from car_tracker.database import database_error_hint, main
|
||||
|
||||
|
||||
class DatabaseCliTests(unittest.TestCase):
|
||||
@ -37,10 +37,10 @@ class DatabaseCliTests(unittest.TestCase):
|
||||
def test_check_only_runs_select(self):
|
||||
connection = MagicMock()
|
||||
with patch.dict("os.environ", {"PGHOST": "synthetic-host"}, clear=True), patch(
|
||||
"sys.argv", ["car-traker-db", "check"]
|
||||
), patch("car_traker.database.psycopg.connect") as connect, patch(
|
||||
"sys.argv", ["car-tracker-db", "check"]
|
||||
), patch("car_tracker.database.psycopg.connect") as connect, patch(
|
||||
"sys.stdout", new_callable=io.StringIO
|
||||
) as output, patch("car_traker.database.migrate") as migrate:
|
||||
) as output, patch("car_tracker.database.migrate") as migrate:
|
||||
connect.return_value.__enter__.return_value = connection
|
||||
self.assertEqual(main(), 0)
|
||||
connection.execute.assert_called_once_with("SELECT 1")
|
||||
@ -50,8 +50,8 @@ class DatabaseCliTests(unittest.TestCase):
|
||||
def test_hba_failure_is_actionable_without_echoing_credentials(self):
|
||||
exc = psycopg.OperationalError('no pg_hba.conf entry for host "private-host"; password=synthetic-secret')
|
||||
with patch.dict("os.environ", {"PGHOST": "synthetic-host"}, clear=True), patch(
|
||||
"sys.argv", ["car-traker-db", "check"]
|
||||
), patch("car_traker.database.psycopg.connect", side_effect=exc), patch(
|
||||
"sys.argv", ["car-tracker-db", "check"]
|
||||
), patch("car_tracker.database.psycopg.connect", side_effect=exc), patch(
|
||||
"sys.stderr", new_callable=io.StringIO
|
||||
) as output:
|
||||
self.assertEqual(main(), 1)
|
||||
@ -61,9 +61,9 @@ class DatabaseCliTests(unittest.TestCase):
|
||||
|
||||
def test_no_implicit_local_connection(self):
|
||||
with patch.dict("os.environ", {}, clear=True), patch(
|
||||
"sys.argv", ["car-traker-db", "check"]
|
||||
"sys.argv", ["car-tracker-db", "check"]
|
||||
), patch("sys.stderr", new_callable=io.StringIO), patch(
|
||||
"car_traker.database.psycopg.connect"
|
||||
"car_tracker.database.psycopg.connect"
|
||||
) as connect, self.assertRaises(SystemExit):
|
||||
main()
|
||||
connect.assert_not_called()
|
||||
|
||||
@ -6,8 +6,8 @@ from tempfile import TemporaryDirectory
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from car_traker.collect import collect, write_json
|
||||
from car_traker.database import InputError, load_json, load_run, prepare_record
|
||||
from car_tracker.collect import collect, write_json
|
||||
from car_tracker.database import InputError, load_json, load_run, prepare_record
|
||||
|
||||
|
||||
class RawInputTests(unittest.TestCase):
|
||||
@ -22,7 +22,7 @@ class RawInputTests(unittest.TestCase):
|
||||
"list_id": 123, "price": [1000], "body": "Synthetic\u2028text\u2029only",
|
||||
}],
|
||||
}}}})
|
||||
with patch("car_traker.collect.time.sleep"):
|
||||
with patch("car_tracker.collect.time.sleep"):
|
||||
path, _ = collect(lambda url: source, [{"family": "synthetic", "text": "test"}],
|
||||
self.root, search_only=True)
|
||||
return path, json.loads(path.read_text())
|
||||
|
||||
@ -19,8 +19,8 @@ import psycopg
|
||||
from psycopg import sql
|
||||
from psycopg.conninfo import conninfo_to_dict
|
||||
|
||||
from car_traker import database
|
||||
from car_traker.collect import search_url, write_json, write_records
|
||||
from car_tracker import database
|
||||
from car_tracker.collect import search_url, write_json, write_records
|
||||
|
||||
|
||||
@unittest.skipUnless(os.environ.get("TEST_DATABASE_URL"), "TEST_DATABASE_URL is not set")
|
||||
@ -247,7 +247,7 @@ class DatabaseIntegrationTests(unittest.TestCase):
|
||||
def test_rejected_record_replay_keeps_nonzero_cli_exit(self):
|
||||
path, _ = self.fixture(pages=[[{"list_id": "invalid"}]])
|
||||
with patch.dict(os.environ, {"DATABASE_URL": os.environ["TEST_DATABASE_URL"]}), patch(
|
||||
"sys.argv", ["car-traker-db", "import", str(path)]
|
||||
"sys.argv", ["car-tracker-db", "import", str(path)]
|
||||
), patch("sys.stdout", new_callable=io.StringIO) as output:
|
||||
self.assertEqual(database.main(), 2)
|
||||
self.assertEqual(database.main(), 2)
|
||||
|
||||
@ -10,7 +10,7 @@ import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from car_traker import pipeline
|
||||
from car_tracker import pipeline
|
||||
|
||||
|
||||
class PipelineTests(unittest.TestCase):
|
||||
|
||||
@ -3,7 +3,7 @@ from datetime import datetime, timezone
|
||||
from decimal import Decimal, localcontext
|
||||
import unittest
|
||||
|
||||
from car_traker.transform import clean_observation
|
||||
from car_tracker.transform import clean_observation
|
||||
|
||||
|
||||
OBSERVED_AT = datetime(2026, 9, 8, tzinfo=timezone.utc)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user