standardize car-tracker naming and verify container entrypoints
All checks were successful
CI / Lint, PostgreSQL tests, and package (push) Successful in 12s
All checks were successful
CI / Lint, PostgreSQL tests, and package (push) Successful in 12s
This commit is contained in:
parent
5f42c24153
commit
a353e1c683
@ -5,8 +5,10 @@
|
|||||||
!uv.lock
|
!uv.lock
|
||||||
!README.md
|
!README.md
|
||||||
!src/
|
!src/
|
||||||
!src/car_traker/
|
!src/car_tracker/
|
||||||
!src/car_traker/**
|
!src/car_tracker/**
|
||||||
src/**/__pycache__/
|
src/**/__pycache__/
|
||||||
src/**/*.pyc
|
src/**/*.pyc
|
||||||
|
# Keep locally generated packaging metadata out of the build context.
|
||||||
|
src/**/*.egg-info/
|
||||||
!searches.toml
|
!searches.toml
|
||||||
|
|||||||
@ -15,15 +15,15 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
POSTGRES_USER: postgres
|
POSTGRES_USER: postgres
|
||||||
POSTGRES_PASSWORD: ci-test-only
|
POSTGRES_PASSWORD: ci-test-only
|
||||||
POSTGRES_DB: car_traker_test
|
POSTGRES_DB: car_tracker_test
|
||||||
options: >-
|
options: >-
|
||||||
--health-cmd "pg_isready -U postgres -d car_traker_test"
|
--health-cmd "pg_isready -U postgres -d car_tracker_test"
|
||||||
--health-interval 5s
|
--health-interval 5s
|
||||||
--health-timeout 5s
|
--health-timeout 5s
|
||||||
--health-retries 12
|
--health-retries 12
|
||||||
env:
|
env:
|
||||||
# Disposable service only. Never replace this with the homelab database URL.
|
# Disposable service only. Never replace this with the homelab database URL.
|
||||||
TEST_DATABASE_URL: postgresql://postgres:ci-test-only@postgres:5432/car_traker_test
|
TEST_DATABASE_URL: postgresql://postgres:ci-test-only@postgres:5432/car_tracker_test
|
||||||
UV_NO_PROGRESS: "1"
|
UV_NO_PROGRESS: "1"
|
||||||
steps:
|
steps:
|
||||||
- name: Check out repository
|
- name: Check out repository
|
||||||
@ -55,6 +55,6 @@ jobs:
|
|||||||
|
|
||||||
- name: Check installed wheel and packaged migrations
|
- name: Check installed wheel and packaged migrations
|
||||||
run: |
|
run: |
|
||||||
uv venv /tmp/car-traker-wheel
|
uv venv /tmp/car-tracker-wheel
|
||||||
uv pip install --python /tmp/car-traker-wheel/bin/python --no-deps dist/*.whl
|
uv pip install --python /tmp/car-tracker-wheel/bin/python --no-deps dist/*.whl
|
||||||
/tmp/car-traker-wheel/bin/python -c 'from importlib.resources import files; assert files("car_traker").joinpath("migrations", "001_silver.sql").is_file(), "SQL migration missing from wheel"'
|
/tmp/car-tracker-wheel/bin/python -c 'from importlib.resources import files; assert files("car_tracker").joinpath("migrations", "001_silver.sql").is_file(), "SQL migration missing from wheel"'
|
||||||
|
|||||||
12
Dockerfile
12
Dockerfile
@ -22,9 +22,15 @@ RUN uv sync --locked --no-dev --no-install-project \
|
|||||||
&& mkdir /data \
|
&& mkdir /data \
|
||||||
&& chown app:app /data
|
&& chown app:app /data
|
||||||
COPY src/ ./src/
|
COPY src/ ./src/
|
||||||
RUN uv sync --locked --no-dev --no-editable && rm -rf /tmp/.cache
|
# Fail the build early if an application executable or migration is missing.
|
||||||
|
RUN uv sync --locked --no-dev --no-editable \
|
||||||
|
&& rm -rf /tmp/.cache \
|
||||||
|
&& for script in car-tracker car-tracker-db car-tracker-pipeline; do \
|
||||||
|
test -x "/opt/venv/bin/${script}" || { echo "missing console script: ${script}" >&2; exit 1; }; \
|
||||||
|
done \
|
||||||
|
&& python -c 'from importlib.resources import files; assert files("car_tracker").joinpath("migrations", "001_silver.sql").is_file()'
|
||||||
COPY searches.toml ./searches.toml
|
COPY searches.toml ./searches.toml
|
||||||
|
|
||||||
USER 1000:1000
|
USER 1000:1000
|
||||||
ENTRYPOINT ["python", "-m", "car_traker.container_entrypoint"]
|
ENTRYPOINT ["python", "-m", "car_tracker.container_entrypoint"]
|
||||||
CMD ["car-traker-pipeline", "--headed", "--config", "/app/searches.toml", "--output", "/data", "--family", "saab-900", "--max-pages", "1", "--max-details", "1"]
|
CMD ["car-tracker-pipeline", "--headed", "--config", "/app/searches.toml", "--output", "/data", "--family", "saab-900", "--max-pages", "1", "--max-details", "1"]
|
||||||
|
|||||||
@ -10,7 +10,7 @@ and precise location below. Seller/contact objects and photos remain excluded;
|
|||||||
free text may still contain personal information and must remain private until
|
free text may still contain personal information and must remain private until
|
||||||
reviewed/redacted for downstream publication. Source coordinates may be approximate.
|
reviewed/redacted for downstream publication. Source coordinates may be approximate.
|
||||||
|
|
||||||
The collector now lives in `src/car_traker/`, storing separate search/detail
|
The collector now lives in `src/car_tracker/`, storing separate search/detail
|
||||||
observations in versioned compressed JSON Lines with a coverage manifest. See
|
observations in versioned compressed JSON Lines with a coverage manifest. See
|
||||||
`README.md` for current commands, capture policy, retention, and verified live
|
`README.md` for current commands, capture policy, retention, and verified live
|
||||||
results. The original handoff below remains planning context, not a description
|
results. The original handoff below remains planning context, not a description
|
||||||
@ -25,7 +25,7 @@ connection setup remain user actions documented in `docs/POSTGRESQL.md`; no home
|
|||||||
database has been accessed or modified during implementation.
|
database has been accessed or modified during implementation.
|
||||||
|
|
||||||
The manual homelab job is now packaged in `Dockerfile` and `ops/app/compose.yaml`.
|
The manual homelab job is now packaged in `Dockerfile` and `ops/app/compose.yaml`.
|
||||||
`car-traker-pipeline` performs a database preflight, bounded collection, and an
|
`car-tracker-pipeline` performs a database preflight, bounded collection, and an
|
||||||
idempotent import of the exact returned manifest, retaining partial coverage and
|
idempotent import of the exact returned manifest, retaining partial coverage and
|
||||||
separate operational outcomes. There is no scheduler or automatic deployment.
|
separate operational outcomes. There is no scheduler or automatic deployment.
|
||||||
See `docs/DEPLOYMENT.md`; container tests use isolated PostgreSQL 17 and synthetic
|
See `docs/DEPLOYMENT.md`; container tests use isolated PostgreSQL 17 and synthetic
|
||||||
@ -65,10 +65,10 @@ identified.
|
|||||||
- PostgreSQL already runs on a Debian VM; reuse it rather than require a second
|
- PostgreSQL already runs on a Debian VM; reuse it rather than require a second
|
||||||
production database.
|
production database.
|
||||||
- Repository hosted on the user's own Gitea instance.
|
- Repository hosted on the user's own Gitea instance.
|
||||||
- Current workspace: `/home/tonio/Workspace/perso/car-traker`.
|
- Current workspace: `/home/tonio/Workspace/perso/car-tracker`.
|
||||||
- At inspection, the repository contained `.gitignore`, `.python-version`,
|
- At inspection, the repository contained `.gitignore`, `.python-version`,
|
||||||
`main.py`, `pyproject.toml`, and an empty `README.md`.
|
`main.py`, `pyproject.toml`, and an empty `README.md`.
|
||||||
- `pyproject.toml` specifies Python >=3.13, project name `car-traker`, version
|
- `pyproject.toml` specifies Python >=3.13, project name `car-tracker`, version
|
||||||
`0.1.0`, and no dependencies.
|
`0.1.0`, and no dependencies.
|
||||||
- The worktree was clean before this handoff document was added.
|
- The worktree was clean before this handoff document was added.
|
||||||
- No application implementation was changed during the planning discussion.
|
- No application implementation was changed during the planning discussion.
|
||||||
|
|||||||
20
README.md
20
README.md
@ -14,14 +14,14 @@ Python 3.13+ and `uv`:
|
|||||||
```sh
|
```sh
|
||||||
uv sync
|
uv sync
|
||||||
uv run playwright install chromium
|
uv run playwright install chromium
|
||||||
uv run car-traker --family saab-900 --max-pages 1 --max-details 1 --headed
|
uv run car-tracker --family saab-900 --max-pages 1 --max-details 1 --headed
|
||||||
```
|
```
|
||||||
|
|
||||||
The smoke command deliberately caps coverage and returns exit code 1 if further
|
The smoke command deliberately caps coverage and returns exit code 1 if further
|
||||||
pages or details exist. Collect all configured searches and their individual ads with:
|
pages or details exist. Collect all configured searches and their individual ads with:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
uv run car-traker --headed
|
uv run car-tracker --headed
|
||||||
```
|
```
|
||||||
|
|
||||||
Options: `--config searches.toml`, `--output data/raw/leboncoin`, `--family`,
|
Options: `--config searches.toml`, `--output data/raw/leboncoin`, `--family`,
|
||||||
@ -47,9 +47,9 @@ results are intentionally preserved with their query provenance.
|
|||||||
## Project Structure
|
## Project Structure
|
||||||
|
|
||||||
```text
|
```text
|
||||||
src/car_traker/ # Installed Python package
|
src/car_tracker/ # Installed Python package
|
||||||
__init__.py
|
__init__.py
|
||||||
__main__.py # python -m car_traker entry point
|
__main__.py # python -m car_tracker entry point
|
||||||
collect.py # Collector and CLI
|
collect.py # Collector and CLI
|
||||||
database.py # PostgreSQL migrations and import CLI
|
database.py # PostgreSQL migrations and import CLI
|
||||||
pipeline.py # Manual collect -> raw -> import job
|
pipeline.py # Manual collect -> raw -> import job
|
||||||
@ -69,7 +69,7 @@ data/ # Local snapshots (ignored by Git)
|
|||||||
|
|
||||||
`uv sync` installs the package in editable mode from `src/`. Run commands from the
|
`uv sync` installs the package in editable mode from `src/`. Run commands from the
|
||||||
repository root so the default configuration and data paths resolve correctly.
|
repository root so the default configuration and data paths resolve correctly.
|
||||||
`uv run python -m car_traker` also works as an alternative to `uv run car-traker`.
|
`uv run python -m car_tracker` also works as an alternative to `uv run car-tracker`.
|
||||||
|
|
||||||
## Silver Import
|
## Silver Import
|
||||||
|
|
||||||
@ -81,8 +81,8 @@ role/database setup, network access, and laptop connection configuration.
|
|||||||
Once the remote connection is configured:
|
Once the remote connection is configured:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
uv run car-traker-db migrate
|
uv run car-tracker-db migrate
|
||||||
uv run car-traker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json
|
uv run car-tracker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json
|
||||||
```
|
```
|
||||||
|
|
||||||
Importing an unchanged run again is a no-op. Search and detail observations retain
|
Importing an unchanged run again is a no-op. Search and detail observations retain
|
||||||
@ -92,7 +92,7 @@ and validation issues. Generation/trim inference, redaction, and scheduling are
|
|||||||
|
|
||||||
## Manual Homelab Job
|
## Manual Homelab Job
|
||||||
|
|
||||||
`car-traker-pipeline` checks database connectivity/schema, collects raw observations,
|
`car-tracker-pipeline` checks database connectivity/schema, collects raw observations,
|
||||||
and imports that exact finalized run. Partial/failed collections remain marked as
|
and imports that exact finalized run. Partial/failed collections remain marked as
|
||||||
such even when their import succeeds. A lock prevents overlapping pipeline jobs on
|
such even when their import succeeds. A lock prevents overlapping pipeline jobs on
|
||||||
the same output directory, and `pipeline.json` records the import outcome separately
|
the same output directory, and `pipeline.json` records the import outcome separately
|
||||||
@ -101,7 +101,7 @@ from the immutable raw manifest.
|
|||||||
For a laptop smoke run with the existing database connection configured:
|
For a laptop smoke run with the existing database connection configured:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
uv run car-traker-pipeline --headed --family saab-900 --max-pages 1 --max-details 1
|
uv run car-tracker-pipeline --headed --family saab-900 --max-pages 1 --max-details 1
|
||||||
```
|
```
|
||||||
|
|
||||||
For Docker on the homelab, follow [manual deployment](docs/DEPLOYMENT.md) first to
|
For Docker on the homelab, follow [manual deployment](docs/DEPLOYMENT.md) first to
|
||||||
@ -110,7 +110,7 @@ Then, from the homelab checkout:
|
|||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml build app
|
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml build app
|
||||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-traker-db check
|
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-tracker-db check
|
||||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app
|
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@ -13,7 +13,7 @@ One Docker-backed job, selected by the runner label `python-ci`, performs:
|
|||||||
2. Install uv 0.9.3 and Python 3.13, then `uv sync --locked --dev`.
|
2. Install uv 0.9.3 and Python 3.13, then `uv sync --locked --dev`.
|
||||||
3. Run Ruff's correctness-focused checks over `src` and `tests`.
|
3. Run Ruff's correctness-focused checks over `src` and `tests`.
|
||||||
4. Run the full unittest suite, including integration tests against a fresh
|
4. Run the full unittest suite, including integration tests against a fresh
|
||||||
PostgreSQL 17 service named `postgres` and database `car_traker_test`, matching
|
PostgreSQL 17 service named `postgres` and database `car_tracker_test`, matching
|
||||||
the homelab's PostgreSQL major version.
|
the homelab's PostgreSQL major version.
|
||||||
5. Build the source distribution and wheel, install that wheel in an independent
|
5. Build the source distribution and wheel, install that wheel in an independent
|
||||||
environment, and verify that its SQL migrations are packaged.
|
environment, and verify that its SQL migrations are packaged.
|
||||||
|
|||||||
@ -110,8 +110,8 @@ necessary source/port through the firewall. Check `pg_hba_file_rules` for errors
|
|||||||
and reload authentication changes with `SELECT pg_reload_conf();` as an admin.
|
and reload authentication changes with `SELECT pg_reload_conf();` as an admin.
|
||||||
Changing `listen_addresses` requires a PostgreSQL restart: coordinate downtime for
|
Changing `listen_addresses` requires a PostgreSQL restart: coordinate downtime for
|
||||||
the shared VM's clients rather than restarting casually. See
|
the shared VM's clients rather than restarting casually. See
|
||||||
[PostgreSQL troubleshooting](POSTGRESQL.md#connection-troubleshooting) for details;
|
[PostgreSQL troubleshooting](POSTGRESQL.md#connection-troubleshooting) for details.
|
||||||
that document's `car_traker` names are examples, not this VM's `car_tracker` names.
|
Use the existing `car_tracker` database and role on this VM.
|
||||||
|
|
||||||
## Build And Smoke Run
|
## Build And Smoke Run
|
||||||
Validate without printing interpolated configuration, then build on the homelab:
|
Validate without printing interpolated configuration, then build on the homelab:
|
||||||
@ -119,7 +119,7 @@ Validate without printing interpolated configuration, then build on the homelab:
|
|||||||
```sh
|
```sh
|
||||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml config --quiet
|
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml config --quiet
|
||||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml build app
|
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml build app
|
||||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-traker-db check
|
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-tracker-db check
|
||||||
```
|
```
|
||||||
|
|
||||||
`check` is read-only connectivity testing, not a schema or migration-privilege
|
`check` is read-only connectivity testing, not a schema or migration-privilege
|
||||||
@ -128,7 +128,7 @@ opening the browser. No new migrations are required for this milestone. If the
|
|||||||
existing packaged migrations have not yet been applied, apply them explicitly:
|
existing packaged migrations have not yet been applied, apply them explicitly:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-traker-db migrate
|
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app car-tracker-db migrate
|
||||||
```
|
```
|
||||||
|
|
||||||
Migrations are not automatic. Once DB access/schema are ready, run the image's
|
Migrations are not automatic. Once DB access/schema are ready, run the image's
|
||||||
@ -144,7 +144,7 @@ smoke run, this is the explicit larger manual command:
|
|||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app \
|
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app \
|
||||||
car-traker-pipeline --headed --config /app/searches.toml --output /data \
|
car-tracker-pipeline --headed --config /app/searches.toml --output /data \
|
||||||
--family saab-900 --max-pages 3 --max-details 5
|
--family saab-900 --max-pages 3 --max-details 5
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -175,7 +175,7 @@ date/run ID printed by the pipeline:
|
|||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app \
|
docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app \
|
||||||
car-traker-db import '/data/<date>/<run-id>/manifest.json'
|
car-tracker-db import '/data/<date>/<run-id>/manifest.json'
|
||||||
```
|
```
|
||||||
|
|
||||||
An identical replay reports `already_imported` without duplicating observations.
|
An identical replay reports `already_imported` without duplicating observations.
|
||||||
|
|||||||
@ -6,7 +6,7 @@ Development and CLI execution currently happen on your laptop. Raw files remain
|
|||||||
on the laptop, and only the database connection crosses the network:
|
on the laptop, and only the database connection crosses the network:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
Laptop: raw JSONL -> Python silver importer -> Homelab VM: PostgreSQL/car_traker
|
Laptop: raw JSONL -> Python silver importer -> Homelab VM: PostgreSQL/car_tracker
|
||||||
```
|
```
|
||||||
|
|
||||||
You do not need to install PostgreSQL on the laptop or move the repository to the
|
You do not need to install PostgreSQL on the laptop or move the repository to the
|
||||||
@ -22,18 +22,18 @@ Create a dedicated database and an application login. Run these commands as your
|
|||||||
PostgreSQL administrator, for example with `sudo -u postgres psql` on the Debian VM:
|
PostgreSQL administrator, for example with `sudo -u postgres psql` on the Debian VM:
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
CREATE ROLE car_traker LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;
|
CREATE ROLE car_tracker LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;
|
||||||
SET password_encryption = 'scram-sha-256';
|
SET password_encryption = 'scram-sha-256';
|
||||||
\password car_traker
|
\password car_tracker
|
||||||
CREATE DATABASE car_traker OWNER car_traker;
|
CREATE DATABASE car_tracker OWNER car_tracker;
|
||||||
REVOKE ALL ON DATABASE car_traker FROM PUBLIC;
|
REVOKE ALL ON DATABASE car_tracker FROM PUBLIC;
|
||||||
\connect car_traker
|
\connect car_tracker
|
||||||
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
|
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
|
||||||
```
|
```
|
||||||
|
|
||||||
`\password` prompts locally without putting the password in SQL history. These are
|
`\password` prompts locally without putting the password in SQL history. These are
|
||||||
one-time provisioning commands; inspect existing roles/databases rather than
|
one-time provisioning commands; inspect existing roles/databases rather than
|
||||||
rerunning blindly if the names already exist. The importer connects as `car_traker`,
|
rerunning blindly if the names already exist. The importer connects as `car_tracker`,
|
||||||
not `postgres`. Owning this dedicated database allows it to create its `silver`
|
not `postgres`. Owning this dedicated database allows it to create its `silver`
|
||||||
schema and run migrations without cluster-wide administrative privileges.
|
schema and run migrations without cluster-wide administrative privileges.
|
||||||
|
|
||||||
@ -47,7 +47,7 @@ Restrict the firewall to the importer machine's IP and PostgreSQL port, normally
|
|||||||
scoped `pg_hba.conf` rule, replacing `<IMPORTER_IP>` with the actual client address:
|
scoped `pg_hba.conf` rule, replacing `<IMPORTER_IP>` with the actual client address:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
hostssl car_traker car_traker <IMPORTER_IP>/32 scram-sha-256
|
hostssl car_tracker car_tracker <IMPORTER_IP>/32 scram-sha-256
|
||||||
```
|
```
|
||||||
|
|
||||||
Rules are first-match; review existing rules and their order. Reload authentication
|
Rules are first-match; review existing rules and their order. Reload authentication
|
||||||
@ -69,8 +69,8 @@ outside the repository. Prefer a password file rather than a password in command
|
|||||||
```sh
|
```sh
|
||||||
export PGHOST=postgres.your-lan.example
|
export PGHOST=postgres.your-lan.example
|
||||||
export PGPORT=5432
|
export PGPORT=5432
|
||||||
export PGDATABASE=car_traker
|
export PGDATABASE=car_tracker
|
||||||
export PGUSER=car_traker
|
export PGUSER=car_tracker
|
||||||
export PGSSLMODE=verify-full
|
export PGSSLMODE=verify-full
|
||||||
export PGSSLROOTCERT=/path/to/your/postgres-ca.crt
|
export PGSSLROOTCERT=/path/to/your/postgres-ca.crt
|
||||||
export PGPASSFILE="$HOME/.pgpass"
|
export PGPASSFILE="$HOME/.pgpass"
|
||||||
@ -79,7 +79,7 @@ export PGPASSFILE="$HOME/.pgpass"
|
|||||||
The matching password-file entry has this form:
|
The matching password-file entry has this form:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
postgres.your-lan.example:5432:car_traker:car_traker:<your-password>
|
postgres.your-lan.example:5432:car_tracker:car_tracker:<your-password>
|
||||||
```
|
```
|
||||||
|
|
||||||
Protect it with `chmod 600 "$HOME/.pgpass"`. Escape `:` and `\` in passwords using
|
Protect it with `chmod 600 "$HOME/.pgpass"`. Escape `:` and `\` in passwords using
|
||||||
@ -99,7 +99,7 @@ psql -c 'SELECT current_database(), current_user, version();'
|
|||||||
Alternatively, no local `psql` installation is needed for the read-only CLI check:
|
Alternatively, no local `psql` installation is needed for the read-only CLI check:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
uv run car-traker-db check
|
uv run car-tracker-db check
|
||||||
```
|
```
|
||||||
|
|
||||||
It connects with the same settings as migrations/imports and executes only `SELECT 1`.
|
It connects with the same settings as migrations/imports and executes only `SELECT 1`.
|
||||||
@ -114,7 +114,7 @@ contain credentials or private data. `no pg_hba.conf entry` means PostgreSQL was
|
|||||||
reached, but its access rules do not allow the client/database/role combination.
|
reached, but its access rules do not allow the client/database/role combination.
|
||||||
The rule must use the **client IP seen by PostgreSQL**, which can be a VPN or NAT
|
The rule must use the **client IP seen by PostgreSQL**, which can be a VPN or NAT
|
||||||
address rather than the laptop's Wi-Fi/LAN address. Use the actual database and
|
address rather than the laptop's Wi-Fi/LAN address. Use the actual database and
|
||||||
role names you created; `car_traker` in this document is an example, not a hardcoded
|
role names you created; `car_tracker` in this document is an example, not a hardcoded
|
||||||
application requirement.
|
application requirement.
|
||||||
|
|
||||||
On the homelab, locate the active configuration and inspect parsed rules as an
|
On the homelab, locate the active configuration and inspect parsed rules as an
|
||||||
@ -130,7 +130,7 @@ Add the appropriate narrow `hostssl` rule in the file returned by `SHOW hba_file
|
|||||||
then run `SELECT pg_reload_conf();`. Check the `error` column and server logs if it
|
then run `SELECT pg_reload_conf();`. Check the `error` column and server logs if it
|
||||||
still fails; a successful reload request alone does not prove every rule is valid.
|
still fails; a successful reload request alone does not prove every rule is valid.
|
||||||
Do not change passwords or use `trust` to work around a missing HBA rule. Once access
|
Do not change passwords or use `trust` to work around a missing HBA rule. Once access
|
||||||
is permitted, rerun `car-traker-db check` from the laptop before migrating.
|
is permitted, rerun `car-tracker-db check` from the laptop before migrating.
|
||||||
|
|
||||||
Other categories point to DNS/VPN, refused/timed-out connections, authentication,
|
Other categories point to DNS/VPN, refused/timed-out connections, authentication,
|
||||||
TLS verification, missing databases, or insufficient privileges. This check does
|
TLS verification, missing databases, or insufficient privileges. This check does
|
||||||
@ -142,8 +142,8 @@ From the repository root, with the connection configured:
|
|||||||
|
|
||||||
```sh
|
```sh
|
||||||
uv sync
|
uv sync
|
||||||
uv run car-traker-db migrate
|
uv run car-tracker-db migrate
|
||||||
uv run car-traker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json
|
uv run car-tracker-db import data/raw/leboncoin/2026-09-08/ed585bcc-188d-401e-bc9f-8bc1318d4ca6/manifest.json
|
||||||
```
|
```
|
||||||
|
|
||||||
The example uses the existing 35-search/one-detail live run. Choose any finalized
|
The example uses the existing 35-search/one-detail live run. Choose any finalized
|
||||||
@ -298,13 +298,13 @@ Database integration tests are skipped unless `TEST_DATABASE_URL` is set. They
|
|||||||
One local example (test password only; localhost-only port; ephemeral data):
|
One local example (test password only; localhost-only port; ephemeral data):
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker run --detach --rm --name car-traker-silver-test \
|
docker run --detach --rm --name car-tracker-silver-test \
|
||||||
--env POSTGRES_PASSWORD=local-test-only --env POSTGRES_DB=car_traker_test \
|
--env POSTGRES_PASSWORD=local-test-only --env POSTGRES_DB=car_tracker_test \
|
||||||
--publish 127.0.0.1:55432:5432 --tmpfs /var/lib/postgresql/data postgres:17
|
--publish 127.0.0.1:55432:5432 --tmpfs /var/lib/postgresql/data postgres:17
|
||||||
docker exec car-traker-silver-test pg_isready -U postgres -d car_traker_test
|
docker exec car-tracker-silver-test pg_isready -U postgres -d car_tracker_test
|
||||||
TEST_DATABASE_URL='postgresql://postgres:local-test-only@127.0.0.1:55432/car_traker_test' \
|
TEST_DATABASE_URL='postgresql://postgres:local-test-only@127.0.0.1:55432/car_tracker_test' \
|
||||||
uv run python -m unittest discover -s tests -v
|
uv run python -m unittest discover -s tests -v
|
||||||
docker stop car-traker-silver-test
|
docker stop car-tracker-silver-test
|
||||||
```
|
```
|
||||||
|
|
||||||
Wait until `pg_isready` succeeds before running the tests. This temporary container
|
Wait until `pg_isready` succeeds before running the tests. This temporary container
|
||||||
|
|||||||
@ -1,11 +1,13 @@
|
|||||||
name: car-traker-app
|
name: car-tracker-app
|
||||||
|
|
||||||
services:
|
services:
|
||||||
app:
|
app:
|
||||||
build:
|
build:
|
||||||
context: ../..
|
context: ../..
|
||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
image: car-traker:local
|
# The reconcile agent sets CAR_TRACKER_IMAGE to an immutable CI-built tag.
|
||||||
|
# The default keeps the documented local build-and-run path working.
|
||||||
|
image: "${CAR_TRACKER_IMAGE:-car-tracker:local}"
|
||||||
# One-shot manual job only. Never restart a blocked scraper automatically.
|
# One-shot manual job only. Never restart a blocked scraper automatically.
|
||||||
restart: "no"
|
restart: "no"
|
||||||
init: true
|
init: true
|
||||||
|
|||||||
@ -1,5 +1,5 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "car-traker"
|
name = "car-tracker"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
description = "French youngtimer asking-price observatory"
|
description = "French youngtimer asking-price observatory"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
@ -16,9 +16,9 @@ target-version = "py313"
|
|||||||
select = ["E4", "E7", "E9", "F"]
|
select = ["E4", "E7", "E9", "F"]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
car-traker = "car_traker.collect:main"
|
car-tracker = "car_tracker.collect:main"
|
||||||
car-traker-db = "car_traker.database:main"
|
car-tracker-db = "car_tracker.database:main"
|
||||||
car-traker-pipeline = "car_traker.pipeline:main"
|
car-tracker-pipeline = "car_tracker.pipeline:main"
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
requires = ["setuptools>=77"]
|
requires = ["setuptools>=77"]
|
||||||
@ -28,4 +28,4 @@ build-backend = "setuptools.build_meta"
|
|||||||
where = ["src"]
|
where = ["src"]
|
||||||
|
|
||||||
[tool.setuptools.package-data]
|
[tool.setuptools.package-data]
|
||||||
car_traker = ["migrations/*.sql"]
|
car_tracker = ["migrations/*.sql"]
|
||||||
|
|||||||
1
src/car_tracker/__init__.py
Normal file
1
src/car_tracker/__init__.py
Normal file
@ -0,0 +1 @@
|
|||||||
|
"""Car-tracker collection and database tools."""
|
||||||
@ -41,8 +41,16 @@ def main():
|
|||||||
child.send_signal(stopped)
|
child.send_signal(stopped)
|
||||||
code = child.wait()
|
code = child.wait()
|
||||||
return code if code >= 0 else 128 - code
|
return code if code >= 0 else 128 - code
|
||||||
|
except FileNotFoundError:
|
||||||
|
# Do not echo caller-supplied paths or arguments.
|
||||||
|
message = ("Virtual display executable could not be found." if display is None
|
||||||
|
else "Container command could not be found; check the executable name.")
|
||||||
|
print(message, file=sys.stderr)
|
||||||
|
return 3
|
||||||
except (OSError, ValueError):
|
except (OSError, ValueError):
|
||||||
print("Could not start the virtual display or container command.", file=sys.stderr)
|
message = ("Could not start the virtual display." if display is None
|
||||||
|
else "Could not start or wait for the container command.")
|
||||||
|
print(message, file=sys.stderr)
|
||||||
return 3
|
return 3
|
||||||
finally:
|
finally:
|
||||||
# Only clean up descendants after the command has finalized its files/transaction.
|
# Only clean up descendants after the command has finalized its files/transaction.
|
||||||
@ -263,7 +263,7 @@ def migrate(conn):
|
|||||||
name text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now()
|
name text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now()
|
||||||
)""")
|
)""")
|
||||||
applied = dict(conn.execute("SELECT name, sha256 FROM silver.schema_migrations").fetchall())
|
applied = dict(conn.execute("SELECT name, sha256 FROM silver.schema_migrations").fetchall())
|
||||||
migrations = sorted(files("car_traker").joinpath("migrations").iterdir(), key=lambda p: p.name)
|
migrations = sorted(files("car_tracker").joinpath("migrations").iterdir(), key=lambda p: p.name)
|
||||||
known = {m.name for m in migrations if m.name.endswith(".sql")}
|
known = {m.name for m in migrations if m.name.endswith(".sql")}
|
||||||
if set(applied) - known:
|
if set(applied) - known:
|
||||||
raise InputError("database_has_unknown_migrations")
|
raise InputError("database_has_unknown_migrations")
|
||||||
@ -367,7 +367,7 @@ def database_error_hint(exc):
|
|||||||
if exc.sqlstate == "42501":
|
if exc.sqlstate == "42501":
|
||||||
return "The database role lacks required privileges. Use the dedicated database owner for migrations/imports, not a superuser."
|
return "The database role lacks required privileges. Use the dedicated database owner for migrations/imports, not a superuser."
|
||||||
if exc.sqlstate in ("42P01", "3F000"):
|
if exc.sqlstate in ("42P01", "3F000"):
|
||||||
return "A required database table/schema is missing. Run car-traker-db migrate against the same database first."
|
return "A required database table/schema is missing. Run car-tracker-db migrate against the same database first."
|
||||||
return "Check the connection configuration and PostgreSQL server logs. Raw error details are withheld to protect credentials and source data."
|
return "Check the connection configuration and PostgreSQL server logs. Raw error details are withheld to protect credentials and source data."
|
||||||
|
|
||||||
|
|
||||||
@ -13,8 +13,8 @@ from unittest.mock import patch
|
|||||||
|
|
||||||
from psycopg import sql
|
from psycopg import sql
|
||||||
|
|
||||||
from car_traker import database, pipeline
|
from car_tracker import database, pipeline
|
||||||
from car_traker.collect import search_url
|
from car_tracker.collect import search_url
|
||||||
import test_database_integration as database_tests
|
import test_database_integration as database_tests
|
||||||
|
|
||||||
|
|
||||||
@ -92,7 +92,7 @@ class BrowserPipelineTests(unittest.TestCase):
|
|||||||
environment["DATABASE_URL"] = os.environ["TEST_DATABASE_URL"]
|
environment["DATABASE_URL"] = os.environ["TEST_DATABASE_URL"]
|
||||||
with patch.dict(os.environ, environment, clear=True), patch.object(
|
with patch.dict(os.environ, environment, clear=True), patch.object(
|
||||||
BrowserContext, "new_page", new_page,
|
BrowserContext, "new_page", new_page,
|
||||||
), patch("car_traker.collect.time.sleep"), redirect_stdout(stdout), redirect_stderr(stderr):
|
), patch("car_tracker.collect.time.sleep"), redirect_stdout(stdout), redirect_stderr(stderr):
|
||||||
code = pipeline.main([
|
code = pipeline.main([
|
||||||
"--headed", "--config", str(self.config), "--output", str(self.output),
|
"--headed", "--config", str(self.config), "--output", str(self.output),
|
||||||
"--delay", "1", "--max-pages", "1", "--max-details", "1",
|
"--delay", "1", "--max-pages", "1", "--max-details", "1",
|
||||||
|
|||||||
@ -5,7 +5,7 @@ import unittest
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from car_traker.collect import SourceAccessError, collect, minimize, parse_detail, parse_search, search_url
|
from car_tracker.collect import SourceAccessError, collect, minimize, parse_detail, parse_search, search_url
|
||||||
|
|
||||||
|
|
||||||
def payload(ids=(123,), total=1, pages=1):
|
def payload(ids=(123,), total=1, pages=1):
|
||||||
@ -85,7 +85,7 @@ class CollectorTests(unittest.TestCase):
|
|||||||
raise result
|
raise result
|
||||||
return result
|
return result
|
||||||
|
|
||||||
with patch("car_traker.collect.time.sleep"):
|
with patch("car_tracker.collect.time.sleep"):
|
||||||
kwargs.setdefault("search_only", True)
|
kwargs.setdefault("search_only", True)
|
||||||
path, status = collect(fetch, [{"family": "synthetic", "text": "test"}],
|
path, status = collect(fetch, [{"family": "synthetic", "text": "test"}],
|
||||||
Path(temporary.name), **kwargs)
|
Path(temporary.name), **kwargs)
|
||||||
@ -221,7 +221,7 @@ class CollectorTests(unittest.TestCase):
|
|||||||
|
|
||||||
def test_interrupt_during_delay_does_not_count_as_detail_attempt(self):
|
def test_interrupt_during_delay_does_not_count_as_detail_attempt(self):
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
with patch("car_traker.collect.time.sleep", side_effect=KeyboardInterrupt()):
|
with patch("car_tracker.collect.time.sleep", side_effect=KeyboardInterrupt()):
|
||||||
path, status = collect(lambda url: payload(),
|
path, status = collect(lambda url: payload(),
|
||||||
[{"family": "synthetic", "text": "test"}], Path(directory))
|
[{"family": "synthetic", "text": "test"}], Path(directory))
|
||||||
manifest = json.loads(path.read_text())
|
manifest = json.loads(path.read_text())
|
||||||
@ -233,7 +233,7 @@ class CollectorTests(unittest.TestCase):
|
|||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
searches = [{"family": "synthetic", "text": "test"},
|
searches = [{"family": "synthetic", "text": "test"},
|
||||||
{"family": "synthetic", "text": "test-2"}]
|
{"family": "synthetic", "text": "test-2"}]
|
||||||
with patch("car_traker.collect.time.sleep") as sleep:
|
with patch("car_tracker.collect.time.sleep") as sleep:
|
||||||
responses = iter([payload(), detail_payload(), payload()])
|
responses = iter([payload(), detail_payload(), payload()])
|
||||||
urls = []
|
urls = []
|
||||||
|
|
||||||
|
|||||||
@ -5,7 +5,7 @@ from tempfile import TemporaryDirectory
|
|||||||
import unittest
|
import unittest
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from car_traker.collect import collect_browser
|
from car_tracker.collect import collect_browser
|
||||||
|
|
||||||
|
|
||||||
class BrowserCleanupTests(unittest.TestCase):
|
class BrowserCleanupTests(unittest.TestCase):
|
||||||
@ -18,7 +18,7 @@ class BrowserCleanupTests(unittest.TestCase):
|
|||||||
max_pages=1, delay=1, search_only=True, max_details=None)
|
max_pages=1, delay=1, search_only=True, max_details=None)
|
||||||
self.playwright = self.enterContext(patch("playwright.sync_api.sync_playwright"))
|
self.playwright = self.enterContext(patch("playwright.sync_api.sync_playwright"))
|
||||||
self.browser = self.playwright.return_value.__enter__.return_value.chromium.launch.return_value
|
self.browser = self.playwright.return_value.__enter__.return_value.chromium.launch.return_value
|
||||||
self.collect = self.enterContext(patch("car_traker.collect.collect", return_value=(self.path, "complete")))
|
self.collect = self.enterContext(patch("car_tracker.collect.collect", return_value=(self.path, "complete")))
|
||||||
self.stderr = self.enterContext(patch("sys.stderr", new_callable=io.StringIO))
|
self.stderr = self.enterContext(patch("sys.stderr", new_callable=io.StringIO))
|
||||||
|
|
||||||
def test_browser_close_failure_retains_finalized_result(self):
|
def test_browser_close_failure_retains_finalized_result(self):
|
||||||
|
|||||||
@ -5,9 +5,9 @@ import signal
|
|||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
from unittest.mock import patch
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
from car_traker import container_entrypoint
|
from car_tracker import container_entrypoint
|
||||||
|
|
||||||
|
|
||||||
class ContainerEntrypointTests(unittest.TestCase):
|
class ContainerEntrypointTests(unittest.TestCase):
|
||||||
@ -15,12 +15,35 @@ class ContainerEntrypointTests(unittest.TestCase):
|
|||||||
with patch("sys.argv", ["entrypoint"]), patch("sys.stderr", new_callable=io.StringIO):
|
with patch("sys.argv", ["entrypoint"]), patch("sys.stderr", new_callable=io.StringIO):
|
||||||
self.assertEqual(container_entrypoint.main(), 3)
|
self.assertEqual(container_entrypoint.main(), 3)
|
||||||
|
|
||||||
|
def test_missing_display_has_distinct_safe_error(self):
|
||||||
|
error = FileNotFoundError(2, "missing", "/private/display")
|
||||||
|
with patch("sys.argv", ["entrypoint", "command"]), \
|
||||||
|
patch("sys.stderr", new_callable=io.StringIO) as stderr, \
|
||||||
|
patch.object(container_entrypoint.subprocess, "Popen", side_effect=error):
|
||||||
|
self.assertEqual(container_entrypoint.main(), 3)
|
||||||
|
self.assertIn("Virtual display executable could not be found", stderr.getvalue())
|
||||||
|
self.assertNotIn("/private/display", stderr.getvalue())
|
||||||
|
|
||||||
|
def test_missing_command_has_distinct_safe_error_and_cleans_display(self):
|
||||||
|
display = Mock()
|
||||||
|
error = FileNotFoundError(2, "missing", "/private/command")
|
||||||
|
with patch("sys.argv", ["entrypoint", "/private/command"]), \
|
||||||
|
patch("sys.stderr", new_callable=io.StringIO) as stderr, \
|
||||||
|
patch.object(container_entrypoint.subprocess, "Popen", side_effect=[display, error]), \
|
||||||
|
patch.object(container_entrypoint.select, "select", return_value=([3], [], [])), \
|
||||||
|
patch.object(container_entrypoint.os, "read", return_value=b"0\n"):
|
||||||
|
self.assertEqual(container_entrypoint.main(), 3)
|
||||||
|
self.assertIn("Container command could not be found", stderr.getvalue())
|
||||||
|
self.assertNotIn("/private/command", stderr.getvalue())
|
||||||
|
display.terminate.assert_called_once()
|
||||||
|
display.wait.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
@unittest.skipUnless(os.environ.get("RUN_BROWSER_TESTS") == "1", "Xvfb container tests are opt-in")
|
@unittest.skipUnless(os.environ.get("RUN_BROWSER_TESTS") == "1", "Xvfb container tests are opt-in")
|
||||||
class VirtualDisplayTests(unittest.TestCase):
|
class VirtualDisplayTests(unittest.TestCase):
|
||||||
def test_child_exit_code_is_preserved(self):
|
def test_child_exit_code_is_preserved(self):
|
||||||
result = subprocess.run([
|
result = subprocess.run([
|
||||||
sys.executable, "-m", "car_traker.container_entrypoint",
|
sys.executable, "-m", "car_tracker.container_entrypoint",
|
||||||
sys.executable, "-c", "raise SystemExit(7)",
|
sys.executable, "-c", "raise SystemExit(7)",
|
||||||
], capture_output=True, text=True, timeout=20)
|
], capture_output=True, text=True, timeout=20)
|
||||||
self.assertEqual(result.returncode, 7, result.stderr)
|
self.assertEqual(result.returncode, 7, result.stderr)
|
||||||
@ -39,7 +62,7 @@ while True:
|
|||||||
signal.pause()
|
signal.pause()
|
||||||
"""
|
"""
|
||||||
process = subprocess.Popen([
|
process = subprocess.Popen([
|
||||||
sys.executable, "-m", "car_traker.container_entrypoint",
|
sys.executable, "-m", "car_tracker.container_entrypoint",
|
||||||
sys.executable, "-c", command,
|
sys.executable, "-c", command,
|
||||||
], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||||
try:
|
try:
|
||||||
|
|||||||
@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch
|
|||||||
|
|
||||||
import psycopg
|
import psycopg
|
||||||
|
|
||||||
from car_traker.database import database_error_hint, main
|
from car_tracker.database import database_error_hint, main
|
||||||
|
|
||||||
|
|
||||||
class DatabaseCliTests(unittest.TestCase):
|
class DatabaseCliTests(unittest.TestCase):
|
||||||
@ -37,10 +37,10 @@ class DatabaseCliTests(unittest.TestCase):
|
|||||||
def test_check_only_runs_select(self):
|
def test_check_only_runs_select(self):
|
||||||
connection = MagicMock()
|
connection = MagicMock()
|
||||||
with patch.dict("os.environ", {"PGHOST": "synthetic-host"}, clear=True), patch(
|
with patch.dict("os.environ", {"PGHOST": "synthetic-host"}, clear=True), patch(
|
||||||
"sys.argv", ["car-traker-db", "check"]
|
"sys.argv", ["car-tracker-db", "check"]
|
||||||
), patch("car_traker.database.psycopg.connect") as connect, patch(
|
), patch("car_tracker.database.psycopg.connect") as connect, patch(
|
||||||
"sys.stdout", new_callable=io.StringIO
|
"sys.stdout", new_callable=io.StringIO
|
||||||
) as output, patch("car_traker.database.migrate") as migrate:
|
) as output, patch("car_tracker.database.migrate") as migrate:
|
||||||
connect.return_value.__enter__.return_value = connection
|
connect.return_value.__enter__.return_value = connection
|
||||||
self.assertEqual(main(), 0)
|
self.assertEqual(main(), 0)
|
||||||
connection.execute.assert_called_once_with("SELECT 1")
|
connection.execute.assert_called_once_with("SELECT 1")
|
||||||
@ -50,8 +50,8 @@ class DatabaseCliTests(unittest.TestCase):
|
|||||||
def test_hba_failure_is_actionable_without_echoing_credentials(self):
|
def test_hba_failure_is_actionable_without_echoing_credentials(self):
|
||||||
exc = psycopg.OperationalError('no pg_hba.conf entry for host "private-host"; password=synthetic-secret')
|
exc = psycopg.OperationalError('no pg_hba.conf entry for host "private-host"; password=synthetic-secret')
|
||||||
with patch.dict("os.environ", {"PGHOST": "synthetic-host"}, clear=True), patch(
|
with patch.dict("os.environ", {"PGHOST": "synthetic-host"}, clear=True), patch(
|
||||||
"sys.argv", ["car-traker-db", "check"]
|
"sys.argv", ["car-tracker-db", "check"]
|
||||||
), patch("car_traker.database.psycopg.connect", side_effect=exc), patch(
|
), patch("car_tracker.database.psycopg.connect", side_effect=exc), patch(
|
||||||
"sys.stderr", new_callable=io.StringIO
|
"sys.stderr", new_callable=io.StringIO
|
||||||
) as output:
|
) as output:
|
||||||
self.assertEqual(main(), 1)
|
self.assertEqual(main(), 1)
|
||||||
@ -61,9 +61,9 @@ class DatabaseCliTests(unittest.TestCase):
|
|||||||
|
|
||||||
def test_no_implicit_local_connection(self):
|
def test_no_implicit_local_connection(self):
|
||||||
with patch.dict("os.environ", {}, clear=True), patch(
|
with patch.dict("os.environ", {}, clear=True), patch(
|
||||||
"sys.argv", ["car-traker-db", "check"]
|
"sys.argv", ["car-tracker-db", "check"]
|
||||||
), patch("sys.stderr", new_callable=io.StringIO), patch(
|
), patch("sys.stderr", new_callable=io.StringIO), patch(
|
||||||
"car_traker.database.psycopg.connect"
|
"car_tracker.database.psycopg.connect"
|
||||||
) as connect, self.assertRaises(SystemExit):
|
) as connect, self.assertRaises(SystemExit):
|
||||||
main()
|
main()
|
||||||
connect.assert_not_called()
|
connect.assert_not_called()
|
||||||
|
|||||||
@ -6,8 +6,8 @@ from tempfile import TemporaryDirectory
|
|||||||
import unittest
|
import unittest
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from car_traker.collect import collect, write_json
|
from car_tracker.collect import collect, write_json
|
||||||
from car_traker.database import InputError, load_json, load_run, prepare_record
|
from car_tracker.database import InputError, load_json, load_run, prepare_record
|
||||||
|
|
||||||
|
|
||||||
class RawInputTests(unittest.TestCase):
|
class RawInputTests(unittest.TestCase):
|
||||||
@ -22,7 +22,7 @@ class RawInputTests(unittest.TestCase):
|
|||||||
"list_id": 123, "price": [1000], "body": "Synthetic\u2028text\u2029only",
|
"list_id": 123, "price": [1000], "body": "Synthetic\u2028text\u2029only",
|
||||||
}],
|
}],
|
||||||
}}}})
|
}}}})
|
||||||
with patch("car_traker.collect.time.sleep"):
|
with patch("car_tracker.collect.time.sleep"):
|
||||||
path, _ = collect(lambda url: source, [{"family": "synthetic", "text": "test"}],
|
path, _ = collect(lambda url: source, [{"family": "synthetic", "text": "test"}],
|
||||||
self.root, search_only=True)
|
self.root, search_only=True)
|
||||||
return path, json.loads(path.read_text())
|
return path, json.loads(path.read_text())
|
||||||
|
|||||||
@ -19,8 +19,8 @@ import psycopg
|
|||||||
from psycopg import sql
|
from psycopg import sql
|
||||||
from psycopg.conninfo import conninfo_to_dict
|
from psycopg.conninfo import conninfo_to_dict
|
||||||
|
|
||||||
from car_traker import database
|
from car_tracker import database
|
||||||
from car_traker.collect import search_url, write_json, write_records
|
from car_tracker.collect import search_url, write_json, write_records
|
||||||
|
|
||||||
|
|
||||||
@unittest.skipUnless(os.environ.get("TEST_DATABASE_URL"), "TEST_DATABASE_URL is not set")
|
@unittest.skipUnless(os.environ.get("TEST_DATABASE_URL"), "TEST_DATABASE_URL is not set")
|
||||||
@ -247,7 +247,7 @@ class DatabaseIntegrationTests(unittest.TestCase):
|
|||||||
def test_rejected_record_replay_keeps_nonzero_cli_exit(self):
|
def test_rejected_record_replay_keeps_nonzero_cli_exit(self):
|
||||||
path, _ = self.fixture(pages=[[{"list_id": "invalid"}]])
|
path, _ = self.fixture(pages=[[{"list_id": "invalid"}]])
|
||||||
with patch.dict(os.environ, {"DATABASE_URL": os.environ["TEST_DATABASE_URL"]}), patch(
|
with patch.dict(os.environ, {"DATABASE_URL": os.environ["TEST_DATABASE_URL"]}), patch(
|
||||||
"sys.argv", ["car-traker-db", "import", str(path)]
|
"sys.argv", ["car-tracker-db", "import", str(path)]
|
||||||
), patch("sys.stdout", new_callable=io.StringIO) as output:
|
), patch("sys.stdout", new_callable=io.StringIO) as output:
|
||||||
self.assertEqual(database.main(), 2)
|
self.assertEqual(database.main(), 2)
|
||||||
self.assertEqual(database.main(), 2)
|
self.assertEqual(database.main(), 2)
|
||||||
|
|||||||
@ -10,7 +10,7 @@ import tempfile
|
|||||||
import unittest
|
import unittest
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
from car_traker import pipeline
|
from car_tracker import pipeline
|
||||||
|
|
||||||
|
|
||||||
class PipelineTests(unittest.TestCase):
|
class PipelineTests(unittest.TestCase):
|
||||||
|
|||||||
@ -3,7 +3,7 @@ from datetime import datetime, timezone
|
|||||||
from decimal import Decimal, localcontext
|
from decimal import Decimal, localcontext
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from car_traker.transform import clean_observation
|
from car_tracker.transform import clean_observation
|
||||||
|
|
||||||
|
|
||||||
OBSERVED_AT = datetime(2026, 9, 8, tzinfo=timezone.utc)
|
OBSERVED_AT = datetime(2026, 9, 8, tzinfo=timezone.utc)
|
||||||
|
|||||||
2
uv.lock
generated
2
uv.lock
generated
@ -3,7 +3,7 @@ revision = 3
|
|||||||
requires-python = ">=3.13"
|
requires-python = ">=3.13"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "car-traker"
|
name = "car-tracker"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
source = { editable = "." }
|
source = { editable = "." }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user