publish tested images and queue manual GitOps app runs

.gitea/workflows/ci.yaml

@@ -2,6 +2,7 @@ name: CI
 
 on:
   push:
+    branches: [master]
   pull_request:
   workflow_dispatch:
 
@@ -45,7 +46,7 @@ jobs:
         run: uv sync --locked --dev
 
       - name: Lint
-        run: uv run --locked ruff check src tests
+        run: uv run --locked ruff check src tests ops/gitops
 
       - name: Run offline and PostgreSQL integration tests
         run: timeout 10m uv run --locked python -m unittest discover -s tests -v
@@ -58,3 +59,54 @@ jobs:
           uv venv /tmp/car-tracker-wheel
           uv pip install --python /tmp/car-tracker-wheel/bin/python --no-deps dist/*.whl
           /tmp/car-tracker-wheel/bin/python -c 'from importlib.resources import files; assert files("car_tracker").joinpath("migrations", "001_silver.sql").is_file(), "SQL migration missing from wheel"'
+
+  publish:
+    name: Build and publish tested image
+    needs: test
+    # PRs and the gitops state branch must never publish images.
+    if: github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
+    runs-on: python-ci
+    env:
+      REGISTRY: gitea.vandeplanque.com
+      IMAGE_REPOSITORY: tonio/car-tracker
+      IMAGE_SHA: ${{ github.sha }}
+      DOCKER_CONFIG: /tmp/car-tracker-docker-auth
+    steps:
+      - name: Check out tested revision
+        uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
+      - name: Install Docker client
+        run: sh ops/ci/install-docker-cli.sh
+
+      - name: Authenticate to Gitea registry
+        env:
+          REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
+          REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
+        run: |
+          set -eu
+          test -n "$REGISTRY_USER" && test -n "$REGISTRY_TOKEN"
+          umask 077
+          mkdir -p "$DOCKER_CONFIG"
+          printf '%s' "$REGISTRY_TOKEN" | docker login "$REGISTRY" --username "$REGISTRY_USER" --password-stdin
+
+      - name: Build and publish SHA-tagged image
+        run: |
+          set -eu
+          image="$REGISTRY/$IMAGE_REPOSITORY:$IMAGE_SHA"
+          # Do not overwrite a previously published SHA tag on a workflow rerun.
+          # Digest pinning, not this mutable registry tag, identifies a run image.
+          if docker manifest inspect "$image" >/dev/null 2>&1; then
+            echo "Image already published: $image"
+            exit 0
+          fi
+          timeout 35m docker build --label "org.opencontainers.image.revision=$IMAGE_SHA" -t "$image" .
+          docker run --rm --network none "$image" car-tracker-db --help
+          timeout 5m docker push "$image"
+          echo "Published: $image"
+
+      - name: Remove job registry credentials
+        if: always()
+        run: rm -rf /tmp/car-tracker-docker-auth

.gitea/workflows/run-app.yaml

@@ -0,0 +1,62 @@
+name: Run app
+
+on:
+  workflow_dispatch:
+    inputs:
+      release:
+        description: "Published full commit SHA or Git tag (not a branch or latest)"
+        required: true
+        type: string
+
+jobs:
+  request:
+    name: Queue one production run through Git
+    # Execute trusted control-plane code, never a user-selected release workflow.
+    if: github.ref == 'refs/heads/master'
+    runs-on: python-ci
+    env:
+      REGISTRY: gitea.vandeplanque.com
+      IMAGE_REPOSITORY: tonio/car-tracker
+      DOCKER_CONFIG: /tmp/car-tracker-docker-auth
+      RELEASE: ${{ github.event.inputs.release }}
+      GITOPS_URL: https://gitea.vandeplanque.com/tonio/car-tracker.git
+      GITOPS_BRANCH: gitops
+    steps:
+      - name: Check out trusted master workflow revision
+        uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Install Docker client
+        run: sh ops/ci/install-docker-cli.sh
+
+      - name: Install Python
+        uses: https://github.com/astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e
+        with:
+          version: "0.9.3"
+          python-version: "3.13"
+          enable-cache: false
+          add-problem-matchers: false
+          github-token: ""
+
+      - name: Authenticate to registry
+        env:
+          REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
+          REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
+        run: |
+          set -eu
+          test -n "$REGISTRY_USER" && test -n "$REGISTRY_TOKEN"
+          umask 077
+          mkdir -p "$DOCKER_CONFIG"
+          printf '%s' "$REGISTRY_TOKEN" | docker login "$REGISTRY" --username "$REGISTRY_USER" --password-stdin
+
+      - name: Resolve published release and commit run request
+        env:
+          GITOPS_USER: ${{ secrets.GITOPS_USER }}
+          GITOPS_TOKEN: ${{ secrets.GITOPS_TOKEN }}
+        run: uv run --no-project --python 3.13 python ops/gitops/request.py
+
+      - name: Remove job registry credentials
+        if: always()
+        run: rm -rf /tmp/car-tracker-docker-auth

PROJECT_PLAN.md

@@ -28,6 +28,10 @@ The manual homelab job is now packaged in `Dockerfile` and `ops/app/compose.yaml
 `car-tracker-pipeline` performs a database preflight, bounded collection, and an
 idempotent import of the exact returned manifest, retaining partial coverage and
 separate operational outcomes. There is no scheduler or automatic deployment.
+The optional Gitea Actions/GitOps request workflow is documented in
+`docs/GITOPS.md`: tests/build/publish occur on a dedicated runner VM; explicit
+requests are pulled and consumed once on the production VM. It is not a daily
+collection schedule, and migrations remain manual.
 See `docs/DEPLOYMENT.md`; container tests use isolated PostgreSQL 17 and synthetic
 browser responses. Actual live access from the homelab still needs verification.
 

README.md

@@ -92,6 +92,11 @@ and validation issues. Generation/trim inference, redaction, and scheduling are
 
 ## Manual Homelab Job
 
+For a manual **Run app** button in Gitea Actions, see [GitOps jobs](docs/GITOPS.md).
+CI publishes tested images; the button queues a digest-pinned request, and the
+production host consumes it once using host-local DB credentials. No automatic
+migrations or retries of failed scrapes are introduced.
+
 `car-tracker-pipeline` checks database connectivity/schema, collects raw observations,
 and imports that exact finalized run. Partial/failed collections remain marked as
 such even when their import succeeds. A lock prevents overlapping pipeline jobs on

deploy/prod.tag

@@ -1,6 +0,0 @@
-# Desired deployment state for the homelab, consumed by
-# ~/containers/car-tracker/reconcile.sh. The value must be a full 40-character
-# commit SHA of a revision whose image has been pushed to the Gitea registry as
-# tonio/car-tracker:<sha>. To roll back, revert the commit that changed this file.
-# Lines beginning with # are ignored.
-2c58234fd9f360f262fde255fae5273ec8f57f6c

docs/CI.md

@@ -2,8 +2,9 @@
 
 This repository targets **Gitea 1.23.8**. The workflow is in
 `.gitea/workflows/ci.yaml`; it runs on pushes, pull requests, and manual dispatch.
-It adds continuous integration only: no application deployment, scheduled scraping,
+It tests and publishes images on master. Manual **Run app** dispatch queues a
-production migrations, or access to the homelab application database.
+GitOps job; see [GITOPS.md](GITOPS.md) for credentials and production installation.
+No live scraping, production migration, or production DB access occurs in CI.
 
 ## What Runs
 
@@ -34,7 +35,7 @@ action commits, uv, and the Python dependency lockfile are explicitly pinned.
 
 Gitea 1.23 ignores several GitHub workflow keys, including job-level
 `timeout-minutes`, `permissions`, and `concurrency`; the workflow does not pretend
-they provide protections. Runner configuration enforces capacity 1 and a 30-minute
+they provide protections. Runner configuration enforces capacity 1 and a 45-minute
 job timeout; the test command also has a 10-minute timeout.
 
 ## Enable Repository Actions
@@ -59,8 +60,8 @@ with your Gitea version.
 
 The runner daemon needs Docker socket access, which is effectively root access to
 the CI host. Do not put production database credentials, backups, or raw listing
-data on that host. Job containers do not receive the socket or arbitrary host bind
+data on that host. Job containers now receive the socket for builds, but arbitrary
-mounts. These restrictions reduce exposure but do not make self-hosted execution
+workflow host bind mounts are not allowlisted. This does not make self-hosted execution
 safe for arbitrary hostile workflows. Register at repository scope, accept only
 trusted contributors/workflows, and review fork pull requests before allowing them
 to run. Runner and server updates should be planned independently of this initial
@@ -82,7 +83,7 @@ from `tonio/car-tracker` -> **Settings -> Actions -> Runners**. Enter:
 | --- | --- |
 | Instance URL | Your Gitea **HTTP(S) web URL**, including its port if needed |
 | Registration token | The repository token, entered locally; do not send it in chat |
-| Runner name | For example `car-tracker-ci` |
+| Runner name | For example `gitea-runner` (the name does not affect job routing) |
 | Labels | `python-ci:docker://node:22-bookworm` |
 
 The instance URL is **not** the Git SSH URL ending in port 2222. It must resolve
@@ -92,8 +93,9 @@ verify DNS and routing inside Docker as well as on the CI host. HTTPS certificat
 verification remains enabled; configure trust for private certificates rather
 than disabling it.
 
-Registration writes credentials to `/data/.runner` in the persistent Docker volume.
+Registration writes credentials to `ops/ci-runner/data/.runner` via the `./data` bind.
-Treat that volume as a secret. No registration token is needed in Compose's
+The deployed copy on the new VM is `~/containers/gitea-runner/data/.runner`.
+Keep this directory private and outside Git. No registration token is needed in Compose's
 environment. Start the registered daemon:
 
 ```sh
@@ -108,7 +110,7 @@ not need your application PostgreSQL credentials or an HBA rule for that databas
 Restrict CI-host network access to production services where practical.
 
 Do not run `docker compose down -v` unless you intend to erase runner registration.
-Normal container recreation preserves the registration volume.
+Normal container recreation preserves the registration directory.
 
 ## First Run Checks
 

docs/DEPLOYMENT.md

@@ -8,7 +8,9 @@ Use the existing database and role, currently both `car_tracker`.
 Do not provision another production PostgreSQL server or recreate those objects.
 
 `ops/app/compose.yaml` is the one-shot application stack, separate from
-`ops/ci-runner`. There is no scheduling, restart loop, or automatic CI deployment.
+`ops/ci-runner`. This section describes direct manual operation, with no scheduling
+or restart loop. For the optional Gitea button → GitOps request → host-run flow,
+see [GITOPS.md](GITOPS.md). Migrations remain manual in both modes.
 Run commands from the homelab checkout's root with Docker Engine and Compose
 available. Review the revision and `searches.toml`; do not run these in CI.
 After these changes are published, update the homelab checkout before building.

docs/GITOPS.md

@@ -0,0 +1,194 @@
+# Gitea-triggered application jobs
+
+## Flow
+
+The app is a one-shot collector/importer, not a continuously running service.
+
+1. Push to **master** (the actual default branch). CI runs lint, offline tests,
+   disposable PostgreSQL 17 integration tests and package checks.
+2. After tests pass, CI builds and publishes
+   `gitea.vandeplanque.com/tonio/car-tracker:<full-commit-SHA>`. Reruns skip an
+   existing SHA tag. Tags are still mutable at the registry; requests pin a digest.
+3. Choose **Actions → Run app → Run workflow**, select workflow branch **master**,
+   and enter a published full SHA or a Git tag pointing to it. Branch names and
+   `latest` are not release selectors. The release must be in master's history.
+4. The workflow pulls the image on the builder VM, verifies its revision label,
+   and commits `deploy/request.json` to the dedicated **gitops** branch. It creates
+   that branch from master on the first request. Every dispatch gets a new UUID,
+   even for the same image. Concurrent non-fast-forward pushes fail; no force push.
+5. A production user-systemd timer reads the request history using a separate bare
+   Git checkout and runs requests serially, once each. It uses the image digest and
+   its bounded default command. Searches come from that image, not a host checkout.
+
+**Green Run app means queued, not successfully collected/imported.** The production
+journal, local request state and raw manifest/sidecar report the actual outcome.
+There is no SSH deployment, incoming deployment API, automatic migration, daily
+collection schedule, or automatic retry after collection starts.
+
+## Runner and network configuration
+
+`ops/ci-runner/compose.yaml` and `config.yaml` match the new setup at
+`~/containers/gitea-runner/`: project `gitea-runner`, `./data` bind,
+`python-ci:docker://node:22-bookworm`, capacity 1, timeout 45 minutes, and
+`container.docker_host: ""` for job socket access. The build job installs Docker
+CLI/Buildx; the daemon runs on the builder VM. No privileged jobs or arbitrary
+workflow host bind mounts are enabled.
+
+Socket access grants jobs root-equivalent control of the **builder VM**, including
+other jobs and runner registration. Run only trusted workflows. A label is routing,
+not a security boundary. No production passwords, backups or raw data on that VM.
+GitOps avoids exposing production credentials to CI, but does not make arbitrary
+images trustworthy: the approved image can read the mounted DB password in production.
+
+Keep runner configuration deployment manual. Inspect before copying repository
+files over deployed files; preserve `data/.runner`. Do not copy the old runner's
+named volume, unregister the working new runner, or use `down -v` during migration.
+
+Runner/job `extra_hosts` map Gitea to `192.168.1.56` with HTTPS verification intact.
+**Docker Engine also needs working registry DNS on the builder VM**; container
+`extra_hosts` does not affect that daemon. If DNS still returns the wrong public
+address, carefully add this entry to the builder VM's `/etc/hosts`:
+
+```text
+192.168.1.56 gitea.vandeplanque.com
+```
+
+Reserve that LAN address in DHCP. Production Docker also needs a working registry
+path. Never configure an insecure registry or disable TLS verification.
+
+## Gitea Actions secrets
+
+Create these repository secrets in Settings → Actions → Secrets:
+
+| Secret | Purpose |
+| --- | --- |
+| `REGISTRY_USER` | `tonio`: owner of the current package namespace |
+| `REGISTRY_TOKEN` | Token with `write:package` (includes package reads) |
+| `GITOPS_USER` | Account allowed to write the repository's gitops branch |
+| `GITOPS_TOKEN` | Separate token with `write:repository` |
+
+For the current user-owned `tonio/car-tracker` package, publishing must authenticate
+as `tonio`. A bot added as a repository collaborator cannot publish to that user's
+package namespace. Use separate narrowly scoped tokens; Gitea 1.23 token scopes
+are category-wide, not restricted to one repository. Private user-owned packages
+also require the owner's identity for reads; production uses a separate
+`read:package` token. Public user profiles can expose their packages publicly,
+regardless of linked repository visibility.
+
+Gitea 1.23 does not enforce GitHub-style
+job `permissions` as a security boundary: set token scopes and account/branch access
+in Gitea itself. Protect master; restrict the state writer to gitops through branch
+protection where possible. Never rewrite the gitops branch's audit log/queue.
+Verify the package owner's visibility; packages do not inherit repository
+access controls.
+
+Enter tokens only in the Gitea UI, not chat, commands, committed files or logs.
+The workflows use stdin for registry login and a temporary Git askpass helper,
+not tokens in command arguments or URLs. The built-in Actions token is not assumed
+to have write privileges on this older release. No production password or SSH
+deployment key belongs in Actions secrets.
+
+## Install the production consumer (manual, no run yet)
+
+Follow [host runtime configuration](DEPLOYMENT.md) first: private directories,
+`app.env`, `pgpass`, TLS and HBA rule must exist. Migrations stay manual. The chosen
+stack root here is `~/containers/car-tracker`, not the `/srv` example.
+
+**Disable the old SHA-polling agent before installing this version.** It checked
+out releases in the development repository and retried failed scrapes. Neither
+behavior is used here. The obsolete `deploy/prod.tag` is removed; do not activate
+the old agent. It will see no desired tag after these changes are published.
+
+```sh
+systemctl --user disable --now car-tracker-reconcile.timer
+systemctl --user status car-tracker-reconcile.service
+```
+
+If a service/job is active, let it finalize before replacing it; investigate uncertain
+imports. From the updated repository root, inspect destination files first, then:
+
+```sh
+install -m 0700 ops/gitops/reconcile.py ~/containers/car-tracker/reconcile.py
+install -m 0600 ops/app/compose.runtime.yaml ~/containers/car-tracker/compose.runtime.yaml
+mkdir -p ~/.config/systemd/user
+install -m 0644 ops/gitops/car-tracker-reconcile.service ~/.config/systemd/user/
+install -m 0644 ops/gitops/car-tracker-reconcile.timer ~/.config/systemd/user/
+systemctl --user daemon-reload
+```
+
+Create a **separate** bare checkout only if `desired.git` doesn't already exist:
+
+```sh
+git clone --bare ssh://git@debian.tailf52e5b.ts.net:2222/tonio/car-tracker.git \
+  ~/containers/car-tracker/desired.git
+chmod 0700 ~/containers/car-tracker/desired.git
+```
+
+Prefer a read-only repository deploy key. Verify SSH host trust/access interactively
+before enabling the timer. The agent never checks out or resets the development repo.
+
+For private registry pulls, run `docker login gitea.vandeplanque.com` interactively
+on production with a separate **read-only package token**, not the publisher token.
+Protect Docker's credential file or use a credential helper. Credentials remain local.
+
+Both Compose stacks use `car-tracker-app` and the same subnet/storage. If the old
+pre-rename `car-traker-app` network occupies the subnet, remove it through the old
+project only when no job is active; do not delete unrelated networks.
+
+Before dispatch, validate with a real published digest (never this placeholder):
+
+```sh
+CAR_TRACKER_IMAGE='gitea.vandeplanque.com/tonio/car-tracker@sha256:<published-digest>' \
+  docker compose --env-file ~/containers/car-tracker/app.env \
+  -f ~/containers/car-tracker/compose.runtime.yaml config --quiet
+```
+
+When ready:
+
+```sh
+sudo loginctl enable-linger tonio
+systemctl --user enable --now car-tracker-reconcile.timer
+```
+
+Review gitops history first: enabling can execute all outstanding requests. If the
+branch does not exist yet, fetching fails and no app starts; later polls try again.
+Repository updates do not automatically replace deployed agent/Compose files.
+
+## Outcomes and recovery
+
+The timer polls approximately every minute after the preceding cycle finishes.
+All request commits are processed in order, so two button presses between polls
+are not silently reduced to the latest image.
+
+```sh
+systemctl --user list-timers car-tracker-reconcile.timer
+journalctl --user -u car-tracker-reconcile.service -n 100 --no-pager
+ls ~/containers/car-tracker/state/requests/
+```
+
+Each `state/requests/<uuid>.json` records a request and pipeline exit code. `finished`
+means exited, **not** complete source coverage. See [pipeline exit meanings](DEPLOYMENT.md#outcomes-and-recovery).
+Raw files and `pipeline.json` stay private on production, never CI artifacts.
+
+An attempt is atomically recorded **before launching** the collector. It is never
+rerun automatically, regardless of exit code. A crash/timeout may leave
+`status: attempted`, exit unknown. This blocks further requests until investigated.
+This favors a missed run over duplicate scraping; it is not an exactly-once guarantee.
+
+Inspect `car-tracker-run-<uuid>`, the raw manifest and database outcome. Recover an
+uncertain import by replaying the exact finalized manifest, not scraping again.
+After investigation, retain the record and explicitly set its local status to
+`resolved` to unblock future requests. Do not delete state to force retries. Back
+up `state/requests`, raw files and the DB: losing the ledger can replay old jobs.
+
+Registry/config failures **before** recording an attempt can retry because no
+collection started. No retries after the marker. Never overlap standalone CLI or
+recovery operations with a pipeline on the same output root.
+
+For an intentional repeat or an older image, dispatch **Run app again**: a new UUID
+is recorded. Reverting a state commit containing a consumed UUID does not repeat
+it. "Rollback" here selects an older version for a new run; it does not undo DB data.
+
+Build/image retention remains manual on the builder VM. Check `docker system df`
+and prune unused build cache deliberately; never prune volumes to free build space.
+No live collection is performed while implementing/testing these files.

ops/app/compose.runtime.yaml

@@ -0,0 +1,52 @@
+# Pull-only production job: no source checkout, build, or searches.toml host bind.
+# Searches and application code come from the same digest-pinned image.
+name: car-tracker-app
+
+services:
+  app:
+    image: "${CAR_TRACKER_IMAGE:?Set a digest-pinned image}"
+    restart: "no"
+    init: true
+    stop_grace_period: 30s
+    user: "${APP_UID:-1000}:${APP_GID:-1000}"
+    read_only: true
+    cap_drop: [ALL]
+    security_opt:
+      - no-new-privileges:true
+    cpus: 1.5
+    mem_limit: 2g
+    pids_limit: 256
+    shm_size: 512m
+    tmpfs:
+      - /tmp:rw,nosuid,nodev,size=512m,mode=1777
+      - /tmp/.X11-unix:rw,nosuid,nodev,size=1m,mode=1777
+    environment:
+      PGHOST: "${PGHOST:?Set the existing PostgreSQL hostname or IP}"
+      PGPORT: "${PGPORT:-5432}"
+      PGDATABASE: "${PGDATABASE:?Set the application database name}"
+      PGUSER: "${PGUSER:?Set the application database role}"
+      PGSSLMODE: "${PGSSLMODE:-require}"
+      PGPASSFILE: /run/db/pgpass
+      PGSSLROOTCERT: /run/db/root.crt
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - type: bind
+        source: "${RAW_DATA_DIR:?Set an existing private raw storage directory}"
+        target: /data
+        bind:
+          create_host_path: false
+      - type: bind
+        source: "${DB_CONFIG_DIR:?Set an existing private DB configuration directory}"
+        target: /run/db
+        read_only: true
+        bind:
+          create_host_path: false
+    networks:
+      app:
+
+networks:
+  app:
+    ipam:
+      config:
+        - subnet: "${APP_SUBNET:-172.30.17.0/28}"

ops/ci-runner/.gitignore

@@ -0,0 +1 @@
+data/

ops/ci-runner/compose.yaml

@@ -1,4 +1,6 @@
-# CI infrastructure only; this does not deploy the application or PostgreSQL.
+# Install on the dedicated CI/build VM, not the application/database VM.
+name: gitea-runner
+
 services:
   runner:
     image: gitea/act_runner:0.2.13
@@ -6,16 +8,16 @@ services:
     entrypoint: ["act_runner"]
     command: ["--config", "/config.yaml", "daemon"]
     working_dir: /data
-    # The Gitea instance is published on loopback only, and its public name still
+    # Preserve HTTPS/SNI while reaching Gitea via its existing LAN listener.
-    # points at an IPv4 address this host no longer has. Resolve the name to the
+    # Keep this mapping in sync with container.options in config.yaml.
-    # host's LAN address so the runner can reach the Gitea API. Update this together
-    # with the `options` entry in config.yaml if the LAN address changes.
     extra_hosts:
       - "gitea.vandeplanque.com:192.168.1.56"
     volumes:
-      - runner-data:/data
+      - ./data:/data
       - ./config.yaml:/config.yaml:ro
       - /var/run/docker.sock:/var/run/docker.sock
-
+    logging:
-volumes:
+      driver: json-file
-  runner-data:
+      options:
+        max-size: "10m"
+        max-file: "3"

ops/ci-runner/config.yaml

@@ -4,7 +4,7 @@ log:
 runner:
   file: /data/.runner
   capacity: 1
-  timeout: 30m
+  timeout: 45m
   shutdown_timeout: 30s
   insecure: false
   env_file: ""
@@ -22,6 +22,7 @@ container:
   # extra_hosts entry in compose.yaml.
   options: "--add-host=gitea.vandeplanque.com:192.168.1.56"
   valid_volumes: []
-  # The daemon uses Docker, but jobs/services do not receive the Docker socket.
+  # Build jobs need Docker. Socket access grants root-equivalent access to the
-  docker_host: "-"
+  # dedicated CI VM; never register this configuration on the production VM.
+  docker_host: ""
   force_pull: true

ops/ci/install-docker-cli.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+# Job containers are Debian-based Node images; install only the client/plugins.
+# Docker Engine runs on the separate builder VM, accessed through its socket.
+set -eu
+apt-get update
+apt-get install -y --no-install-recommends ca-certificates curl
+install -d -m 0755 /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
+chmod a+r /etc/apt/keyrings/docker.asc
+cat > /etc/apt/sources.list.d/docker.sources <<EOF
+Types: deb
+URIs: https://download.docker.com/linux/debian
+Suites: $(. /etc/os-release && echo "$VERSION_CODENAME")
+Components: stable
+Architectures: $(dpkg --print-architecture)
+Signed-By: /etc/apt/keyrings/docker.asc
+EOF
+apt-get update
+apt-get install -y --no-install-recommends docker-ce-cli docker-buildx-plugin
+docker version

ops/gitops/car-tracker-reconcile.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Pull and consume car-tracker GitOps run requests
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/python3 %h/containers/car-tracker/reconcile.py
+TimeoutStartSec=45min
+TimeoutStopSec=60s
+UMask=0077
+# No Restart= policy: a failed or interrupted collection must not be retried.

ops/gitops/car-tracker-reconcile.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Poll Git for explicitly requested car-tracker jobs
+
+[Timer]
+OnBootSec=2min
+OnUnitInactiveSec=1min
+RandomizedDelaySec=15s
+
+[Install]
+WantedBy=timers.target

ops/gitops/reconcile.py

@@ -0,0 +1,142 @@
+"""Pull run requests from Git; attempt each request at most once, never retry a scrape."""
+
+import fcntl
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import tempfile
+import uuid
+
+
+IMAGE_REPOSITORY = "gitea.vandeplanque.com/tonio/car-tracker"
+
+
+def validate_request(data):
+    if not isinstance(data, dict) or not all(isinstance(data.get(key), str)
+                                             for key in ("request_id", "revision", "image")):
+        raise ValueError("Invalid request types")
+    if set(data) != {"schema_version", "request_id", "revision", "image"} or data["schema_version"] != 1:
+        raise ValueError("Invalid request schema")
+    if str(uuid.UUID(data["request_id"])) != data["request_id"]:
+        raise ValueError("Invalid request identifier")
+    if not re.fullmatch(r"[0-9a-f]{40}", data["revision"]):
+        raise ValueError("Invalid revision")
+    if not re.fullmatch(re.escape(IMAGE_REPOSITORY) + r"@sha256:[0-9a-f]{64}", data["image"]):
+        raise ValueError("Invalid image")
+    return data
+
+
+def atomic_json(path, data):
+    with tempfile.NamedTemporaryFile(mode="w", dir=path.parent, delete=False) as output:
+        temporary = Path(output.name)
+        try:
+            json.dump(data, output, indent=2)
+            output.write("\n")
+            output.flush()
+            os.fsync(output.fileno())
+        except BaseException:
+            temporary.unlink(missing_ok=True)
+            raise
+    os.replace(temporary, path)
+    directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(directory)
+    finally:
+        os.close(directory)
+
+
+def command(*args, env=None, capture=True):
+    return subprocess.run(args, env=env, check=True, text=True,
+                          stdout=subprocess.PIPE if capture else None,
+                          stderr=subprocess.PIPE if capture else None).stdout
+
+
+def compose_environment(image):
+    env = os.environ.copy()
+    # Shell variables override --env-file; don't inherit stale app settings.
+    for key in ("APP_UID", "APP_GID", "RAW_DATA_DIR", "DB_CONFIG_DIR", "APP_SUBNET",
+                "PGHOST", "PGPORT", "PGDATABASE", "PGUSER", "PGSSLMODE", "COMPOSE_FILE",
+                "COMPOSE_PROJECT_NAME", "COMPOSE_PROFILES", "COMPOSE_ENV_FILES"):
+        env.pop(key, None)
+    env["CAR_TRACKER_IMAGE"] = image
+    return env
+
+
+def attempt(request, state, compose_file, env_file):
+    outcome = state / f"{request['request_id']}.json"
+    if outcome.exists():
+        recorded = json.loads(outcome.read_text())
+        if recorded["request"] != request:
+            raise ValueError("Request identifier reused with different content")
+        return
+    env = compose_environment(request["image"])
+    compose = ["docker", "compose", "--env-file", str(env_file), "-f", str(compose_file)]
+    # Fail-closed checks occur before consuming a request. Network/pull/config
+    # failures here may be retried: no collector has started.
+    command(*compose, "config", "--quiet", env=env)
+    if command(*compose, "ps", "--all", "--status", "running", "--quiet", env=env).strip():
+        raise ValueError("An application job is already running")
+    command("docker", "pull", request["image"])
+    image = json.loads(command("docker", "image", "inspect", request["image"]))[0]
+    if (image["Config"].get("Labels") or {}).get("org.opencontainers.image.revision") != request["revision"]:
+        raise ValueError("Image revision mismatch")
+    # Persist before launch. A crash here can lose a run, but can never silently
+    # repeat one. Unknown/failed attempts require explicit investigation.
+    result = {"request": request, "status": "attempted", "exit_code": None}
+    atomic_json(outcome, result)
+    print(f"Starting request {request['request_id']}", flush=True)
+    try:
+        child = subprocess.run([*compose, "run", "--rm", "--no-deps", "--pull", "never",
+                                "--name", f"car-tracker-run-{request['request_id']}", "app"], env=env)
+        result.update(status="finished", exit_code=child.returncode)
+        atomic_json(outcome, result)
+        print(f"Request {request['request_id']} finished with pipeline exit {child.returncode}", flush=True)
+    except (OSError, KeyboardInterrupt):
+        # Keep attempted/unknown instead of retrying: the child or DB transaction
+        # may still be running or have committed. Recovery uses the raw manifest.
+        print("Run outcome unknown; investigate the container and manifest before another request.", flush=True)
+        raise
+
+
+def reconcile(root):
+    state = root / "state/requests"
+    state.mkdir(parents=True, mode=0o700, exist_ok=True)
+    with (root / "state/agent.lock").open("a") as lock:
+        try:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            return
+        for outcome in state.glob("*.json"):
+            if json.loads(outcome.read_text()).get("status") == "attempted":
+                raise ValueError("An earlier run has an unknown outcome; resolve it before consuming more requests")
+        repo = root / "desired.git"
+        if not repo.exists():
+            raise ValueError("Initialize the dedicated read-only Git checkout first")
+        # No git checkout, reset, or modification of a developer's worktree.
+        command("git", "-C", str(repo), "fetch", "origin",
+                "+refs/heads/gitops:refs/remotes/origin/gitops")
+        commits = command("git", "-C", str(repo), "log", "--reverse", "--format=%H",
+                          "refs/remotes/origin/gitops", "--", "deploy/request.json").splitlines()
+        # Read every request commit, not just the latest: two button presses
+        # between polls must not silently discard one request.
+        for commit in commits:
+            raw = command("git", "-C", str(repo), "show", f"{commit}:deploy/request.json")
+            request = validate_request(json.loads(raw))
+            attempt(request, state, root / "compose.runtime.yaml", root / "app.env")
+
+
+def main():
+    os.umask(0o077)
+    root = Path(os.environ.get("CAR_TRACKER_STACK_DIR", str(Path.home() / "containers/car-tracker"))).resolve()
+    try:
+        reconcile(root)
+        return 0
+    except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError):
+        print("Agent failed: check local configuration, Git/registry access and request state. No automatic scrape retry.")
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ops/gitops/request.py

@@ -0,0 +1,97 @@
+"""Queue a digest-pinned, uniquely identified run on the gitops branch."""
+
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import tempfile
+import uuid
+
+
+def command(*args, cwd=None, env=None):
+    return subprocess.run(args, cwd=cwd, env=env, check=True, text=True,
+                          stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.strip()
+
+
+def resolve_release(release):
+    if re.fullmatch(r"[0-9a-f]{40}", release):
+        ref = release
+    else:
+        command("git", "check-ref-format", f"refs/tags/{release}")
+        ref = f"refs/tags/{release}"
+    sha = command("git", "rev-parse", "--verify", f"{ref}^{{commit}}")
+    if not re.fullmatch(r"[0-9a-f]{40}", sha):
+        raise ValueError("Invalid revision")
+    # Only revisions in the trusted default branch's history may run.
+    command("git", "merge-base", "--is-ancestor", sha, "origin/master")
+    return sha
+
+
+def image_digest(reference, sha):
+    command("docker", "pull", reference)
+    image = json.loads(command("docker", "image", "inspect", reference))[0]
+    if (image["Config"].get("Labels") or {}).get("org.opencontainers.image.revision") != sha:
+        raise ValueError("Image revision label does not match the selected release")
+    repository = reference.rsplit(":", 1)[0]
+    matches = [item for item in image.get("RepoDigests", [])
+               if item.startswith(repository + "@sha256:")]
+    if len(matches) != 1 or not re.fullmatch(re.escape(repository) + r"@sha256:[0-9a-f]{64}", matches[0]):
+        raise ValueError("No unambiguous registry digest")
+    return matches[0]
+
+
+def queue(request):
+    user, token = os.environ["GITOPS_USER"], os.environ["GITOPS_TOKEN"]
+    if not user or not token:
+        raise ValueError("GitOps credentials are required")
+    url = os.environ["GITOPS_URL"]
+    if url != "https://gitea.vandeplanque.com/tonio/car-tracker.git":
+        raise ValueError("Unexpected GitOps repository")
+    branch = os.environ.get("GITOPS_BRANCH", "gitops")
+    if branch != "gitops":
+        raise ValueError("Unexpected state branch")
+    with tempfile.TemporaryDirectory(prefix="car-tracker-request-") as temporary:
+        root = Path(temporary)
+        askpass = root / "askpass.sh"
+        askpass.write_text('#!/bin/sh\ncase "$1" in\n*Username*) printf "%s\\n" "$GITOPS_USER" ;;\n*) printf "%s\\n" "$GITOPS_TOKEN" ;;\nesac\n')
+        askpass.chmod(0o700)
+        env = {**os.environ, "GIT_ASKPASS": str(askpass), "GIT_TERMINAL_PROMPT": "0"}
+        repo = root / "state"
+        repo.mkdir()
+        command("git", "init", cwd=repo, env=env)
+        command("git", "remote", "add", "origin", url, cwd=repo, env=env)
+        exists = command("git", "ls-remote", "--heads", "origin", f"refs/heads/{branch}", cwd=repo, env=env)
+        source = branch if exists else "master"
+        command("git", "fetch", "origin", source, cwd=repo, env=env)
+        command("git", "checkout", "-b", branch, "FETCH_HEAD", cwd=repo, env=env)
+        path = repo / "deploy/request.json"
+        path.parent.mkdir(exist_ok=True)
+        path.write_text(json.dumps(request, indent=2) + "\n")
+        command("git", "add", "deploy/request.json", cwd=repo, env=env)
+        command("git", "-c", "user.name=Car tracker Actions", "-c", "user.email=actions@localhost",
+                "commit", "-m", f"Queue app run {request['request_id']} ({request['revision'][:12]})", cwd=repo, env=env)
+        # Normal push is compare-and-swap: concurrent requests fail rather than
+        # silently overwriting each other. Never force-push the request history.
+        command("git", "push", "origin", f"HEAD:refs/heads/{branch}", cwd=repo, env=env)
+
+
+def main():
+    try:
+        sha = resolve_release(os.environ["RELEASE"])
+        reference = f"{os.environ['REGISTRY']}/{os.environ['IMAGE_REPOSITORY']}:{sha}"
+        digest = image_digest(reference, sha)
+        request = {"schema_version": 1, "request_id": str(uuid.uuid4()), "revision": sha, "image": digest}
+        queue(request)
+        print(f"Queued request: {request['request_id']}")
+        print(f"Image: {digest}")
+        print("This confirms Git publication only. Inspect the production agent's outcome for collection/import status.")
+        return 0
+    except (KeyError, ValueError, OSError, subprocess.SubprocessError):
+        # Git/registry errors may contain authentication details: don't echo them.
+        print("Request failed: check the release, published image, credentials and state branch. No run was confirmed.")
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_gitops.py

@@ -0,0 +1,183 @@
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+import uuid
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def load(name):
+    spec = importlib.util.spec_from_file_location(name, ROOT / "ops/gitops" / f"{name}.py")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+agent = load("reconcile")
+requester = load("request")
+
+
+class GitOpsTests(unittest.TestCase):
+    def setUp(self):
+        self.request = {"schema_version": 1, "request_id": str(uuid.uuid4()),
+                        "revision": "a" * 40, "image": agent.IMAGE_REPOSITORY + "@sha256:" + "b" * 64}
+        self.temporary = self.enterContext(tempfile.TemporaryDirectory())
+        self.state = Path(self.temporary)
+
+    def attempt(self):
+        agent.attempt(self.request, self.state, self.state / "compose.yaml", self.state / "app.env")
+
+    def commands(self, *args, **kwargs):
+        if "inspect" in args:
+            return json.dumps([{"Config": {"Labels": {"org.opencontainers.image.revision": "a" * 40}}}])
+        return ""
+
+    def test_validate_request_rejects_mutable_tag_extra_keys_and_invalid_id(self):
+        agent.validate_request(self.request)
+        for field, value in [("image", agent.IMAGE_REPOSITORY + ":latest"),
+                             ("request_id", "../../secret"), ("revision", "master"),
+                             ("command", "migrate")]:
+            with self.subTest(field=field), self.assertRaises(ValueError):
+                agent.validate_request({**self.request, field: value})
+
+    def test_atomic_state_is_private(self):
+        path = self.state / "outcome.json"
+        agent.atomic_json(path, self.request)
+        self.assertEqual(json.loads(path.read_text()), self.request)
+        self.assertEqual(path.stat().st_mode & 0o777, 0o600)
+
+    def test_attempt_is_written_before_launch_and_failure_is_not_retried(self):
+        path = self.state / f"{self.request['request_id']}.json"
+
+        def launch(*args, **kwargs):
+            self.assertEqual(json.loads(path.read_text())["status"], "attempted")
+            return subprocess.CompletedProcess(args, 3)
+
+        with patch.object(agent, "command", side_effect=self.commands), \
+                patch.object(agent.subprocess, "run", side_effect=launch) as run:
+            self.attempt()
+            self.attempt()
+        self.assertEqual(run.call_count, 1)
+        self.assertEqual(json.loads(path.read_text())["exit_code"], 3)
+
+    def test_crash_preserves_unknown_attempt(self):
+        with patch.object(agent, "command", side_effect=self.commands), \
+                patch.object(agent.subprocess, "run", side_effect=OSError):
+            with self.assertRaises(OSError):
+                self.attempt()
+        outcome = json.loads((self.state / f"{self.request['request_id']}.json").read_text())
+        self.assertEqual(outcome["status"], "attempted")
+        self.assertIsNone(outcome["exit_code"])
+
+    def test_pull_failure_does_not_consume_request(self):
+        def commands(*args, **kwargs):
+            if "pull" in args:
+                raise subprocess.CalledProcessError(1, args)
+            return ""
+
+        with patch.object(agent, "command", side_effect=commands), \
+                patch.object(agent.subprocess, "run") as run:
+            with self.assertRaises(subprocess.CalledProcessError):
+                self.attempt()
+            run.assert_not_called()
+        self.assertFalse(list(self.state.glob("*.json")))
+
+    def test_revision_mismatch_does_not_launch(self):
+        def commands(*args, **kwargs):
+            if "inspect" in args:
+                return '[{"Config": {"Labels": {}}}]'
+            return ""
+
+        with patch.object(agent, "command", side_effect=commands), self.assertRaises(ValueError):
+            self.attempt()
+        self.assertFalse(list(self.state.glob("*.json")))
+
+    def test_request_id_cannot_be_reused_for_different_image(self):
+        path = self.state / f"{self.request['request_id']}.json"
+        agent.atomic_json(path, {"request": {**self.request, "revision": "c" * 40}, "status": "finished"})
+        with self.assertRaises(ValueError):
+            self.attempt()
+
+    def test_environment_does_not_override_external_app_env(self):
+        with patch.dict("os.environ", {"PGHOST": "wrong", "RAW_DATA_DIR": "wrong", "APP_UID": "0"}):
+            env = agent.compose_environment(self.request["image"])
+        self.assertNotIn("PGHOST", env)
+        self.assertNotIn("RAW_DATA_DIR", env)
+        self.assertNotIn("APP_UID", env)
+        self.assertEqual(env["CAR_TRACKER_IMAGE"], self.request["image"])
+
+    def test_branch_names_are_not_release_inputs(self):
+        calls = []
+
+        def command(*args):
+            calls.append(args)
+            if "rev-parse" in args:
+                raise subprocess.CalledProcessError(1, args)
+            return ""
+
+        with patch.object(requester, "command", side_effect=command), \
+                self.assertRaises(subprocess.CalledProcessError):
+            requester.resolve_release("master")
+        self.assertIn(("git", "rev-parse", "--verify", "refs/tags/master^{commit}"), calls)
+
+    def test_multiple_requests_between_polls_are_consumed_in_order(self):
+        root = self.state
+        (root / "desired.git").mkdir()
+        second = {**self.request, "request_id": str(uuid.uuid4())}
+
+        def commands(*args, **kwargs):
+            if "log" in args:
+                return "commit1\ncommit2\n"
+            if "show" in args:
+                return json.dumps(second if args[-1].startswith("commit2:") else self.request)
+            return ""
+
+        with patch.object(agent, "command", side_effect=commands), patch.object(agent, "attempt") as attempt:
+            agent.reconcile(root)
+        self.assertEqual([call.args[0] for call in attempt.call_args_list], [self.request, second])
+
+    def test_unknown_attempt_blocks_new_requests(self):
+        state = self.state / "state/requests"
+        state.mkdir(parents=True)
+        agent.atomic_json(state / "old.json", {"status": "attempted"})
+        with patch.object(agent, "command") as command, self.assertRaises(ValueError):
+            agent.reconcile(self.state)
+        command.assert_not_called()
+
+    def test_queue_creates_only_state_branch_and_retains_request_history(self):
+        seed = self.state / "seed"
+        remote = self.state / "remote.git"
+        seed.mkdir()
+        requester.command("git", "init", "--initial-branch=master", cwd=seed)
+        (seed / "README.md").write_text("Trusted application source\n")
+        requester.command("git", "add", "README.md", cwd=seed)
+        requester.command("git", "-c", "user.name=Test", "-c", "user.email=test@localhost",
+                          "commit", "-m", "initial", cwd=seed)
+        requester.command("git", "clone", "--bare", str(seed), str(remote))
+        before = requester.command("git", "-C", str(remote), "rev-parse", "master")
+        real_command = requester.command
+
+        def commands(*args, **kwargs):
+            if args[:4] == ("git", "remote", "add", "origin"):
+                args = (*args[:4], str(remote))
+            return real_command(*args, **kwargs)
+
+        second = {**self.request, "request_id": str(uuid.uuid4())}
+        with patch.dict("os.environ", {
+            "GITOPS_USER": "test", "GITOPS_TOKEN": "test-only",
+            "GITOPS_BRANCH": "gitops", "GITOPS_URL": "https://gitea.vandeplanque.com/tonio/car-tracker.git",
+        }), patch.object(requester, "command", side_effect=commands):
+            requester.queue(self.request)
+            requester.queue(second)
+        self.assertEqual(real_command("git", "-C", str(remote), "rev-parse", "master"), before)
+        commits = real_command("git", "-C", str(remote), "log", "--reverse", "--format=%H",
+                               "gitops", "--", "deploy/request.json").splitlines()
+        self.assertEqual(len(commits), 2)
+        requests = [json.loads(real_command("git", "-C", str(remote), "show",
+                                            f"{commit}:deploy/request.json")) for commit in commits]
+        self.assertEqual(requests, [self.request, second])