From e57144c9b1778271190a2f6e1f39ee48be9058a2 Mon Sep 17 00:00:00 2001 From: Antoine Vandeplanque Date: Thu, 1 Oct 2026 17:20:40 +0200 Subject: [PATCH] run selected releases through restricted SSH instead of GitOps polling --- .gitea/workflows/ci.yaml | 2 +- .gitea/workflows/run-app.yaml | 18 +++-- PROJECT_PLAN.md | 6 +- README.md | 6 +- docs/CI.md | 4 +- docs/DEPLOYMENT.md | 4 +- docs/GITOPS.md | 4 + docs/SSH_RUN.md | 134 ++++++++++++++++++++++++++++++++++ ops/gitops/reconcile.py | 20 ++++- ops/ssh/host_run.py | 48 ++++++++++++ ops/ssh/run.py | 47 ++++++++++++ tests/test_ssh_run.py | 76 +++++++++++++++++++ 12 files changed, 347 insertions(+), 22 deletions(-) create mode 100644 docs/SSH_RUN.md create mode 100644 ops/ssh/host_run.py create mode 100644 ops/ssh/run.py create mode 100644 tests/test_ssh_run.py diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 4dd7528..710a35a 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -46,7 +46,7 @@ jobs: run: uv sync --locked --dev - name: Lint - run: uv run --locked ruff check src tests ops/gitops + run: uv run --locked ruff check src tests ops/gitops ops/ssh - name: Run offline and PostgreSQL integration tests run: timeout 10m uv run --locked python -m unittest discover -s tests -v diff --git a/.gitea/workflows/run-app.yaml b/.gitea/workflows/run-app.yaml index d7049dc..73939fa 100644 --- a/.gitea/workflows/run-app.yaml +++ b/.gitea/workflows/run-app.yaml @@ -10,7 +10,7 @@ on: jobs: request: - name: Queue one production run through Git + name: Run selected image over restricted SSH # Execute trusted control-plane code, never a user-selected release workflow. if: github.ref == 'refs/heads/master' runs-on: python-ci @@ -19,8 +19,6 @@ jobs: IMAGE_REPOSITORY: tonio/car-tracker DOCKER_CONFIG: /tmp/car-tracker-docker-auth RELEASE: ${{ github.event.inputs.release }} - GITOPS_URL: https://gitea.vandeplanque.com/tonio/car-tracker.git - GITOPS_BRANCH: gitops steps: - name: Check out trusted master workflow revision uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 @@ -51,11 +49,17 @@ jobs: mkdir -p "$DOCKER_CONFIG" printf '%s' "$REGISTRY_TOKEN" | docker login "$REGISTRY" --username "$REGISTRY_USER" --password-stdin - - name: Resolve published release and commit run request + - name: Install SSH client + run: | + apt-get update + apt-get install -y --no-install-recommends openssh-client + + - name: Run selected release on homelab env: - GITOPS_USER: ${{ secrets.GITOPS_USER }} - GITOPS_TOKEN: ${{ secrets.GITOPS_TOKEN }} - run: uv run --no-project --python 3.13 python ops/gitops/request.py + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }} + GITHUB_RUN_ID: ${{ github.run_id }} + run: uv run --no-project --python 3.13 python ops/ssh/run.py - name: Remove job registry credentials if: always() diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md index daffab1..f95f048 100644 --- a/PROJECT_PLAN.md +++ b/PROJECT_PLAN.md @@ -28,9 +28,9 @@ The manual homelab job is now packaged in `Dockerfile` and `ops/app/compose.yaml `car-tracker-pipeline` performs a database preflight, bounded collection, and an idempotent import of the exact returned manifest, retaining partial coverage and separate operational outcomes. There is no scheduler or automatic deployment. -The optional Gitea Actions/GitOps request workflow is documented in -`docs/GITOPS.md`: tests/build/publish occur on a dedicated runner VM; explicit -requests are pulled and consumed once on the production VM. It is not a daily +The optional Gitea Actions/restricted SSH workflow is documented in +`docs/SSH_RUN.md`: tests/build/publish occur on a dedicated runner VM; explicit +requests run selected images on the production VM over SSH. It is not a daily collection schedule, and migrations remain manual. See `docs/DEPLOYMENT.md`; container tests use isolated PostgreSQL 17 and synthetic browser responses. Actual live access from the homelab still needs verification. diff --git a/README.md b/README.md index 27209e2..455605a 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,9 @@ and validation issues. Generation/trim inference, redaction, and scheduling are ## Manual Homelab Job -For a manual **Run app** button in Gitea Actions, see [GitOps jobs](docs/GITOPS.md). -CI publishes tested images; the button queues a digest-pinned request, and the -production host consumes it once using host-local DB credentials. No automatic +For a manual **Run app** button in Gitea Actions, see [SSH jobs](docs/SSH_RUN.md). +CI publishes tested images; the button runs a digest-pinned image through a +restricted SSH command using host-local DB credentials. No automatic migrations or retries of failed scrapes are introduced. `car-tracker-pipeline` checks database connectivity/schema, collects raw observations, diff --git a/docs/CI.md b/docs/CI.md index 41f97dc..49e83b8 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -2,8 +2,8 @@ This repository targets **Gitea 1.23.8**. The workflow is in `.gitea/workflows/ci.yaml`; it runs on pushes, pull requests, and manual dispatch. -It tests and publishes images on master. Manual **Run app** dispatch queues a -GitOps job; see [GITOPS.md](GITOPS.md) for credentials and production installation. +It tests and publishes images on master. Manual **Run app** dispatch runs a +selected image over restricted SSH; see [SSH_RUN.md](SSH_RUN.md) for setup. No live scraping, production migration, or production DB access occurs in CI. ## What Runs diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 870b352..c8af85d 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -9,8 +9,8 @@ Do not provision another production PostgreSQL server or recreate those objects. `ops/app/compose.yaml` is the one-shot application stack, separate from `ops/ci-runner`. This section describes direct manual operation, with no scheduling -or restart loop. For the optional Gitea button → GitOps request → host-run flow, -see [GITOPS.md](GITOPS.md). Migrations remain manual in both modes. +or restart loop. For the optional Gitea button → restricted SSH → host-run flow, +see [SSH_RUN.md](SSH_RUN.md). Migrations remain manual in both modes. Run commands from the homelab checkout's root with Docker Engine and Compose available. Review the revision and `searches.toml`; do not run these in CI. After these changes are published, update the homelab checkout before building. diff --git a/docs/GITOPS.md b/docs/GITOPS.md index 983e996..43925bb 100644 --- a/docs/GITOPS.md +++ b/docs/GITOPS.md @@ -1,5 +1,9 @@ # Gitea-triggered application jobs +> **Retired:** the active Run app workflow now uses restricted SSH. Follow +> [SSH_RUN.md](SSH_RUN.md), keep the reconcile timer disabled, and do not use +> the installation/activation steps below. Retained for historical recovery context. + ## Flow The app is a one-shot collector/importer, not a continuously running service. diff --git a/docs/SSH_RUN.md b/docs/SSH_RUN.md new file mode 100644 index 0000000..27edca2 --- /dev/null +++ b/docs/SSH_RUN.md @@ -0,0 +1,134 @@ +# Manual application runs through restricted SSH + +Pushes to master still test, build and publish SHA-tagged images. **Run app** now +resolves a published full SHA or Git tag to an immutable image digest and invokes +a fixed SSH command on `tonio@192.168.1.56:22`. The build VM must reach that LAN +SSH listener (not Gitea's port 2222). No timer or Git request branch is used. + +Actions waits for the pipeline's exit status. Exit 0 is green; any nonzero pipeline +exit fails the job. Green does not guarantee complete source coverage: see +[pipeline outcomes](DEPLOYMENT.md#outcomes-and-recovery). Scraper output stays in +private host logs; CI gets only request/revision/exit summaries. Migrations are +manual, DB credentials stay on the homelab, and there are no automatic retries. + +## 1. Replace the old consumer manually + +On the homelab, disable polling and check that no old job is active: + +```sh +systemctl --user disable --now car-tracker-reconcile.timer +systemctl --user is-active car-tracker-reconcile.service +``` + +If active, let the job finish. Preserve existing state and the bare repository; +do not delete them or rewrite request history. Outstanding GitOps requests are +not executed by the SSH workflow. Review them before requesting any replacement. + +From the updated project root, install these files: + +```sh +install -m 0700 ops/gitops/reconcile.py ~/containers/car-tracker/reconcile.py +install -m 0700 ops/ssh/host_run.py ~/containers/car-tracker/host_run.py +install -m 0600 ops/app/compose.runtime.yaml ~/containers/car-tracker/compose.runtime.yaml +``` + +`reconcile.py` now supplies shared validation, state and execution helpers; its +timer entry point is unused. Existing `app.env`, raw/db mounts and production +read-only registry login remain unchanged. It uses the same consumed-request +ledger and lock as the retired agent. + +## 2. Create a dedicated SSH key + +On a trusted machine (the homelab is fine), generate a key specifically for Actions: + +```sh +ssh-keygen -t ed25519 -N '' -f ~/.ssh/car-tracker-actions -C car-tracker-actions +``` + +Do not overwrite an existing key. The unencrypted private key goes **only** into +the repository Actions secret `SSH_PRIVATE_KEY`, including its BEGIN/END lines. +Never commit it or paste it into chat. The CI runner gains a narrow production +access credential; trust all workflows with access to repository secrets. + +Append the PUBLIC key to the homelab user's `~/.ssh/authorized_keys` as ONE line, +prefixed with these exact restrictions (replace the public-key placeholder): + +```text +restrict,command="/usr/bin/python3 /home/tonio/containers/car-tracker/host_run.py" ssh-ed25519 car-tracker-actions +``` + +Keep existing authorized keys intact. The directory should be mode 0700 and +authorized_keys mode 0600. If the builder has a reserved LAN address, additionally +use `from=""` before `restrict`; don't guess its source address. + +`restrict` disables PTY, forwarding and user rc execution. The forced command +does not eval the caller's command: it accepts only `run ` +for the fixed image repository. Arbitrary shell, SFTP/SCP, migrations and Compose +arguments are rejected. This is not a sandbox against malicious application +images: approved images still have the application's mounted DB credentials. + +## 3. Pin the homelab SSH host key + +On the homelab, obtain the public host key and fingerprint through your trusted +existing session: + +```sh +cat /etc/ssh/ssh_host_ed25519_key.pub +ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub +``` + +Create repository Actions secret `SSH_KNOWN_HOSTS` with: + +```text +192.168.1.56 ssh-ed25519 +``` + +Use the host key's first two fields, not the Actions key. Never trust an unverified +ssh-keyscan result. The workflow enforces StrictHostKeyChecking and key-only auth. +No Docker/DB/admin access is granted to the SSH key directly; however the host +wrapper runs as the existing Docker-enabled user, so protect its files and account. + +Keep `REGISTRY_USER` and `REGISTRY_TOKEN`. Remove `GITOPS_USER`/`GITOPS_TOKEN` +Actions secrets and revoke the now-unused GitOps token after switching. Keep the +separate production read-only package token. + +## 4. Test restrictions without scraping, then trigger + +From a machine that can reach the homelab, using the dedicated key and a verified +known_hosts entry: + +```sh +ssh -T -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ + -i ~/.ssh/car-tracker-actions tonio@192.168.1.56 true +``` + +Expected: refusal and exit 70, not a shell or successful `true`. This verifies the +forced command without running the app. Verify LAN connectivity from the builder +as well. Keep polling disabled permanently. + +After publishing the updated workflows, select **Actions → Run app → Run workflow**, +branch **master**, and enter a published full SHA or Git tag. This starts a real +one-shot collection/import immediately, not just a queue request. + +## State, failures and recovery + +Local outcomes: `~/containers/car-tracker/state/requests/.json`. +Private runtime logs: the neighboring `.log`. Host failures before launch +return 70; SSH connection failures return 255. Inspect host state/logs locally. + +The request UUID is derived from the Actions run ID. Retrying the same workflow +run does not repeat a consumed request; it returns the recorded exit status. A +new manual dispatch creates a new request, even for the same release. Parallel +runs are refused by the host lock, not queued. + +Attempts are persisted before launch. Unknown outcomes block new requests. SSH +disconnects or Actions cancellation can leave a container running or an import +committed; the client cannot safely infer failure. Inspect the named container, +raw manifest and DB before any new dispatch. Recover uncertain imports from the +exact finalized manifest rather than scraping again. Only after investigation, +retain the record and set status to `resolved` to unblock new requests. Do not +delete the ledger to force retries. Back it up with raw files and the database. + +The workflow waits synchronously within the runner's 45-minute timeout. If that +timeout fires, treat the outcome as unknown, not as permission to retry. Manual +CLI/recovery operations must also respect the application's existing pipeline lock. diff --git a/ops/gitops/reconcile.py b/ops/gitops/reconcile.py index 5695fe4..57841fb 100644 --- a/ops/gitops/reconcile.py +++ b/ops/gitops/reconcile.py @@ -64,13 +64,16 @@ def compose_environment(image): return env -def attempt(request, state, compose_file, env_file): +def attempt(request, state, compose_file, env_file, private_log=False): outcome = state / f"{request['request_id']}.json" if outcome.exists(): recorded = json.loads(outcome.read_text()) if recorded["request"] != request: raise ValueError("Request identifier reused with different content") - return + if recorded.get("status") != "finished" or not isinstance(recorded.get("exit_code"), int): + raise ValueError("Previous request outcome is unknown or manually resolved; do not retry") + print(f"Previously finished request {request['request_id']}: exit {recorded['exit_code']}", flush=True) + return recorded["exit_code"] env = compose_environment(request["image"]) compose = ["docker", "compose", "--env-file", str(env_file), "-f", str(compose_file)] # Fail-closed checks occur before consuming a request. Network/pull/config @@ -88,11 +91,20 @@ def attempt(request, state, compose_file, env_file): atomic_json(outcome, result) print(f"Starting request {request['request_id']}", flush=True) try: - child = subprocess.run([*compose, "run", "--rm", "--no-deps", "--pull", "never", - "--name", f"car-tracker-run-{request['request_id']}", "app"], env=env) + args = [*compose, "run", "--rm", "--no-deps", "--pull", "never", + "--name", f"car-tracker-run-{request['request_id']}", "app"] + if private_log: + log_path = state / f"{request['request_id']}.log" + descriptor = os.open(log_path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "a") as output: + child = subprocess.run(args, env=env, stdout=output, stderr=subprocess.STDOUT) + else: + child = subprocess.run(args, env=env) result.update(status="finished", exit_code=child.returncode) atomic_json(outcome, result) print(f"Request {request['request_id']} finished with pipeline exit {child.returncode}", flush=True) + return child.returncode if child.returncode >= 0 else 70 except (OSError, KeyboardInterrupt): # Keep attempted/unknown instead of retrying: the child or DB transaction # may still be running or have committed. Recovery uses the raw manifest. diff --git a/ops/ssh/host_run.py b/ops/ssh/host_run.py new file mode 100644 index 0000000..6ad0b41 --- /dev/null +++ b/ops/ssh/host_run.py @@ -0,0 +1,48 @@ +"""Forced SSH command: accept only a fixed verb, UUID, revision and image digest.""" + +import fcntl +import json +import os +from pathlib import Path +import subprocess +import sys + +from reconcile import attempt, validate_request + + +def parse_command(value): + parts = value.split() + if len(parts) != 4 or parts[0] != "run": + raise ValueError("Only the fixed run command is allowed") + return validate_request({"schema_version": 1, "request_id": parts[1], + "revision": parts[2], "image": parts[3]}) + + +def run(root, request): + state = root / "state/requests" + state.mkdir(parents=True, mode=0o700, exist_ok=True) + with (root / "state/agent.lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise ValueError("Another job is active") + for path in state.glob("*.json"): + if json.loads(path.read_text()).get("status") == "attempted": + raise ValueError("Unknown previous outcome requires investigation") + return attempt(request, state, root / "compose.runtime.yaml", root / "app.env", private_log=True) + + +def main(): + os.umask(0o077) + try: + # Fixed location: the SSH caller cannot override the stack or environment. + root = Path.home() / "containers/car-tracker" + request = parse_command(os.environ.get("SSH_ORIGINAL_COMMAND", "")) + return run(root, request) + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError): + print("Run refused or failed before completion. Inspect private host state/logs; do not automatically retry.") + return 70 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/ssh/run.py b/ops/ssh/run.py new file mode 100644 index 0000000..9deed64 --- /dev/null +++ b/ops/ssh/run.py @@ -0,0 +1,47 @@ +"""Resolve an approved image and invoke the restricted production SSH command.""" + +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "gitops")) +from request import image_digest, resolve_release # noqa: E402 + + +def main(): + os.umask(0o077) + try: + sha = resolve_release(os.environ["RELEASE"]) + image = image_digest(f"gitea.vandeplanque.com/tonio/car-tracker:{sha}", sha) + # Stable across job retries; a new manual dispatch has a different run ID. + request_id = str(uuid.uuid5(uuid.NAMESPACE_URL, + "https://gitea.vandeplanque.com/tonio/car-tracker/actions/runs/" + + os.environ["GITHUB_RUN_ID"])) + with tempfile.TemporaryDirectory(prefix="car-tracker-ssh-") as directory: + key, hosts = Path(directory) / "key", Path(directory) / "known_hosts" + key.write_text(os.environ["SSH_PRIVATE_KEY"].strip() + "\n") + hosts.write_text(os.environ["SSH_KNOWN_HOSTS"].strip() + "\n") + key.chmod(0o600) + hosts.chmod(0o600) + print(f"Running request {request_id}; revision {sha}", flush=True) + result = subprocess.run([ + "ssh", "-F", "/dev/null", "-T", "-i", str(key), + "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes", + "-o", "StrictHostKeyChecking=yes", "-o", f"UserKnownHostsFile={hosts}", + "-o", "ConnectTimeout=15", "-o", "ServerAliveInterval=30", + "-o", "ServerAliveCountMax=3", "tonio@192.168.1.56", + f"run {request_id} {sha} {image}", + ]) + if result.returncode == 255: + print("SSH connection failed/lost. The job may still be running; inspect the host before another dispatch.") + return result.returncode if result.returncode >= 0 else 70 + except (KeyError, ValueError, OSError, subprocess.SubprocessError): + print("Run failed: check release, image and SSH configuration. No completion confirmed.") + return 70 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_ssh_run.py b/tests/test_ssh_run.py new file mode 100644 index 0000000..3f4cacb --- /dev/null +++ b/tests/test_ssh_run.py @@ -0,0 +1,76 @@ +import importlib.util +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch +import uuid + + +ROOT = Path(__file__).resolve().parents[1] + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, ROOT / path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +agent = load("reconcile", "ops/gitops/reconcile.py") +with patch.dict("sys.modules", {"reconcile": agent}): + host = load("host_run", "ops/ssh/host_run.py") + + +class SSHRunTests(unittest.TestCase): + def setUp(self): + self.root = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.request = {"schema_version": 1, "request_id": str(uuid.uuid4()), + "revision": "a" * 40, "image": agent.IMAGE_REPOSITORY + "@sha256:" + "b" * 64} + self.command = f"run {self.request['request_id']} {self.request['revision']} {self.request['image']}" + + def test_only_fixed_command_is_accepted(self): + self.assertEqual(host.parse_command(self.command), self.request) + for command in ("", "bash", "migrate", self.command + " ; id", + self.command.replace("@sha256:" + "b" * 64, ":latest"), + self.command.replace("run ", "run; ", 1)): + with self.subTest(command=command), self.assertRaises(ValueError): + host.parse_command(command) + + def test_host_returns_pipeline_exit_and_keeps_output_private(self): + def command(*args, **kwargs): + if "inspect" in args: + return json.dumps([{"Config": {"Labels": {"org.opencontainers.image.revision": "a" * 40}}}]) + return "" + + def launch(args, **kwargs): + self.assertEqual(kwargs["stderr"], subprocess.STDOUT) + kwargs["stdout"].write("Private collection output\n") + return subprocess.CompletedProcess(args, 3) + + with patch.object(agent, "command", side_effect=command), \ + patch.object(agent.subprocess, "run", side_effect=launch) as child: + self.assertEqual(host.run(self.root, self.request), 3) + self.assertEqual(host.run(self.root, self.request), 3) + self.assertEqual(child.call_count, 1) + log = self.root / "state/requests" / f"{self.request['request_id']}.log" + self.assertIn("Private collection output", log.read_text()) + self.assertEqual(log.stat().st_mode & 0o777, 0o600) + + def test_unknown_prior_attempt_blocks_launch(self): + state = self.root / "state/requests" + state.mkdir(parents=True) + agent.atomic_json(state / "old.json", {"status": "attempted"}) + with patch.object(host, "attempt") as attempt, self.assertRaises(ValueError): + host.run(self.root, self.request) + attempt.assert_not_called() + + def test_busy_lock_refuses_another_job(self): + import fcntl + + (self.root / "state").mkdir() + with (self.root / "state/agent.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + with self.assertRaises(ValueError): + host.run(self.root, self.request)