391 lines
20 KiB
Python
391 lines
20 KiB
Python
"""Opt-in tests against a disposable PostgreSQL database named *_test."""
|
|
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timedelta, timezone
|
|
from decimal import Decimal
|
|
import gzip
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from threading import Barrier
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from uuid import uuid4
|
|
|
|
import psycopg
|
|
from psycopg import sql
|
|
from psycopg.conninfo import conninfo_to_dict
|
|
|
|
from car_traker import database
|
|
from car_traker.collect import search_url, write_json, write_records
|
|
|
|
|
|
@unittest.skipUnless(os.environ.get("TEST_DATABASE_URL"), "TEST_DATABASE_URL is not set")
|
|
class DatabaseIntegrationTests(unittest.TestCase):
|
|
tables = (
|
|
"collection_runs", "raw_files", "search_coverage", "detail_coverage",
|
|
"listings", "observations", "rejected_records",
|
|
)
|
|
|
|
@classmethod
|
|
def connect(cls):
|
|
url = os.environ["TEST_DATABASE_URL"]
|
|
# Do not infer a target from libpq defaults or service configuration.
|
|
name = conninfo_to_dict(url).get("dbname", "")
|
|
if not name.endswith("_test"):
|
|
raise AssertionError("TEST_DATABASE_URL must explicitly name a database ending '_test'")
|
|
conn = psycopg.connect(url, autocommit=True, connect_timeout=10)
|
|
try:
|
|
if not conn.execute("SELECT current_database()").fetchone()[0].endswith("_test"):
|
|
raise AssertionError("Connected database name must end '_test'")
|
|
conn.execute("SET statement_timeout = '15s'")
|
|
conn.execute("SET lock_timeout = '10s'")
|
|
except BaseException:
|
|
conn.close()
|
|
raise
|
|
return conn
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.conn = cls.connect()
|
|
cls.addClassCleanup(cls.conn.close)
|
|
database.migrate(cls.conn)
|
|
|
|
def setUp(self):
|
|
self.conn.execute(sql.SQL("TRUNCATE {} RESTART IDENTITY CASCADE").format(
|
|
sql.SQL(", ").join(sql.Identifier("silver", table) for table in self.tables)
|
|
))
|
|
temporary = TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
self.directory = Path(temporary.name)
|
|
|
|
def fixture(self, *, version=2, pages=None, detail=None, day=1):
|
|
"""Write raw envelopes directly; no collector or network client is run."""
|
|
if pages is None:
|
|
pages = [[{"list_id": 123, "price": ["12000"]}]]
|
|
run_id = str(uuid4())
|
|
directory = self.directory / run_id
|
|
directory.mkdir()
|
|
start = datetime(2026, 1, day, tzinfo=timezone.utc)
|
|
search = {"family": "synthetic", "text": "synthetic car"}
|
|
scope = {**search, "status": "complete", "pages": [], "records": 0}
|
|
manifest = {
|
|
"schema_version": version, "source": "leboncoin", "run_id": run_id,
|
|
"started_at": start.isoformat(),
|
|
"finished_at": (start + timedelta(hours=1)).isoformat(),
|
|
"status": "complete", "planned_searches": [search], "searches": [scope],
|
|
}
|
|
for number, payloads in enumerate(pages, 1):
|
|
observed = (start + timedelta(minutes=number)).isoformat()
|
|
filename = f"search-001-page-{number:04d}.jsonl.gz"
|
|
records = [{
|
|
"schema_version": version, "source": "leboncoin", "run_id": run_id,
|
|
"observed_at": observed, "family": search["family"],
|
|
"search_text": search["text"], "search_url": search_url(search["text"], number),
|
|
"page": number, "payload": payload,
|
|
**({"record_type": "search"} if version == 2 else {}),
|
|
} for payload in payloads]
|
|
write_records(directory / filename, records)
|
|
scope["pages"].append({
|
|
"page": number, "file": filename, "observed_at": observed,
|
|
"records": len(records), "reported_total": sum(map(len, pages)),
|
|
"reported_max_pages": len(pages),
|
|
})
|
|
scope["records"] += len(records)
|
|
if version == 2:
|
|
manifest["details"] = {"enabled": False, "status": "disabled", "listings": {}}
|
|
if detail is not None:
|
|
identifier = str(detail["list_id"])
|
|
observed = (start + timedelta(minutes=30)).isoformat()
|
|
filename = f"detail-{identifier}.jsonl.gz"
|
|
write_records(directory / filename, [{
|
|
"schema_version": version, "source": "leboncoin", "run_id": run_id,
|
|
"record_type": "detail", "observed_at": observed, "payload": detail,
|
|
"listing_url": f"https://www.leboncoin.fr/ad/voitures/{identifier}",
|
|
}])
|
|
manifest["details"] = {
|
|
"enabled": True, "status": "complete", "listings": {
|
|
identifier: {"status": "complete", "file": filename, "observed_at": observed},
|
|
},
|
|
}
|
|
path = directory / "manifest.json"
|
|
write_json(path, manifest)
|
|
return path, manifest
|
|
|
|
def state(self):
|
|
# Compare complete persisted rows, not just counts, when asserting no mutation.
|
|
return {table: self.conn.execute(sql.SQL(
|
|
"SELECT to_jsonb(t) FROM silver.{} t ORDER BY to_jsonb(t)::text"
|
|
).format(sql.Identifier(table))).fetchall() for table in (*self.tables, "schema_migrations")}
|
|
|
|
def test_migrations_are_repeatable(self):
|
|
before = self.state()
|
|
self.assertTrue(before["schema_migrations"])
|
|
database.migrate(self.conn)
|
|
database.migrate(self.conn)
|
|
self.assertEqual(self.state(), before)
|
|
|
|
def test_applied_migration_checksum_mismatch_rolls_back(self):
|
|
before = self.state()
|
|
# The outer transaction also rolls back the deliberate checksum corruption.
|
|
with self.assertRaisesRegex(database.InputError, "^applied_migration_changed$"):
|
|
with self.conn.transaction():
|
|
self.conn.execute("UPDATE silver.schema_migrations SET sha256 = %s", ("0" * 64,))
|
|
database.migrate(self.conn)
|
|
self.assertEqual(self.state(), before)
|
|
database.migrate(self.conn)
|
|
|
|
def test_import_and_replay_preserve_counts_and_rows(self):
|
|
path, manifest = self.fixture(detail={"list_id": 123, "price": ["12500"]})
|
|
self.assertEqual(database.import_run(self.conn, path), {
|
|
"run_id": manifest["run_id"], "status": "imported", "source_status": "complete",
|
|
"accepted": 2, "rejected": 0, "field_issues": 0,
|
|
})
|
|
before = self.state()
|
|
self.assertEqual({table: len(before[table]) for table in self.tables}, {
|
|
"collection_runs": 1, "raw_files": 2, "search_coverage": 1,
|
|
"detail_coverage": 1, "listings": 1, "observations": 2, "rejected_records": 0,
|
|
})
|
|
self.assertEqual(database.import_run(self.conn, path), {
|
|
"run_id": manifest["run_id"], "status": "already_imported", "rejected": 0,
|
|
})
|
|
self.assertEqual(self.state(), before)
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT expected_records, accepted_records, rejected_records FROM silver.raw_files"
|
|
).fetchall(), [(1, 1, 0), (1, 1, 0)])
|
|
|
|
def test_v1_search_without_record_type_or_details(self):
|
|
path, _ = self.fixture(version=1)
|
|
result = database.import_run(self.conn, path)
|
|
self.assertEqual((result["accepted"], result["rejected"]), (1, 0))
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT raw_schema_version, record_type, price_eur FROM silver.collection_runs "
|
|
"JOIN silver.observations USING (run_id)"
|
|
).fetchone(), (1, "search", Decimal("12000")))
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT detail_status FROM silver.v_collection_coverage"
|
|
).fetchone(), ("disabled",))
|
|
|
|
def test_later_run_preserves_past_and_updates_views(self):
|
|
first, _ = self.fixture(detail={"list_id": 123, "price": ["12500"], "brand": "Synthetic"})
|
|
database.import_run(self.conn, first)
|
|
original = self.state()["observations"]
|
|
later, manifest = self.fixture(day=2, pages=[[{"list_id": 123, "price": ["11000"]}]])
|
|
database.import_run(self.conn, later)
|
|
for row in original:
|
|
self.assertIn(row, self.state()["observations"])
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT run_id::text, price_eur, make FROM silver.v_latest_listings"
|
|
).fetchall(), [(manifest["run_id"], Decimal("11000"), None)])
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT price_eur, previous_observed_price_eur FROM silver.v_price_history "
|
|
"ORDER BY observed_at"
|
|
).fetchall(), [(Decimal("12000"), None), (Decimal("12500"), Decimal("12000")),
|
|
(Decimal("11000"), Decimal("12500"))])
|
|
self.assertEqual(len(self.state()["listings"]), 1)
|
|
|
|
def test_search_and_detail_keep_distinct_prices_and_times(self):
|
|
path, manifest = self.fixture(detail={"list_id": 123, "price": ["12500"]})
|
|
database.import_run(self.conn, path)
|
|
search = manifest["searches"][0]["pages"][0]
|
|
detail = manifest["details"]["listings"]["123"]
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT record_type, price_eur, observed_at, discovery_family "
|
|
"FROM silver.v_listing_observations ORDER BY observed_at"
|
|
).fetchall(), [
|
|
("search", Decimal("12000"), datetime.fromisoformat(search["observed_at"]), "synthetic"),
|
|
("detail", Decimal("12500"), datetime.fromisoformat(detail["observed_at"]), None),
|
|
])
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT record_type, price_eur, observed_at FROM silver.v_latest_listings"
|
|
).fetchone(), ("detail", Decimal("12500"), datetime.fromisoformat(detail["observed_at"])))
|
|
|
|
def test_invalid_fields_are_null_with_sanitized_issues(self):
|
|
path, _ = self.fixture(pages=[[{
|
|
"list_id": 123, "price": ["private-invalid-price"],
|
|
"attributes": [{"key": "mileage", "value": "-1"}],
|
|
"first_publication_date": "2026-01-01T00:00:00",
|
|
}]])
|
|
result = database.import_run(self.conn, path)
|
|
self.assertEqual((result["accepted"], result["rejected"], result["field_issues"]), (1, 0, 3))
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT price_eur, currency, mileage_km, source_published_at FROM silver.observations"
|
|
).fetchone(), (None, None, None, None))
|
|
self.assertEqual(set(self.conn.execute(
|
|
"SELECT field, code FROM silver.v_validation_issues"
|
|
).fetchall()), {("price_eur", "invalid_value"), ("mileage_km", "out_of_range"),
|
|
("source_published_at", "timezone_missing")})
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT accepted_records, rejected_records, field_issues FROM silver.v_collection_coverage"
|
|
).fetchone(), (1, 0, 3))
|
|
|
|
def test_malformed_record_is_rejected_with_private_safe_provenance(self):
|
|
secret = "synthetic-private-message-do-not-log"
|
|
path, manifest = self.fixture(pages=[[
|
|
{"list_id": 123, "price": ["12000"]}, {"list_id": secret, "body": secret},
|
|
]])
|
|
raw_file = manifest["searches"][0]["pages"][0]["file"]
|
|
line = gzip.decompress((path.parent / raw_file).read_bytes()).decode().splitlines()[1]
|
|
result = database.import_run(self.conn, path)
|
|
self.assertEqual((result["accepted"], result["rejected"]), (1, 1))
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT run_id::text, raw_file, raw_line, code, input_sha256 FROM silver.rejected_records"
|
|
).fetchone(), (manifest["run_id"], raw_file, 2, "invalid_listing_id",
|
|
hashlib.sha256(line.encode()).hexdigest()))
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT raw_file, raw_line, issue_type, field, code FROM silver.v_validation_issues"
|
|
).fetchall(), [(raw_file, 2, "rejected_record", "$record", "invalid_listing_id")])
|
|
self.assertNotIn(secret, repr(result))
|
|
self.assertNotIn(secret, repr(self.state()))
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT expected_records, accepted_records, rejected_records FROM silver.raw_files"
|
|
).fetchone(), (2, 1, 1))
|
|
|
|
def test_rejected_record_replay_keeps_nonzero_cli_exit(self):
|
|
path, _ = self.fixture(pages=[[{"list_id": "invalid"}]])
|
|
with patch.dict(os.environ, {"DATABASE_URL": os.environ["TEST_DATABASE_URL"]}), patch(
|
|
"sys.argv", ["car-traker-db", "import", str(path)]
|
|
), patch("sys.stdout", new_callable=io.StringIO) as output:
|
|
self.assertEqual(database.main(), 2)
|
|
self.assertEqual(database.main(), 2)
|
|
results = [json.loads(line) for line in output.getvalue().splitlines()]
|
|
self.assertEqual([r["rejected"] for r in results], [1, 1])
|
|
self.assertEqual(results[1]["status"], "already_imported")
|
|
|
|
def test_json_number_precision_survives_database_storage(self):
|
|
path, manifest = self.fixture(pages=[[
|
|
{"list_id": 123, "price": ["EXCESS_PRECISION"]},
|
|
{"list_id": 456, "price": ["UNDERFLOW"]},
|
|
]])
|
|
snapshot = path.parent / manifest["searches"][0]["pages"][0]["file"]
|
|
content = gzip.decompress(snapshot.read_bytes()).decode()
|
|
content = content.replace('"EXCESS_PRECISION"', '1000.00000000000001').replace('"UNDERFLOW"', '1e-400')
|
|
snapshot.write_bytes(gzip.compress(content.encode()))
|
|
result = database.import_run(self.conn, path)
|
|
self.assertEqual((result["accepted"], result["rejected"], result["field_issues"]), (2, 0, 2))
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT price_eur, (raw_payload #>> '{price,0}')::numeric "
|
|
"FROM silver.observations ORDER BY source_listing_id"
|
|
).fetchall(), [(None, Decimal('1000.00000000000001')), (None, Decimal('1e-400'))])
|
|
|
|
def test_complete_run_cannot_omit_detail_coverage(self):
|
|
path, manifest = self.fixture()
|
|
manifest["details"] = {"enabled": True, "status": "complete", "listings": {}}
|
|
write_json(path, manifest)
|
|
before = self.state()
|
|
with self.assertRaisesRegex(database.InputError, "missing_detail_coverage"):
|
|
database.import_run(self.conn, path)
|
|
self.assertEqual(self.state(), before)
|
|
|
|
def test_partial_and_failed_runs_preserve_unattempted_coverage(self):
|
|
for status in ("partial", "failed"):
|
|
with self.subTest(status=status):
|
|
path, manifest = self.fixture(pages=None if status == "partial" else [])
|
|
manifest["status"] = manifest["searches"][0]["status"] = status
|
|
manifest["planned_searches"].append({"family": "later", "text": "never attempted"})
|
|
manifest["details"] = {
|
|
"enabled": True, "status": "not_attempted",
|
|
"listings": {"123": {"status": "not_attempted", "reason": "stopped"}},
|
|
}
|
|
write_json(path, manifest)
|
|
result = database.import_run(self.conn, path)
|
|
self.assertEqual(result["source_status"], status)
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT status, pages_observed, records_reported FROM silver.search_coverage "
|
|
"WHERE run_id = %s ORDER BY search_index", (manifest["run_id"],)
|
|
).fetchall(), [(status, int(status == "partial"), int(status == "partial")),
|
|
("not_attempted", 0, 0)])
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT status FROM silver.detail_coverage WHERE run_id = %s", (manifest["run_id"],)
|
|
).fetchall(), [("not_attempted",)])
|
|
self.assertEqual(self.conn.execute(
|
|
"SELECT source_status, searches_planned, searches_complete, details_complete "
|
|
"FROM silver.v_collection_coverage WHERE run_id = %s", (manifest["run_id"],)
|
|
).fetchone(), (status, 2, 0, 0))
|
|
|
|
def test_changed_raw_file_replay_is_refused_without_mutation(self):
|
|
path, manifest = self.fixture()
|
|
database.import_run(self.conn, path)
|
|
before = self.state()
|
|
raw = path.parent / manifest["searches"][0]["pages"][0]["file"]
|
|
records = [json.loads(line) for line in gzip.decompress(raw.read_bytes()).decode().splitlines()]
|
|
records[0]["payload"]["price"] = ["9999"]
|
|
write_records(raw, records)
|
|
with self.assertRaisesRegex(database.InputError, "^imported_run_changed$"):
|
|
database.import_run(self.conn, path)
|
|
self.assertEqual(self.state(), before)
|
|
|
|
def test_broken_later_file_aborts_entire_run(self):
|
|
baseline, _ = self.fixture()
|
|
database.import_run(self.conn, baseline)
|
|
before = self.state()
|
|
path, manifest = self.fixture(pages=[[{"list_id": 456}], [{"list_id": 789}]])
|
|
raw = path.parent / manifest["searches"][0]["pages"][1]["file"]
|
|
write_json(raw, {"not": "gzip"})
|
|
with self.assertRaises(gzip.BadGzipFile):
|
|
database.import_run(self.conn, path)
|
|
self.assertEqual(self.state(), before)
|
|
|
|
def test_bad_later_file_count_aborts_entire_run(self):
|
|
before = self.state()
|
|
path, manifest = self.fixture(pages=[[{"list_id": 456}], [{"list_id": 789}]])
|
|
write_records(path.parent / manifest["searches"][0]["pages"][1]["file"], [])
|
|
with self.assertRaisesRegex(database.InputError, "^raw_file_count_mismatch$"):
|
|
database.import_run(self.conn, path)
|
|
self.assertEqual(self.state(), before)
|
|
|
|
def test_sql_failure_mid_import_rolls_back_all_writes(self):
|
|
baseline, _ = self.fixture()
|
|
database.import_run(self.conn, baseline)
|
|
before = self.state()
|
|
path, _ = self.fixture(pages=[[{"list_id": 456}], [{"list_id": 789}]])
|
|
real_insert = database.insert
|
|
observations = 0
|
|
|
|
def failing_insert(conn, table, row):
|
|
nonlocal observations
|
|
if table == "observations":
|
|
observations += 1
|
|
if observations == 2:
|
|
self.assertEqual(conn.execute(
|
|
"SELECT count(*) FROM silver.observations WHERE source_listing_id = '456'"
|
|
).fetchone(), (1,))
|
|
# Execute an invalid INSERT, putting PostgreSQL itself into failed state.
|
|
row = {**row, "raw_line": 0}
|
|
return real_insert(conn, table, row)
|
|
|
|
with patch.object(database, "insert", side_effect=failing_insert):
|
|
with self.assertRaises(psycopg.errors.CheckViolation):
|
|
database.import_run(self.conn, path)
|
|
self.assertEqual(observations, 2)
|
|
self.assertEqual(self.state(), before)
|
|
self.assertEqual(database.import_run(self.conn, path)["accepted"], 2)
|
|
|
|
def test_concurrent_same_run_import_is_idempotent(self):
|
|
path, manifest = self.fixture()
|
|
barrier = Barrier(2, timeout=10)
|
|
|
|
def import_once():
|
|
with self.connect() as conn:
|
|
barrier.wait()
|
|
return database.import_run(conn, path)
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
futures = [executor.submit(import_once) for _ in range(2)]
|
|
results = [future.result(timeout=30) for future in futures]
|
|
self.assertCountEqual([result["status"] for result in results], ["imported", "already_imported"])
|
|
self.assertTrue(all(result["run_id"] == manifest["run_id"] for result in results))
|
|
imported = next(result for result in results if result["status"] == "imported")
|
|
self.assertEqual((imported["accepted"], imported["rejected"]), (1, 0))
|
|
self.assertEqual({table: len(rows) for table, rows in self.state().items() if table in self.tables}, {
|
|
"collection_runs": 1, "raw_files": 1, "search_coverage": 1,
|
|
"detail_coverage": 0, "listings": 1, "observations": 1, "rejected_records": 0,
|
|
})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|