import gzip import json from decimal import Decimal from pathlib import Path from tempfile import TemporaryDirectory import unittest from unittest.mock import patch from car_traker.collect import collect, write_json from car_traker.database import InputError, load_json, load_run, prepare_record class RawInputTests(unittest.TestCase): def setUp(self): temporary = TemporaryDirectory() self.addCleanup(temporary.cleanup) self.root = Path(temporary.name) def fixture(self): source = json.dumps({"props": {"pageProps": {"searchData": { "total": 1, "max_pages": 1, "ads": [{ "list_id": 123, "price": [1000], "body": "Synthetic\u2028text\u2029only", }], }}}}) with patch("car_traker.collect.time.sleep"): path, _ = collect(lambda url: source, [{"family": "synthetic", "text": "test"}], self.root, search_only=True) return path, json.loads(path.read_text()) def test_collector_output_and_unicode_text_are_replayable(self): path, _ = self.fixture() manifest, digest, coverage, inputs = load_run(path) self.assertEqual(len(digest), 64) self.assertEqual(coverage[0]["records_reported"], 1) result = prepare_record(inputs[0]["lines"][0], inputs[0], manifest) self.assertEqual(result["description"], "Synthetic\u2028text\u2029only") self.assertEqual(result["source_listing_id"], "123") def test_rejects_nonfinal_run(self): path, manifest = self.fixture() manifest["status"] = "running" write_json(path, manifest) with self.assertRaisesRegex(InputError, "run_not_finalized"): load_run(path) def test_v2_requires_detail_coverage(self): path, manifest = self.fixture() del manifest["details"] write_json(path, manifest) with self.assertRaises(InputError): load_run(path) def test_complete_search_must_cover_reported_pages(self): path, manifest = self.fixture() manifest["searches"][0]["pages"][0]["reported_max_pages"] = 3 write_json(path, manifest) with self.assertRaisesRegex(InputError, "inconsistent_complete_search"): load_run(path) def test_discovered_ids_cannot_be_omitted_from_detail_coverage(self): path, manifest = self.fixture() for listings in ({}, {"456": {"status": "not_attempted"}}): with self.subTest(listings=listings): manifest["status"] = "partial" manifest["details"] = {"enabled": True, "status": "partial", "listings": listings} write_json(path, manifest) with self.assertRaisesRegex(InputError, "missing_detail_coverage"): load_run(path) def test_unknown_schema_is_rejected(self): path, manifest = self.fixture() for version in (True, 3, "2"): with self.subTest(version=version): manifest["schema_version"] = version write_json(path, manifest) with self.assertRaisesRegex(InputError, "unsupported_raw_schema"): load_run(path) def test_rejects_traversal_and_unfinalized_filenames(self): path, manifest = self.fixture() for name in ("../snapshot.jsonl.gz", "/tmp/snapshot.jsonl.gz", "snapshot.jsonl.gz.tmp"): with self.subTest(name=name): manifest["searches"][0]["pages"][0]["file"] = name write_json(path, manifest) with self.assertRaisesRegex(InputError, "invalid_raw_file_reference"): load_run(path) def test_rejects_symlink_escape(self): path, manifest = self.fixture() name = manifest["searches"][0]["pages"][0]["file"] outside = self.root / "outside.jsonl.gz" (path.parent / name).replace(outside) (path.parent / name).symlink_to(outside) with self.assertRaisesRegex(InputError, "raw_file_outside_run"): load_run(path) def test_unreferenced_files_are_ignored(self): path, _ = self.fixture() (path.parent / "unreferenced.jsonl.gz").write_bytes(b"not gzip") _, _, _, inputs = load_run(path) self.assertEqual(len(inputs), 1) def test_rejects_missing_and_truncated_snapshots(self): path, manifest = self.fixture() snapshot = path.parent / manifest["searches"][0]["pages"][0]["file"] snapshot.unlink() with self.assertRaises(FileNotFoundError): load_run(path) snapshot.write_bytes(gzip.compress(b"")) with self.assertRaisesRegex(InputError, "raw_file_count_mismatch"): load_run(path) def test_provenance_mismatch_is_rejected(self): path, _ = self.fixture() manifest, _, _, inputs = load_run(path) original = load_json(inputs[0]["lines"][0]) for field, value in (("run_id", "wrong"), ("record_type", "detail"), ("page", 2), ("family", "wrong"), ("observed_at", "2020-01-01T00:00:00Z")): with self.subTest(field=field), self.assertRaises(InputError): prepare_record(json.dumps({**original, field: value}), inputs[0], manifest) def test_rejects_json_values_postgresql_cannot_store(self): for text in ('{"x": NaN}', '{"x": 1e999999}', '{"x": "\\u0000"}', '{"x": "\\ud800"}', '{"x": Infinity}', "not json"): with self.subTest(text=text), self.assertRaisesRegex(InputError, "invalid_json"): load_json(text) def test_numeric_json_tokens_keep_decimal_precision(self): values = load_json('[1000.00000000000001, 1e-400]') self.assertEqual(values, [Decimal('1000.00000000000001'), Decimal('1e-400')]) path, _ = self.fixture() manifest, _, _, inputs = load_run(path) for token in ('1000.00000000000001', '1e-400'): with self.subTest(token=token): line = inputs[0]["lines"][0].replace('"price": [1000]', f'"price": [{token}]') result = prepare_record(line, inputs[0], manifest) self.assertIsNone(result["price_eur"]) self.assertIn({"field": "price_eur", "code": "invalid_precision"}, result["issues"].obj) self.assertEqual(result["raw_payload"].obj["price"][0], Decimal(token)) def test_missing_listing_identity_is_rejected_not_invented(self): path, _ = self.fixture() manifest, _, _, inputs = load_run(path) record = load_json(inputs[0]["lines"][0]) record["payload"] = {"subject": "Synthetic car"} with self.assertRaisesRegex(InputError, "invalid_record"): prepare_record(json.dumps(record), inputs[0], manifest) def test_invalid_vehicle_fields_are_issues_not_record_rejections(self): path, _ = self.fixture() manifest, _, _, inputs = load_run(path) record = load_json(inputs[0]["lines"][0]) record["payload"]["price"] = ["unknown"] result = prepare_record(json.dumps(record), inputs[0], manifest) self.assertIsNone(result["price_eur"]) self.assertTrue(result["issues"].obj) if __name__ == "__main__": unittest.main()