a353e1c683
All checks were successful
CI / Lint, PostgreSQL tests, and package (push) Successful in 12s
162 lines
7.2 KiB
Python
162 lines
7.2 KiB
Python
import gzip
|
|
import json
|
|
from decimal import Decimal
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from car_tracker.collect import collect, write_json
|
|
from car_tracker.database import InputError, load_json, load_run, prepare_record
|
|
|
|
|
|
class RawInputTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name)
|
|
|
|
def fixture(self):
|
|
source = json.dumps({"props": {"pageProps": {"searchData": {
|
|
"total": 1, "max_pages": 1, "ads": [{
|
|
"list_id": 123, "price": [1000], "body": "Synthetic\u2028text\u2029only",
|
|
}],
|
|
}}}})
|
|
with patch("car_tracker.collect.time.sleep"):
|
|
path, _ = collect(lambda url: source, [{"family": "synthetic", "text": "test"}],
|
|
self.root, search_only=True)
|
|
return path, json.loads(path.read_text())
|
|
|
|
def test_collector_output_and_unicode_text_are_replayable(self):
|
|
path, _ = self.fixture()
|
|
manifest, digest, coverage, inputs = load_run(path)
|
|
self.assertEqual(len(digest), 64)
|
|
self.assertEqual(coverage[0]["records_reported"], 1)
|
|
result = prepare_record(inputs[0]["lines"][0], inputs[0], manifest)
|
|
self.assertEqual(result["description"], "Synthetic\u2028text\u2029only")
|
|
self.assertEqual(result["source_listing_id"], "123")
|
|
|
|
def test_rejects_nonfinal_run(self):
|
|
path, manifest = self.fixture()
|
|
manifest["status"] = "running"
|
|
write_json(path, manifest)
|
|
with self.assertRaisesRegex(InputError, "run_not_finalized"):
|
|
load_run(path)
|
|
|
|
def test_v2_requires_detail_coverage(self):
|
|
path, manifest = self.fixture()
|
|
del manifest["details"]
|
|
write_json(path, manifest)
|
|
with self.assertRaises(InputError):
|
|
load_run(path)
|
|
|
|
def test_complete_search_must_cover_reported_pages(self):
|
|
path, manifest = self.fixture()
|
|
manifest["searches"][0]["pages"][0]["reported_max_pages"] = 3
|
|
write_json(path, manifest)
|
|
with self.assertRaisesRegex(InputError, "inconsistent_complete_search"):
|
|
load_run(path)
|
|
|
|
def test_discovered_ids_cannot_be_omitted_from_detail_coverage(self):
|
|
path, manifest = self.fixture()
|
|
for listings in ({}, {"456": {"status": "not_attempted"}}):
|
|
with self.subTest(listings=listings):
|
|
manifest["status"] = "partial"
|
|
manifest["details"] = {"enabled": True, "status": "partial", "listings": listings}
|
|
write_json(path, manifest)
|
|
with self.assertRaisesRegex(InputError, "missing_detail_coverage"):
|
|
load_run(path)
|
|
|
|
def test_unknown_schema_is_rejected(self):
|
|
path, manifest = self.fixture()
|
|
for version in (True, 3, "2"):
|
|
with self.subTest(version=version):
|
|
manifest["schema_version"] = version
|
|
write_json(path, manifest)
|
|
with self.assertRaisesRegex(InputError, "unsupported_raw_schema"):
|
|
load_run(path)
|
|
|
|
def test_rejects_traversal_and_unfinalized_filenames(self):
|
|
path, manifest = self.fixture()
|
|
for name in ("../snapshot.jsonl.gz", "/tmp/snapshot.jsonl.gz", "snapshot.jsonl.gz.tmp"):
|
|
with self.subTest(name=name):
|
|
manifest["searches"][0]["pages"][0]["file"] = name
|
|
write_json(path, manifest)
|
|
with self.assertRaisesRegex(InputError, "invalid_raw_file_reference"):
|
|
load_run(path)
|
|
|
|
def test_rejects_symlink_escape(self):
|
|
path, manifest = self.fixture()
|
|
name = manifest["searches"][0]["pages"][0]["file"]
|
|
outside = self.root / "outside.jsonl.gz"
|
|
(path.parent / name).replace(outside)
|
|
(path.parent / name).symlink_to(outside)
|
|
with self.assertRaisesRegex(InputError, "raw_file_outside_run"):
|
|
load_run(path)
|
|
|
|
def test_unreferenced_files_are_ignored(self):
|
|
path, _ = self.fixture()
|
|
(path.parent / "unreferenced.jsonl.gz").write_bytes(b"not gzip")
|
|
_, _, _, inputs = load_run(path)
|
|
self.assertEqual(len(inputs), 1)
|
|
|
|
def test_rejects_missing_and_truncated_snapshots(self):
|
|
path, manifest = self.fixture()
|
|
snapshot = path.parent / manifest["searches"][0]["pages"][0]["file"]
|
|
snapshot.unlink()
|
|
with self.assertRaises(FileNotFoundError):
|
|
load_run(path)
|
|
snapshot.write_bytes(gzip.compress(b""))
|
|
with self.assertRaisesRegex(InputError, "raw_file_count_mismatch"):
|
|
load_run(path)
|
|
|
|
def test_provenance_mismatch_is_rejected(self):
|
|
path, _ = self.fixture()
|
|
manifest, _, _, inputs = load_run(path)
|
|
original = load_json(inputs[0]["lines"][0])
|
|
for field, value in (("run_id", "wrong"), ("record_type", "detail"),
|
|
("page", 2), ("family", "wrong"),
|
|
("observed_at", "2020-01-01T00:00:00Z")):
|
|
with self.subTest(field=field), self.assertRaises(InputError):
|
|
prepare_record(json.dumps({**original, field: value}), inputs[0], manifest)
|
|
|
|
def test_rejects_json_values_postgresql_cannot_store(self):
|
|
for text in ('{"x": NaN}', '{"x": 1e999999}', '{"x": "\\u0000"}',
|
|
'{"x": "\\ud800"}', '{"x": Infinity}', "not json"):
|
|
with self.subTest(text=text), self.assertRaisesRegex(InputError, "invalid_json"):
|
|
load_json(text)
|
|
|
|
def test_numeric_json_tokens_keep_decimal_precision(self):
|
|
values = load_json('[1000.00000000000001, 1e-400]')
|
|
self.assertEqual(values, [Decimal('1000.00000000000001'), Decimal('1e-400')])
|
|
path, _ = self.fixture()
|
|
manifest, _, _, inputs = load_run(path)
|
|
for token in ('1000.00000000000001', '1e-400'):
|
|
with self.subTest(token=token):
|
|
line = inputs[0]["lines"][0].replace('"price": [1000]', f'"price": [{token}]')
|
|
result = prepare_record(line, inputs[0], manifest)
|
|
self.assertIsNone(result["price_eur"])
|
|
self.assertIn({"field": "price_eur", "code": "invalid_precision"}, result["issues"].obj)
|
|
self.assertEqual(result["raw_payload"].obj["price"][0], Decimal(token))
|
|
|
|
def test_missing_listing_identity_is_rejected_not_invented(self):
|
|
path, _ = self.fixture()
|
|
manifest, _, _, inputs = load_run(path)
|
|
record = load_json(inputs[0]["lines"][0])
|
|
record["payload"] = {"subject": "Synthetic car"}
|
|
with self.assertRaisesRegex(InputError, "invalid_record"):
|
|
prepare_record(json.dumps(record), inputs[0], manifest)
|
|
|
|
def test_invalid_vehicle_fields_are_issues_not_record_rejections(self):
|
|
path, _ = self.fixture()
|
|
manifest, _, _, inputs = load_run(path)
|
|
record = load_json(inputs[0]["lines"][0])
|
|
record["payload"]["price"] = ["unknown"]
|
|
result = prepare_record(json.dumps(record), inputs[0], manifest)
|
|
self.assertIsNone(result["price_eur"])
|
|
self.assertTrue(result["issues"].obj)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|