diff --git a/README.md b/README.md index 2955d9a..554a5f3 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,14 @@ uv run car-tracker --headed ``` Options: `--config searches.toml`, `--output data/raw/leboncoin`, `--family`, -`--max-pages 10` (per search), `--delay 5` (seconds between requests), `--headed`. +`--max-pages 10` (per search), `--delay 5` (minimum seconds between requests), `--headed`. Detail visits are enabled by default, once per unique listing ID per run. `--max-details N` caps detail requests across the whole run (unlimited by default). -`--search-only` disables detail visits. Delay applies to both search and detail -requests. More detail visits mean substantially longer runs and greater exposure +`--search-only` disables detail visits. Each pause is sampled uniformly between +`--delay` and 1.5 times that value (5–7.5 seconds by default), for both search and +detail requests. The first request has no added pause. The manifest records the +minimum and maximum as `delay_seconds` and `delay_max_seconds`. +More detail visits mean substantially longer runs and greater exposure to source access failures. Linux may need Playwright's system libraries installed by an administrator. Headed mode requires a graphical session. Omitting `--headed` runs headless, but diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index c8af85d..4174335 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -135,7 +135,8 @@ docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run - Migrations are not automatic. Once DB access/schema are ready, run the image's default bounded headed pipeline: Saab 900, one page per selected search, at most -one unique detail request for the run, with the default five-second request delay. +one unique detail request for the run, with a randomized 5–7.5-second request delay +by default. ```sh docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app diff --git a/docs/SSH_RUN.md b/docs/SSH_RUN.md index bdf835a..ee1fe87 100644 --- a/docs/SSH_RUN.md +++ b/docs/SSH_RUN.md @@ -25,8 +25,9 @@ be inspected; it never converts a failure to success. ### Investigating source access failures HTTP 403 is a confirmed forbidden response, not proof of a particular cause. -The collector reuses one browser context and waits COLLECT_DELAY seconds between -requests; it stops on 403 without retrying and imports already finalized records. +The collector reuses one browser context and waits a random interval between +COLLECT_DELAY and 1.5 times COLLECT_DELAY seconds between requests; it stops on 403 +without retrying and imports already finalized records. Older images retained only the HTTP status. Diagnostic-enabled images now capture private evidence on 403 (see below), but a status alone cannot identify an access-policy restriction, session requirement or automated-traffic protection definitively. Do not infer diff --git a/src/car_tracker/collect.py b/src/car_tracker/collect.py index 2c4ac41..a52eaa0 100644 --- a/src/car_tracker/collect.py +++ b/src/car_tracker/collect.py @@ -4,6 +4,7 @@ import argparse import gzip import json import math +import random import sys import time import tomllib @@ -24,6 +25,7 @@ EXCLUDED_ATTRIBUTES = { "vehicle_history_report_public_url", } EXCLUDED_ATTRIBUTE_PREFIXES = ("owner_", "seller_", "profile_", "rating_", "contact_") +DELAY_JITTER_FACTOR = 1.5 class SourceAccessError(RuntimeError): @@ -121,7 +123,7 @@ def collect(fetch, searches, output, max_pages=10, delay=5, search_only=False, m manifest = { "schema_version": 2, "source": "leboncoin", "run_id": run_id, "started_at": now(), "status": "running", "max_pages": max_pages, - "delay_seconds": delay, "searches": [], + "delay_seconds": delay, "delay_max_seconds": delay * DELAY_JITTER_FACTOR, "searches": [], "privacy_policy": "v2-text-location-filtered-attributes", "planned_searches": searches, "details": { @@ -140,7 +142,7 @@ def collect(fetch, searches, output, max_pages=10, delay=5, search_only=False, m def fetch_page(url, item=None): nonlocal requests, detail_attempts if requests: - time.sleep(delay) + time.sleep(random.uniform(delay, delay * DELAY_JITTER_FACTOR)) if item is not None: detail_attempts += 1 item.update(status="running", attempted_at=now()) @@ -284,7 +286,8 @@ def parse_collection_args(argv=None, description=None): parser.add_argument("--output", type=Path, default=Path("data/raw/leboncoin")) parser.add_argument("--family", help="Collect only this configured family") parser.add_argument("--max-pages", type=int, default=10) - parser.add_argument("--delay", type=float, default=5) + parser.add_argument("--delay", type=float, default=5, + help="Minimum seconds between requests; each pause is randomized up to 1.5x (default: 5)") parser.add_argument("--headed", action="store_true") parser.add_argument("--search-only", action="store_true", help="Skip individual listing pages") parser.add_argument("--max-details", type=int, help="Maximum unique detail requests per run") @@ -335,6 +338,7 @@ def collect_browser(args): def capture(directory): return capture_403(directory, response, page, browser, { "headed": args.headed, "locale": "fr-FR", "delay_seconds": args.delay, + "delay_max_seconds": args.delay * DELAY_JITTER_FACTOR, "max_pages": args.max_pages, "max_details": args.max_details, "stealth": True, }) diff --git a/tests/test_collect.py b/tests/test_collect.py index 2102e25..491859c 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -3,7 +3,7 @@ import json import tempfile import unittest from pathlib import Path -from unittest.mock import patch +from unittest.mock import call, patch from car_tracker.collect import SourceAccessError, collect, minimize, parse_detail, parse_search, search_url @@ -273,11 +273,14 @@ class CollectorTests(unittest.TestCase): with tempfile.TemporaryDirectory() as directory: searches = [{"family": "synthetic", "text": "test"}, {"family": "synthetic", "text": "test-2"}] - with patch("car_tracker.collect.time.sleep") as sleep: + with patch("car_tracker.collect.time.sleep") as sleep, patch( + "car_tracker.collect.random.uniform", side_effect=[7, 10.5], + ) as uniform: responses = iter([payload(), detail_payload(), payload()]) urls = [] def fetch(url): + self.assertEqual(sleep.call_count, len(urls)) urls.append(url) return next(responses) @@ -287,8 +290,33 @@ class CollectorTests(unittest.TestCase): self.assertEqual(manifest["details"]["attempted"], 1) self.assertEqual(len(urls), 3) self.assertEqual(urls[1], "https://www.leboncoin.fr/ad/voitures/123") - self.assertEqual(sleep.call_count, 2) - sleep.assert_called_with(7) + self.assertEqual(uniform.call_args_list, [call(7, 10.5), call(7, 10.5)]) + self.assertEqual(sleep.call_args_list, [call(7), call(10.5)]) + self.assertEqual(manifest["delay_seconds"], 7) + self.assertEqual(manifest["delay_max_seconds"], 10.5) + + def test_pagination_samples_a_fresh_delay_after_the_first_request(self): + with tempfile.TemporaryDirectory() as directory, patch( + "car_tracker.collect.time.sleep", + ) as sleep, patch( + "car_tracker.collect.random.uniform", side_effect=[5.5, 6.25], + ) as uniform: + responses = iter([ + payload([identifier], total=3, pages=3) for identifier in (123, 456, 789) + ]) + requests = 0 + + def fetch(url): + nonlocal requests + self.assertEqual(sleep.call_count, requests) + requests += 1 + return next(responses) + + _, status = collect(fetch, [{"family": "synthetic", "text": "test"}], + Path(directory), search_only=True) + self.assertEqual(status, "complete") + self.assertEqual(uniform.call_args_list, [call(5, 7.5), call(5, 7.5)]) + self.assertEqual(sleep.call_args_list, [call(5.5), call(6.25)]) def test_search_only_explicitly_disables_detail_coverage(self): _, manifest = self.run_collection([payload()])