randomize pauses between collection requests

README.md

@@ -25,11 +25,14 @@ uv run car-tracker --headed
 ```
 
 Options: `--config searches.toml`, `--output data/raw/leboncoin`, `--family`,
-`--max-pages 10` (per search), `--delay 5` (seconds between requests), `--headed`.
+`--max-pages 10` (per search), `--delay 5` (minimum seconds between requests), `--headed`.
 Detail visits are enabled by default, once per unique listing ID per run.
 `--max-details N` caps detail requests across the whole run (unlimited by default).
-`--search-only` disables detail visits. Delay applies to both search and detail
+`--search-only` disables detail visits. Each pause is sampled uniformly between
-requests. More detail visits mean substantially longer runs and greater exposure
+`--delay` and 1.5 times that value (5–7.5 seconds by default), for both search and
+detail requests. The first request has no added pause. The manifest records the
+minimum and maximum as `delay_seconds` and `delay_max_seconds`.
+More detail visits mean substantially longer runs and greater exposure
 to source access failures.
 Linux may need Playwright's system libraries installed by an administrator.
 Headed mode requires a graphical session. Omitting `--headed` runs headless, but

docs/DEPLOYMENT.md

@@ -135,7 +135,8 @@ docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run -
 
 Migrations are not automatic. Once DB access/schema are ready, run the image's
 default bounded headed pipeline: Saab 900, one page per selected search, at most
-one unique detail request for the run, with the default five-second request delay.
+one unique detail request for the run, with a randomized 5–7.5-second request delay
+by default.
 
 ```sh
 docker compose --env-file /srv/car-tracker/app.env -f ops/app/compose.yaml run --rm app

docs/SSH_RUN.md

@@ -25,8 +25,9 @@ be inspected; it never converts a failure to success.
 ### Investigating source access failures
 
 HTTP 403 is a confirmed forbidden response, not proof of a particular cause.
-The collector reuses one browser context and waits COLLECT_DELAY seconds between
+The collector reuses one browser context and waits a random interval between
-requests; it stops on 403 without retrying and imports already finalized records.
+COLLECT_DELAY and 1.5 times COLLECT_DELAY seconds between requests; it stops on 403
+without retrying and imports already finalized records.
 Older images retained only the HTTP status. Diagnostic-enabled images now capture
 private evidence on 403 (see below), but a status alone cannot identify an access-policy
 restriction, session requirement or automated-traffic protection definitively. Do not infer

src/car_tracker/collect.py

@@ -4,6 +4,7 @@ import argparse
 import gzip
 import json
 import math
+import random
 import sys
 import time
 import tomllib
@@ -24,6 +25,7 @@ EXCLUDED_ATTRIBUTES = {
     "vehicle_history_report_public_url",
 }
 EXCLUDED_ATTRIBUTE_PREFIXES = ("owner_", "seller_", "profile_", "rating_", "contact_")
+DELAY_JITTER_FACTOR = 1.5
 
 
 class SourceAccessError(RuntimeError):
@@ -121,7 +123,7 @@ def collect(fetch, searches, output, max_pages=10, delay=5, search_only=False, m
     manifest = {
         "schema_version": 2, "source": "leboncoin", "run_id": run_id,
         "started_at": now(), "status": "running", "max_pages": max_pages,
-        "delay_seconds": delay, "searches": [],
+        "delay_seconds": delay, "delay_max_seconds": delay * DELAY_JITTER_FACTOR, "searches": [],
         "privacy_policy": "v2-text-location-filtered-attributes",
         "planned_searches": searches,
         "details": {
@@ -140,7 +142,7 @@ def collect(fetch, searches, output, max_pages=10, delay=5, search_only=False, m
     def fetch_page(url, item=None):
         nonlocal requests, detail_attempts
         if requests:
-            time.sleep(delay)
+            time.sleep(random.uniform(delay, delay * DELAY_JITTER_FACTOR))
         if item is not None:
             detail_attempts += 1
             item.update(status="running", attempted_at=now())
@@ -284,7 +286,8 @@ def parse_collection_args(argv=None, description=None):
     parser.add_argument("--output", type=Path, default=Path("data/raw/leboncoin"))
     parser.add_argument("--family", help="Collect only this configured family")
     parser.add_argument("--max-pages", type=int, default=10)
-    parser.add_argument("--delay", type=float, default=5)
+    parser.add_argument("--delay", type=float, default=5,
+                        help="Minimum seconds between requests; each pause is randomized up to 1.5x (default: 5)")
     parser.add_argument("--headed", action="store_true")
     parser.add_argument("--search-only", action="store_true", help="Skip individual listing pages")
     parser.add_argument("--max-details", type=int, help="Maximum unique detail requests per run")
@@ -335,6 +338,7 @@ def collect_browser(args):
                             def capture(directory):
                                 return capture_403(directory, response, page, browser, {
                                     "headed": args.headed, "locale": "fr-FR", "delay_seconds": args.delay,
+                                    "delay_max_seconds": args.delay * DELAY_JITTER_FACTOR,
                                     "max_pages": args.max_pages, "max_details": args.max_details,
                                     "stealth": True,
                                 })

tests/test_collect.py

@@ -3,7 +3,7 @@ import json
 import tempfile
 import unittest
 from pathlib import Path
-from unittest.mock import patch
+from unittest.mock import call, patch
 
 from car_tracker.collect import SourceAccessError, collect, minimize, parse_detail, parse_search, search_url
 
@@ -273,11 +273,14 @@ class CollectorTests(unittest.TestCase):
         with tempfile.TemporaryDirectory() as directory:
             searches = [{"family": "synthetic", "text": "test"},
                         {"family": "synthetic", "text": "test-2"}]
-            with patch("car_tracker.collect.time.sleep") as sleep:
+            with patch("car_tracker.collect.time.sleep") as sleep, patch(
+                "car_tracker.collect.random.uniform", side_effect=[7, 10.5],
+            ) as uniform:
                 responses = iter([payload(), detail_payload(), payload()])
                 urls = []
 
                 def fetch(url):
+                    self.assertEqual(sleep.call_count, len(urls))
                     urls.append(url)
                     return next(responses)
 
@@ -287,8 +290,33 @@ class CollectorTests(unittest.TestCase):
         self.assertEqual(manifest["details"]["attempted"], 1)
         self.assertEqual(len(urls), 3)
         self.assertEqual(urls[1], "https://www.leboncoin.fr/ad/voitures/123")
-        self.assertEqual(sleep.call_count, 2)
+        self.assertEqual(uniform.call_args_list, [call(7, 10.5), call(7, 10.5)])
-        sleep.assert_called_with(7)
+        self.assertEqual(sleep.call_args_list, [call(7), call(10.5)])
+        self.assertEqual(manifest["delay_seconds"], 7)
+        self.assertEqual(manifest["delay_max_seconds"], 10.5)
+
+    def test_pagination_samples_a_fresh_delay_after_the_first_request(self):
+        with tempfile.TemporaryDirectory() as directory, patch(
+            "car_tracker.collect.time.sleep",
+        ) as sleep, patch(
+            "car_tracker.collect.random.uniform", side_effect=[5.5, 6.25],
+        ) as uniform:
+            responses = iter([
+                payload([identifier], total=3, pages=3) for identifier in (123, 456, 789)
+            ])
+            requests = 0
+
+            def fetch(url):
+                nonlocal requests
+                self.assertEqual(sleep.call_count, requests)
+                requests += 1
+                return next(responses)
+
+            _, status = collect(fetch, [{"family": "synthetic", "text": "test"}],
+                                Path(directory), search_only=True)
+        self.assertEqual(status, "complete")
+        self.assertEqual(uniform.call_args_list, [call(5, 7.5), call(5, 7.5)])
+        self.assertEqual(sleep.call_args_list, [call(5.5), call(6.25)])
 
     def test_search_only_explicitly_disables_detail_coverage(self):
         _, manifest = self.run_collection([payload()])