diff --git a/README.md b/README.md index 455605a..2955d9a 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,10 @@ to source access failures. Linux may need Playwright's system libraries installed by an administrator. Headed mode requires a graphical session. Omitting `--headed` runs headless, but headless access failed in the initial live test on this machine. -Do not run overlapping jobs. No credentials, proxies, CAPTCHA solvers, stealth -plugins, or automatic access-block retries are used. +Playwright Stealth is enabled for every collection, in headed and headless mode, +with French language overrides matching the browser locale. It does not guarantee +access or solve challenges; collection still stops on access blocks without retries. +Do not run overlapping jobs. No credentials, proxies, or CAPTCHA solvers are used. `searches.toml` defines broad nationwide car-category searches. Add queries/families there without changing code. Saab 900 and Volvo 850 queries do not restrict diff --git a/pyproject.toml b/pyproject.toml index e1e0691..41e943a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "0.1.0" description = "French youngtimer asking-price observatory" readme = "README.md" requires-python = ">=3.13" -dependencies = ["playwright>=1.58,<2", "psycopg[binary]>=3.2,<4", "simplejson>=3.20,<4"] +dependencies = ["playwright>=1.58,<2", "playwright-stealth>=2,<3", "psycopg[binary]>=3.2,<4", "simplejson>=3.20,<4"] [dependency-groups] dev = ["ruff>=0.14,<0.15"] diff --git a/src/car_tracker/collect.py b/src/car_tracker/collect.py index 73ec4c5..2c4ac41 100644 --- a/src/car_tracker/collect.py +++ b/src/car_tracker/collect.py @@ -315,10 +315,11 @@ def parse_collection_args(argv=None, description=None): def collect_browser(args): from playwright.sync_api import sync_playwright + from playwright_stealth import Stealth result = None try: - with sync_playwright() as playwright: + with Stealth(navigator_languages_override=("fr-FR", "fr")).use_sync(sync_playwright()) as playwright: browser = playwright.chromium.launch(headless=not args.headed) try: context = browser.new_context(locale="fr-FR") @@ -335,6 +336,7 @@ def collect_browser(args): return capture_403(directory, response, page, browser, { "headed": args.headed, "locale": "fr-FR", "delay_seconds": args.delay, "max_pages": args.max_pages, "max_details": args.max_details, + "stealth": True, }) raise SourceAccessError(response.status if response else None, capture=capture) return page.locator("script#__NEXT_DATA__").text_content(timeout=15_000) diff --git a/tests/test_collect_browser.py b/tests/test_collect_browser.py index 8d3f0c1..5f37949 100644 --- a/tests/test_collect_browser.py +++ b/tests/test_collect_browser.py @@ -17,10 +17,34 @@ class BrowserCleanupTests(unittest.TestCase): self.args = Namespace(headed=False, searches=[], output=self.path.parent, max_pages=1, delay=1, search_only=True, max_details=None) self.playwright = self.enterContext(patch("playwright.sync_api.sync_playwright")) + self.stealth = self.enterContext(patch("playwright_stealth.Stealth")) + self.stealth.return_value.use_sync.side_effect = lambda manager: manager self.browser = self.playwright.return_value.__enter__.return_value.chromium.launch.return_value self.collect = self.enterContext(patch("car_tracker.collect.collect", return_value=(self.path, "complete"))) self.stderr = self.enterContext(patch("sys.stderr", new_callable=io.StringIO)) + def test_stealth_wraps_playwright_with_french_languages_in_both_modes(self): + for headed in (False, True): + with self.subTest(headed=headed): + self.args.headed = headed + self.stealth.reset_mock() + self.browser.reset_mock() + self.assertEqual(collect_browser(self.args), (self.path, "complete")) + self.stealth.assert_called_once_with(navigator_languages_override=("fr-FR", "fr")) + self.stealth.return_value.use_sync.assert_called_once_with(self.playwright.return_value) + self.playwright.return_value.__enter__.return_value.chromium.launch.assert_called_with( + headless=not headed, + ) + self.browser.new_context.assert_called_once_with(locale="fr-FR") + self.browser.new_context.return_value.new_page.assert_called_once_with() + self.browser.close.assert_called_once_with() + + def test_stealth_setup_failure_is_not_suppressed(self): + self.stealth.return_value.use_sync.side_effect = RuntimeError("stealth failed") + with self.assertRaisesRegex(RuntimeError, "stealth failed"): + collect_browser(self.args) + self.collect.assert_not_called() + def test_browser_close_failure_retains_finalized_result(self): self.browser.close.side_effect = RuntimeError("SYNTHETIC_SECRET") self.assertEqual(collect_browser(self.args), (self.path, "complete")) diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index 19b1f05..3f9cdc6 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -76,11 +76,16 @@ class DiagnosticTests(unittest.TestCase): search_only=False, output=self.root.parent.parent, searches=[{"family": "synthetic", "text": "synthetic"}]) module = SimpleNamespace(sync_playwright=lambda: nullcontext(playwright)) - with patch.dict("sys.modules", {"playwright.sync_api": module}): + with patch("playwright_stealth.Stealth") as stealth, patch.dict( + "sys.modules", {"playwright.sync_api": module}, + ): + stealth.return_value.use_sync.side_effect = lambda manager: manager path, status = collect_browser(args) manifest = json.loads(path.read_text()) self.assertEqual(status, "failed") self.assertEqual(manifest["diagnostics"]["capture_status"], "complete") self.assertTrue((path.parent / "diagnostics/http-403/response-body.html").exists()) + metadata = json.loads((path.parent / "diagnostics/http-403/metadata.json").read_text()) + self.assertTrue(metadata["settings"]["stealth"]) self.page.goto.assert_called_once() self.browser.close.assert_called_once() diff --git a/uv.lock b/uv.lock index 01c2791..45c3551 100644 --- a/uv.lock +++ b/uv.lock @@ -8,6 +8,7 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "playwright" }, + { name = "playwright-stealth" }, { name = "psycopg", extra = ["binary"] }, { name = "simplejson" }, ] @@ -20,6 +21,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "playwright", specifier = ">=1.58,<2" }, + { name = "playwright-stealth", specifier = ">=2,<3" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, { name = "simplejson", specifier = ">=3.20,<4" }, ] @@ -103,6 +105,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/ec/06b55d619a7082a766aa04f2c6bb31435c87f02930087d8a0517119408fa/playwright-1.62.0-py3-none-win_arm64.whl", hash = "sha256:ea8d3055aa9d5a9f1832ac82517bd8b42c78fac7ebcbebb0107116735c8cb6a1", size = 34208868, upload-time = "2026-07-31T17:01:11.818Z" }, ] +[[package]] +name = "playwright-stealth" +version = "2.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "playwright" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a4/db/6ade5d539c7d151b9defc78fafa8b65aa52352617d0e7699b47008bd801f/playwright_stealth-2.0.3.tar.gz", hash = "sha256:1d8e488fbdd8f190f1269ea8cf5d57d14df3a9f1af1001c41ee3588b2aac3133", size = 25751, upload-time = "2026-04-04T02:50:33.88Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/10/607c409712c02a26c4cb794820514cb7fdaaeac15fb05bed917fb8a354b3/playwright_stealth-2.0.3-py3-none-any.whl", hash = "sha256:1887ade423ab7ff8ae16d363a30a38de0b5817e1e4a29d47b74bf3a0e3dbfcb4", size = 34385, upload-time = "2026-04-04T02:50:35.246Z" }, +] + [[package]] name = "psycopg" version = "3.3.5"