enable Playwright Stealth with French locale for collection
All checks were successful
CI / Lint, PostgreSQL tests, and package (push) Successful in 15s
CI / Build and publish tested image (push) Successful in 1m19s

This commit is contained in:
Antoine Vandeplanque 2026-10-13 14:16:13 +02:00
parent c9ee4bde30
commit 5d68f0f776
6 changed files with 52 additions and 5 deletions

View File

@ -34,8 +34,10 @@ to source access failures.
Linux may need Playwright's system libraries installed by an administrator. Linux may need Playwright's system libraries installed by an administrator.
Headed mode requires a graphical session. Omitting `--headed` runs headless, but Headed mode requires a graphical session. Omitting `--headed` runs headless, but
headless access failed in the initial live test on this machine. headless access failed in the initial live test on this machine.
Do not run overlapping jobs. No credentials, proxies, CAPTCHA solvers, stealth Playwright Stealth is enabled for every collection, in headed and headless mode,
plugins, or automatic access-block retries are used. with French language overrides matching the browser locale. It does not guarantee
access or solve challenges; collection still stops on access blocks without retries.
Do not run overlapping jobs. No credentials, proxies, or CAPTCHA solvers are used.
`searches.toml` defines broad nationwide car-category searches. Add queries/families `searches.toml` defines broad nationwide car-category searches. Add queries/families
there without changing code. Saab 900 and Volvo 850 queries do not restrict there without changing code. Saab 900 and Volvo 850 queries do not restrict

View File

@ -4,7 +4,7 @@ version = "0.1.0"
description = "French youngtimer asking-price observatory" description = "French youngtimer asking-price observatory"
readme = "README.md" readme = "README.md"
requires-python = ">=3.13" requires-python = ">=3.13"
dependencies = ["playwright>=1.58,<2", "psycopg[binary]>=3.2,<4", "simplejson>=3.20,<4"] dependencies = ["playwright>=1.58,<2", "playwright-stealth>=2,<3", "psycopg[binary]>=3.2,<4", "simplejson>=3.20,<4"]
[dependency-groups] [dependency-groups]
dev = ["ruff>=0.14,<0.15"] dev = ["ruff>=0.14,<0.15"]

View File

@ -315,10 +315,11 @@ def parse_collection_args(argv=None, description=None):
def collect_browser(args): def collect_browser(args):
from playwright.sync_api import sync_playwright from playwright.sync_api import sync_playwright
from playwright_stealth import Stealth
result = None result = None
try: try:
with sync_playwright() as playwright: with Stealth(navigator_languages_override=("fr-FR", "fr")).use_sync(sync_playwright()) as playwright:
browser = playwright.chromium.launch(headless=not args.headed) browser = playwright.chromium.launch(headless=not args.headed)
try: try:
context = browser.new_context(locale="fr-FR") context = browser.new_context(locale="fr-FR")
@ -335,6 +336,7 @@ def collect_browser(args):
return capture_403(directory, response, page, browser, { return capture_403(directory, response, page, browser, {
"headed": args.headed, "locale": "fr-FR", "delay_seconds": args.delay, "headed": args.headed, "locale": "fr-FR", "delay_seconds": args.delay,
"max_pages": args.max_pages, "max_details": args.max_details, "max_pages": args.max_pages, "max_details": args.max_details,
"stealth": True,
}) })
raise SourceAccessError(response.status if response else None, capture=capture) raise SourceAccessError(response.status if response else None, capture=capture)
return page.locator("script#__NEXT_DATA__").text_content(timeout=15_000) return page.locator("script#__NEXT_DATA__").text_content(timeout=15_000)

View File

@ -17,10 +17,34 @@ class BrowserCleanupTests(unittest.TestCase):
self.args = Namespace(headed=False, searches=[], output=self.path.parent, self.args = Namespace(headed=False, searches=[], output=self.path.parent,
max_pages=1, delay=1, search_only=True, max_details=None) max_pages=1, delay=1, search_only=True, max_details=None)
self.playwright = self.enterContext(patch("playwright.sync_api.sync_playwright")) self.playwright = self.enterContext(patch("playwright.sync_api.sync_playwright"))
self.stealth = self.enterContext(patch("playwright_stealth.Stealth"))
self.stealth.return_value.use_sync.side_effect = lambda manager: manager
self.browser = self.playwright.return_value.__enter__.return_value.chromium.launch.return_value self.browser = self.playwright.return_value.__enter__.return_value.chromium.launch.return_value
self.collect = self.enterContext(patch("car_tracker.collect.collect", return_value=(self.path, "complete"))) self.collect = self.enterContext(patch("car_tracker.collect.collect", return_value=(self.path, "complete")))
self.stderr = self.enterContext(patch("sys.stderr", new_callable=io.StringIO)) self.stderr = self.enterContext(patch("sys.stderr", new_callable=io.StringIO))
def test_stealth_wraps_playwright_with_french_languages_in_both_modes(self):
for headed in (False, True):
with self.subTest(headed=headed):
self.args.headed = headed
self.stealth.reset_mock()
self.browser.reset_mock()
self.assertEqual(collect_browser(self.args), (self.path, "complete"))
self.stealth.assert_called_once_with(navigator_languages_override=("fr-FR", "fr"))
self.stealth.return_value.use_sync.assert_called_once_with(self.playwright.return_value)
self.playwright.return_value.__enter__.return_value.chromium.launch.assert_called_with(
headless=not headed,
)
self.browser.new_context.assert_called_once_with(locale="fr-FR")
self.browser.new_context.return_value.new_page.assert_called_once_with()
self.browser.close.assert_called_once_with()
def test_stealth_setup_failure_is_not_suppressed(self):
self.stealth.return_value.use_sync.side_effect = RuntimeError("stealth failed")
with self.assertRaisesRegex(RuntimeError, "stealth failed"):
collect_browser(self.args)
self.collect.assert_not_called()
def test_browser_close_failure_retains_finalized_result(self): def test_browser_close_failure_retains_finalized_result(self):
self.browser.close.side_effect = RuntimeError("SYNTHETIC_SECRET") self.browser.close.side_effect = RuntimeError("SYNTHETIC_SECRET")
self.assertEqual(collect_browser(self.args), (self.path, "complete")) self.assertEqual(collect_browser(self.args), (self.path, "complete"))

View File

@ -76,11 +76,16 @@ class DiagnosticTests(unittest.TestCase):
search_only=False, output=self.root.parent.parent, search_only=False, output=self.root.parent.parent,
searches=[{"family": "synthetic", "text": "synthetic"}]) searches=[{"family": "synthetic", "text": "synthetic"}])
module = SimpleNamespace(sync_playwright=lambda: nullcontext(playwright)) module = SimpleNamespace(sync_playwright=lambda: nullcontext(playwright))
with patch.dict("sys.modules", {"playwright.sync_api": module}): with patch("playwright_stealth.Stealth") as stealth, patch.dict(
"sys.modules", {"playwright.sync_api": module},
):
stealth.return_value.use_sync.side_effect = lambda manager: manager
path, status = collect_browser(args) path, status = collect_browser(args)
manifest = json.loads(path.read_text()) manifest = json.loads(path.read_text())
self.assertEqual(status, "failed") self.assertEqual(status, "failed")
self.assertEqual(manifest["diagnostics"]["capture_status"], "complete") self.assertEqual(manifest["diagnostics"]["capture_status"], "complete")
self.assertTrue((path.parent / "diagnostics/http-403/response-body.html").exists()) self.assertTrue((path.parent / "diagnostics/http-403/response-body.html").exists())
metadata = json.loads((path.parent / "diagnostics/http-403/metadata.json").read_text())
self.assertTrue(metadata["settings"]["stealth"])
self.page.goto.assert_called_once() self.page.goto.assert_called_once()
self.browser.close.assert_called_once() self.browser.close.assert_called_once()

14
uv.lock generated
View File

@ -8,6 +8,7 @@ version = "0.1.0"
source = { editable = "." } source = { editable = "." }
dependencies = [ dependencies = [
{ name = "playwright" }, { name = "playwright" },
{ name = "playwright-stealth" },
{ name = "psycopg", extra = ["binary"] }, { name = "psycopg", extra = ["binary"] },
{ name = "simplejson" }, { name = "simplejson" },
] ]
@ -20,6 +21,7 @@ dev = [
[package.metadata] [package.metadata]
requires-dist = [ requires-dist = [
{ name = "playwright", specifier = ">=1.58,<2" }, { name = "playwright", specifier = ">=1.58,<2" },
{ name = "playwright-stealth", specifier = ">=2,<3" },
{ name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" },
{ name = "simplejson", specifier = ">=3.20,<4" }, { name = "simplejson", specifier = ">=3.20,<4" },
] ]
@ -103,6 +105,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/6c/ec/06b55d619a7082a766aa04f2c6bb31435c87f02930087d8a0517119408fa/playwright-1.62.0-py3-none-win_arm64.whl", hash = "sha256:ea8d3055aa9d5a9f1832ac82517bd8b42c78fac7ebcbebb0107116735c8cb6a1", size = 34208868, upload-time = "2026-07-31T17:01:11.818Z" }, { url = "https://files.pythonhosted.org/packages/6c/ec/06b55d619a7082a766aa04f2c6bb31435c87f02930087d8a0517119408fa/playwright-1.62.0-py3-none-win_arm64.whl", hash = "sha256:ea8d3055aa9d5a9f1832ac82517bd8b42c78fac7ebcbebb0107116735c8cb6a1", size = 34208868, upload-time = "2026-07-31T17:01:11.818Z" },
] ]
[[package]]
name = "playwright-stealth"
version = "2.0.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "playwright" },
]
sdist = { url = "https://files.pythonhosted.org/packages/a4/db/6ade5d539c7d151b9defc78fafa8b65aa52352617d0e7699b47008bd801f/playwright_stealth-2.0.3.tar.gz", hash = "sha256:1d8e488fbdd8f190f1269ea8cf5d57d14df3a9f1af1001c41ee3588b2aac3133", size = 25751, upload-time = "2026-04-04T02:50:33.88Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/b5/10/607c409712c02a26c4cb794820514cb7fdaaeac15fb05bed917fb8a354b3/playwright_stealth-2.0.3-py3-none-any.whl", hash = "sha256:1887ade423ab7ff8ae16d363a30a38de0b5817e1e4a29d47b74bf3a0e3dbfcb4", size = 34385, upload-time = "2026-04-04T02:50:35.246Z" },
]
[[package]] [[package]]
name = "psycopg" name = "psycopg"
version = "3.3.5" version = "3.3.5"