enable Playwright Stealth with French locale for collection
This commit is contained in:
parent
c9ee4bde30
commit
5d68f0f776
@ -34,8 +34,10 @@ to source access failures.
|
||||
Linux may need Playwright's system libraries installed by an administrator.
|
||||
Headed mode requires a graphical session. Omitting `--headed` runs headless, but
|
||||
headless access failed in the initial live test on this machine.
|
||||
Do not run overlapping jobs. No credentials, proxies, CAPTCHA solvers, stealth
|
||||
plugins, or automatic access-block retries are used.
|
||||
Playwright Stealth is enabled for every collection, in headed and headless mode,
|
||||
with French language overrides matching the browser locale. It does not guarantee
|
||||
access or solve challenges; collection still stops on access blocks without retries.
|
||||
Do not run overlapping jobs. No credentials, proxies, or CAPTCHA solvers are used.
|
||||
|
||||
`searches.toml` defines broad nationwide car-category searches. Add queries/families
|
||||
there without changing code. Saab 900 and Volvo 850 queries do not restrict
|
||||
|
||||
@ -4,7 +4,7 @@ version = "0.1.0"
|
||||
description = "French youngtimer asking-price observatory"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.13"
|
||||
dependencies = ["playwright>=1.58,<2", "psycopg[binary]>=3.2,<4", "simplejson>=3.20,<4"]
|
||||
dependencies = ["playwright>=1.58,<2", "playwright-stealth>=2,<3", "psycopg[binary]>=3.2,<4", "simplejson>=3.20,<4"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["ruff>=0.14,<0.15"]
|
||||
|
||||
@ -315,10 +315,11 @@ def parse_collection_args(argv=None, description=None):
|
||||
|
||||
def collect_browser(args):
|
||||
from playwright.sync_api import sync_playwright
|
||||
from playwright_stealth import Stealth
|
||||
|
||||
result = None
|
||||
try:
|
||||
with sync_playwright() as playwright:
|
||||
with Stealth(navigator_languages_override=("fr-FR", "fr")).use_sync(sync_playwright()) as playwright:
|
||||
browser = playwright.chromium.launch(headless=not args.headed)
|
||||
try:
|
||||
context = browser.new_context(locale="fr-FR")
|
||||
@ -335,6 +336,7 @@ def collect_browser(args):
|
||||
return capture_403(directory, response, page, browser, {
|
||||
"headed": args.headed, "locale": "fr-FR", "delay_seconds": args.delay,
|
||||
"max_pages": args.max_pages, "max_details": args.max_details,
|
||||
"stealth": True,
|
||||
})
|
||||
raise SourceAccessError(response.status if response else None, capture=capture)
|
||||
return page.locator("script#__NEXT_DATA__").text_content(timeout=15_000)
|
||||
|
||||
@ -17,10 +17,34 @@ class BrowserCleanupTests(unittest.TestCase):
|
||||
self.args = Namespace(headed=False, searches=[], output=self.path.parent,
|
||||
max_pages=1, delay=1, search_only=True, max_details=None)
|
||||
self.playwright = self.enterContext(patch("playwright.sync_api.sync_playwright"))
|
||||
self.stealth = self.enterContext(patch("playwright_stealth.Stealth"))
|
||||
self.stealth.return_value.use_sync.side_effect = lambda manager: manager
|
||||
self.browser = self.playwright.return_value.__enter__.return_value.chromium.launch.return_value
|
||||
self.collect = self.enterContext(patch("car_tracker.collect.collect", return_value=(self.path, "complete")))
|
||||
self.stderr = self.enterContext(patch("sys.stderr", new_callable=io.StringIO))
|
||||
|
||||
def test_stealth_wraps_playwright_with_french_languages_in_both_modes(self):
|
||||
for headed in (False, True):
|
||||
with self.subTest(headed=headed):
|
||||
self.args.headed = headed
|
||||
self.stealth.reset_mock()
|
||||
self.browser.reset_mock()
|
||||
self.assertEqual(collect_browser(self.args), (self.path, "complete"))
|
||||
self.stealth.assert_called_once_with(navigator_languages_override=("fr-FR", "fr"))
|
||||
self.stealth.return_value.use_sync.assert_called_once_with(self.playwright.return_value)
|
||||
self.playwright.return_value.__enter__.return_value.chromium.launch.assert_called_with(
|
||||
headless=not headed,
|
||||
)
|
||||
self.browser.new_context.assert_called_once_with(locale="fr-FR")
|
||||
self.browser.new_context.return_value.new_page.assert_called_once_with()
|
||||
self.browser.close.assert_called_once_with()
|
||||
|
||||
def test_stealth_setup_failure_is_not_suppressed(self):
|
||||
self.stealth.return_value.use_sync.side_effect = RuntimeError("stealth failed")
|
||||
with self.assertRaisesRegex(RuntimeError, "stealth failed"):
|
||||
collect_browser(self.args)
|
||||
self.collect.assert_not_called()
|
||||
|
||||
def test_browser_close_failure_retains_finalized_result(self):
|
||||
self.browser.close.side_effect = RuntimeError("SYNTHETIC_SECRET")
|
||||
self.assertEqual(collect_browser(self.args), (self.path, "complete"))
|
||||
|
||||
@ -76,11 +76,16 @@ class DiagnosticTests(unittest.TestCase):
|
||||
search_only=False, output=self.root.parent.parent,
|
||||
searches=[{"family": "synthetic", "text": "synthetic"}])
|
||||
module = SimpleNamespace(sync_playwright=lambda: nullcontext(playwright))
|
||||
with patch.dict("sys.modules", {"playwright.sync_api": module}):
|
||||
with patch("playwright_stealth.Stealth") as stealth, patch.dict(
|
||||
"sys.modules", {"playwright.sync_api": module},
|
||||
):
|
||||
stealth.return_value.use_sync.side_effect = lambda manager: manager
|
||||
path, status = collect_browser(args)
|
||||
manifest = json.loads(path.read_text())
|
||||
self.assertEqual(status, "failed")
|
||||
self.assertEqual(manifest["diagnostics"]["capture_status"], "complete")
|
||||
self.assertTrue((path.parent / "diagnostics/http-403/response-body.html").exists())
|
||||
metadata = json.loads((path.parent / "diagnostics/http-403/metadata.json").read_text())
|
||||
self.assertTrue(metadata["settings"]["stealth"])
|
||||
self.page.goto.assert_called_once()
|
||||
self.browser.close.assert_called_once()
|
||||
|
||||
14
uv.lock
generated
14
uv.lock
generated
@ -8,6 +8,7 @@ version = "0.1.0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "playwright" },
|
||||
{ name = "playwright-stealth" },
|
||||
{ name = "psycopg", extra = ["binary"] },
|
||||
{ name = "simplejson" },
|
||||
]
|
||||
@ -20,6 +21,7 @@ dev = [
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "playwright", specifier = ">=1.58,<2" },
|
||||
{ name = "playwright-stealth", specifier = ">=2,<3" },
|
||||
{ name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" },
|
||||
{ name = "simplejson", specifier = ">=3.20,<4" },
|
||||
]
|
||||
@ -103,6 +105,18 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/6c/ec/06b55d619a7082a766aa04f2c6bb31435c87f02930087d8a0517119408fa/playwright-1.62.0-py3-none-win_arm64.whl", hash = "sha256:ea8d3055aa9d5a9f1832ac82517bd8b42c78fac7ebcbebb0107116735c8cb6a1", size = 34208868, upload-time = "2026-07-31T17:01:11.818Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "playwright-stealth"
|
||||
version = "2.0.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "playwright" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a4/db/6ade5d539c7d151b9defc78fafa8b65aa52352617d0e7699b47008bd801f/playwright_stealth-2.0.3.tar.gz", hash = "sha256:1d8e488fbdd8f190f1269ea8cf5d57d14df3a9f1af1001c41ee3588b2aac3133", size = 25751, upload-time = "2026-04-04T02:50:33.88Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b5/10/607c409712c02a26c4cb794820514cb7fdaaeac15fb05bed917fb8a354b3/playwright_stealth-2.0.3-py3-none-any.whl", hash = "sha256:1887ade423ab7ff8ae16d363a30a38de0b5817e1e4a29d47b74bf3a0e3dbfcb4", size = 34385, upload-time = "2026-04-04T02:50:35.246Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psycopg"
|
||||
version = "3.3.5"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user