diff --git a/src/monitor_utils/constants.py b/src/monitor_utils/constants.py index 3b0f7b0..d018ae2 100644 --- a/src/monitor_utils/constants.py +++ b/src/monitor_utils/constants.py @@ -33,20 +33,8 @@ FAILURE_DELAY = 3 TIMEOUT_SECONDS = 30 CYCLE_TIMEOUT_SECONDS = 600 -# curl_cffi's AsyncSession defaults to 10 handles and blocks on an untimed -# pool.get() past that, so a monitor fanning out wider silently serialises with -# no timeout covering the wait. Handles are allocated lazily (the pool holds -# None placeholders until popped), so a generous ceiling costs a low-concurrency -# monitor nothing. Monitors fanning out past this pass max_clients explicitly. MAX_CURL_HANDLES = 64 -# Budget for closing a session. curl_cffi's close awaits the multi handle with -# no timeout of its own, so a wedged transfer would otherwise park whatever is -# closing the session, including an `async with` exit. SESSION_CLOSE_TIMEOUT_SECONDS = 10 -# Explicit curl_cffi profiles rather than the "chrome" / "chrome_android" -# aliases, which float to the newest build (chrome146 / chrome131_android in -# curl_cffi 0.15). Chrome 116, 123 and everything from 133a up were refused by -# B&H's Cloudflare on every try (2026-09-26); the ones listed passed. CLIENT_IDENTIFIERS = [ "chrome119", "chrome120", diff --git a/src/monitor_utils/runner.py b/src/monitor_utils/runner.py index d1c2a0f..c8f3e19 100644 --- a/src/monitor_utils/runner.py +++ b/src/monitor_utils/runner.py @@ -6,6 +6,7 @@ from contextlib import asynccontextmanager, nullcontext import aiohttp from aioproxies import AioProxies from curl_cffi.requests import AsyncSession +from curl_cffi.requests.impersonate import BrowserTypeLiteral from log_setup import setup_logging from motor.motor_asyncio import AsyncIOMotorCollection from motor.motor_asyncio import AsyncIOMotorClient @@ -377,6 +378,7 @@ async def run_bounded_cycle( session_timeout: int = TIMEOUT_SECONDS, max_clients: int = MAX_CURL_HANDLES, cycle_timeout: int = CYCLE_TIMEOUT_SECONDS, + impersonate: BrowserTypeLiteral | None = None, **kwargs, ) -> CycleResult | None: """Run one monitor cycle on a fresh session under a hard time budget. diff --git a/src/monitor_utils/sessions.py b/src/monitor_utils/sessions.py index 7587d51..d18ca1e 100644 --- a/src/monitor_utils/sessions.py +++ b/src/monitor_utils/sessions.py @@ -13,6 +13,7 @@ import random from aioproxies import AioProxies from aioweb import Response from aioweb_tls import CurlCffi, TLSSession +from curl_cffi.requests.impersonate import BrowserTypeLiteral from monitor_utils.constants import ( CLIENT_IDENTIFIERS, MAX_CURL_HANDLES, @@ -70,6 +71,7 @@ def get_default_session( headers: dict | None = None, timeout: int = TIMEOUT_SECONDS, max_clients: int = MAX_CURL_HANDLES, + impersonate: BrowserTypeLiteral | None = None, ) -> TLSSession: """Return a TLSSession impersonating a random client fingerprint. @@ -98,13 +100,17 @@ def get_default_session( checkout. Keep it at or above the monitor's real fan-out (the sum of its pagination semaphores plus any unsemaphored per-cycle requests) so the wait is never reachable. + impersonate: A curl_cffi profile to pin instead of drawing one from + CLIENT_IDENTIFIERS, for a retailer whose bot protection only + tolerates some profiles from a given IP (B&H's Cloudflare refuses + every chrome profile on a flagged IP but still accepts safari). Returns: A BoundedCloseSession (a TLSSession whose close is bounded) over a curl_cffi backend with a random impersonation, with the proxy pool attached as proxy_pool. """ - impersonate = random.choice(CLIENT_IDENTIFIERS) + impersonate = impersonate or random.choice(CLIENT_IDENTIFIERS) session = BoundedCloseSession( backend=CurlCffi(impersonate=impersonate), headers=headers,