small cleanup of comments/added optional impersonate into run_bounded_cycle for monitors like bhphoto that need different clients than what we support/dont need to rotate
This commit is contained in:
1 parent
4563cdf920
commit
6720592685
3 files changed
+9
-13
No files matched your search
@@ -33,20 +33,8 @@ FAILURE_DELAY = 3
|
|||||||
|
|
||||||
TIMEOUT_SECONDS = 30
|
TIMEOUT_SECONDS = 30
|
||||||
CYCLE_TIMEOUT_SECONDS = 600
|
CYCLE_TIMEOUT_SECONDS = 600
|
||||||
# curl_cffi's AsyncSession defaults to 10 handles and blocks on an untimed
|
|
||||||
# pool.get() past that, so a monitor fanning out wider silently serialises with
|
|
||||||
# no timeout covering the wait. Handles are allocated lazily (the pool holds
|
|
||||||
# None placeholders until popped), so a generous ceiling costs a low-concurrency
|
|
||||||
# monitor nothing. Monitors fanning out past this pass max_clients explicitly.
|
|
||||||
MAX_CURL_HANDLES = 64
|
MAX_CURL_HANDLES = 64
|
||||||
# Budget for closing a session. curl_cffi's close awaits the multi handle with
|
|
||||||
# no timeout of its own, so a wedged transfer would otherwise park whatever is
|
|
||||||
# closing the session, including an `async with` exit.
|
|
||||||
SESSION_CLOSE_TIMEOUT_SECONDS = 10
|
SESSION_CLOSE_TIMEOUT_SECONDS = 10
|
||||||
# Explicit curl_cffi profiles rather than the "chrome" / "chrome_android"
|
|
||||||
# aliases, which float to the newest build (chrome146 / chrome131_android in
|
|
||||||
# curl_cffi 0.15). Chrome 116, 123 and everything from 133a up were refused by
|
|
||||||
# B&H's Cloudflare on every try (2026-09-26); the ones listed passed.
|
|
||||||
CLIENT_IDENTIFIERS = [
|
CLIENT_IDENTIFIERS = [
|
||||||
"chrome119",
|
"chrome119",
|
||||||
"chrome120",
|
"chrome120",
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ from contextlib import asynccontextmanager, nullcontext
|
|||||||
import aiohttp
|
import aiohttp
|
||||||
from aioproxies import AioProxies
|
from aioproxies import AioProxies
|
||||||
from curl_cffi.requests import AsyncSession
|
from curl_cffi.requests import AsyncSession
|
||||||
|
from curl_cffi.requests.impersonate import BrowserTypeLiteral
|
||||||
from log_setup import setup_logging
|
from log_setup import setup_logging
|
||||||
from motor.motor_asyncio import AsyncIOMotorCollection
|
from motor.motor_asyncio import AsyncIOMotorCollection
|
||||||
from motor.motor_asyncio import AsyncIOMotorClient
|
from motor.motor_asyncio import AsyncIOMotorClient
|
||||||
@@ -377,6 +378,7 @@ async def run_bounded_cycle(
|
|||||||
session_timeout: int = TIMEOUT_SECONDS,
|
session_timeout: int = TIMEOUT_SECONDS,
|
||||||
max_clients: int = MAX_CURL_HANDLES,
|
max_clients: int = MAX_CURL_HANDLES,
|
||||||
cycle_timeout: int = CYCLE_TIMEOUT_SECONDS,
|
cycle_timeout: int = CYCLE_TIMEOUT_SECONDS,
|
||||||
|
impersonate: BrowserTypeLiteral | None = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> CycleResult | None:
|
) -> CycleResult | None:
|
||||||
"""Run one monitor cycle on a fresh session under a hard time budget.
|
"""Run one monitor cycle on a fresh session under a hard time budget.
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import random
|
|||||||
from aioproxies import AioProxies
|
from aioproxies import AioProxies
|
||||||
from aioweb import Response
|
from aioweb import Response
|
||||||
from aioweb_tls import CurlCffi, TLSSession
|
from aioweb_tls import CurlCffi, TLSSession
|
||||||
|
from curl_cffi.requests.impersonate import BrowserTypeLiteral
|
||||||
from monitor_utils.constants import (
|
from monitor_utils.constants import (
|
||||||
CLIENT_IDENTIFIERS,
|
CLIENT_IDENTIFIERS,
|
||||||
MAX_CURL_HANDLES,
|
MAX_CURL_HANDLES,
|
||||||
@@ -70,6 +71,7 @@ def get_default_session(
|
|||||||
headers: dict | None = None,
|
headers: dict | None = None,
|
||||||
timeout: int = TIMEOUT_SECONDS,
|
timeout: int = TIMEOUT_SECONDS,
|
||||||
max_clients: int = MAX_CURL_HANDLES,
|
max_clients: int = MAX_CURL_HANDLES,
|
||||||
|
impersonate: BrowserTypeLiteral | None = None,
|
||||||
) -> TLSSession:
|
) -> TLSSession:
|
||||||
"""Return a TLSSession impersonating a random client fingerprint.
|
"""Return a TLSSession impersonating a random client fingerprint.
|
||||||
|
|
||||||
@@ -98,13 +100,17 @@ def get_default_session(
|
|||||||
checkout. Keep it at or above the monitor's real fan-out (the sum
|
checkout. Keep it at or above the monitor's real fan-out (the sum
|
||||||
of its pagination semaphores plus any unsemaphored per-cycle
|
of its pagination semaphores plus any unsemaphored per-cycle
|
||||||
requests) so the wait is never reachable.
|
requests) so the wait is never reachable.
|
||||||
|
impersonate: A curl_cffi profile to pin instead of drawing one from
|
||||||
|
CLIENT_IDENTIFIERS, for a retailer whose bot protection only
|
||||||
|
tolerates some profiles from a given IP (B&H's Cloudflare refuses
|
||||||
|
every chrome profile on a flagged IP but still accepts safari).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A BoundedCloseSession (a TLSSession whose close is bounded) over a
|
A BoundedCloseSession (a TLSSession whose close is bounded) over a
|
||||||
curl_cffi backend with a random impersonation, with the proxy pool
|
curl_cffi backend with a random impersonation, with the proxy pool
|
||||||
attached as proxy_pool.
|
attached as proxy_pool.
|
||||||
"""
|
"""
|
||||||
impersonate = random.choice(CLIENT_IDENTIFIERS)
|
impersonate = impersonate or random.choice(CLIENT_IDENTIFIERS)
|
||||||
session = BoundedCloseSession(
|
session = BoundedCloseSession(
|
||||||
backend=CurlCffi(impersonate=impersonate),
|
backend=CurlCffi(impersonate=impersonate),
|
||||||
headers=headers,
|
headers=headers,
|
||||||
|
|||||||
Reference in new issue
Block a user