added max_clients arg to get_default_session/cycle timeout seconds

src/monitor_utils/constants.py

@@ -35,6 +35,12 @@ FAILURE_DELAY = 3
 
 TIMEOUT_SECONDS = 30
 CYCLE_TIMEOUT_SECONDS = 600
+# curl_cffi's AsyncSession defaults to 10 handles and blocks on an untimed
+# pool.get() past that, so a monitor fanning out wider silently serialises with
+# no timeout covering the wait. Handles are allocated lazily (the pool holds
+# None placeholders until popped), so a generous ceiling costs a low-concurrency
+# monitor nothing. Monitors fanning out past this pass max_clients explicitly.
+MAX_CURL_HANDLES = 64
 CLIENT_IDENTIFIERS = ["chrome", "chrome_android", "safari", "safari_ios"]
 CLIENT_TIMEOUT = aiohttp.ClientTimeout(
     total=TIMEOUT_SECONDS, sock_connect=30, sock_read=30

src/monitor_utils/sessions.py

@@ -11,13 +11,19 @@ import random
 from aioproxies import AioProxies
 from aioweb import Response
 from aioweb_tls import CurlCffi, TLSSession
-from monitor_utils.constants import CLIENT_IDENTIFIERS, RETRY_LIMIT, TIMEOUT_SECONDS
+from monitor_utils.constants import (
+    CLIENT_IDENTIFIERS,
+    MAX_CURL_HANDLES,
+    RETRY_LIMIT,
+    TIMEOUT_SECONDS,
+)
 
 
 def get_default_session(
     proxies: AioProxies,
     headers: dict | None = None,
     timeout: int = TIMEOUT_SECONDS,
+    max_clients: int = MAX_CURL_HANDLES,
 ) -> TLSSession:
     """Return a TLSSession impersonating a random client fingerprint.
 
@@ -39,6 +45,13 @@ def get_default_session(
         timeout: Total request timeout in seconds, defaulting to TIMEOUT_SECONDS.
             aioweb's own default is only 10s, so this restores the wider budget
             the old sessions used and slow retailers depend on.
+        max_clients: Size of the curl handle pool, defaulting to
+            MAX_CURL_HANDLES. This is a hard concurrency ceiling: a request
+            finding the pool empty waits in pop_curl's untimed pool.get(), and
+            `timeout` does not cover that wait, only the transfer after
+            checkout. Keep it at or above the monitor's real fan-out (the sum
+            of its pagination semaphores plus any unsemaphored per-cycle
+            requests) so the wait is never reachable.
 
     Returns:
         A TLSSession over a curl_cffi backend with a random impersonation, with
@@ -46,7 +59,10 @@ def get_default_session(
     """
     impersonate = random.choice(CLIENT_IDENTIFIERS)
     session = TLSSession(
-        backend=CurlCffi(impersonate=impersonate), headers=headers, timeout=timeout
+        backend=CurlCffi(impersonate=impersonate),
+        headers=headers,
+        timeout=timeout,
+        max_clients=max_clients,
     )
     session.proxy_pool = proxies
     return session