fix: copy config dicts, case-insensitive scheme/headers, guard honors ClientTimeout

T1-4: proxies/overwrites/domain_overwrites were aliased, so clear_proxies()
on one session emptied a sibling built from the same config dict, and the
proxy lookup used the raw scheme so HTTP:// missed the http key. both sent
the request direct. copy the dicts; resolve the scheme via yarl (lowercase).

T2-7: the hang-guard ignored a per-call ClientTimeout and killed the call at
the session timeout + slack. read .total off it.

T2-8: header names merged and overwritten case-sensitively, so curl_cffi
sent both User-Agent and user-agent (400 on strict servers). merge through
CIMultiDict and match overwrites case-insensitively.

Signed-off-by: disqualifier <dev@disqualifier.me>
This commit is contained in:
dsql committed 2026-10-16 04:00:33 -04:00
1 parent b009c0cf50
commit 250c9cf2eb
1 file changed
+28 -22
+28 -22
View File
@@ -15,6 +15,7 @@ import warnings
from http.cookies import SimpleCookie
import aiohttp
from multidict import CIMultiDict
from yarl import URL
from commons import aretry
@@ -85,12 +86,15 @@ class ExtendedSession:
kwargs: passed through to aiohttp.ClientSession
"""
self.inject = inject
self.header_overwrites = overwrites or {}
self.domain_overwrites = domain_overwrites or {}
# copied, not aliased: a shared config dict must not be mutated by (or leak a
# clear_proxies() into) a sibling session
self.header_overwrites = dict(overwrites or {})
self.domain_overwrites = dict(domain_overwrites or {})
self.ephemeral_headers = {}
self.proxies = proxies or {}
# own header layer, not aiohttp privates, so update_headers/clear_headers work
self._default_headers = dict(headers or {})
self.proxies = dict(proxies or {})
# own header layer, not aiohttp privates, so update_headers/clear_headers work;
# case-insensitive like HTTP header names
self._default_headers = CIMultiDict(headers or {})
self._session_timeout = timeout
self._session_kwargs = kwargs
# aiohttp>=3.14 needs a running loop to build ClientSession; built lazily instead
@@ -139,8 +143,12 @@ class ExtendedSession:
self.header_overwrites[header] = value
def _apply_overwrites(self, request_headers):
"""apply static overwrites and ephemeral headers to a request's headers"""
request_headers = dict(request_headers or {})
"""apply static overwrites and ephemeral headers to a request's headers
header names match case-insensitively, so an overwrite replaces a `user-agent`
rather than sending it alongside a second `User-Agent`
"""
request_headers = CIMultiDict(request_headers or {})
# list()-snapshot so a concurrent mutation of the shared dicts can't raise
# "dict changed size during iteration"
for header, value in list(self.header_overwrites.items()):
@@ -153,7 +161,7 @@ class ExtendedSession:
request_headers.update(value)
else:
request_headers[header] = value
return request_headers
return dict(request_headers)
def set_ephemeral(self, header, value_callable):
"""register a header generated fresh (via callable) for each request"""
@@ -262,7 +270,7 @@ class ExtendedSession:
falling back to them.
"""
proxies = self.proxies if proxies is None else proxies
scheme = url.split("://")[0]
scheme = URL(url).scheme
return proxies.get(scheme)
# -------------------------------------------------------------------------
@@ -353,7 +361,8 @@ class ExtendedSession:
"""
url = self._apply_domain_overwrites(url)
proxy = self._get_proxy(url, kwargs.pop("proxies", None))
merged = {**self._default_headers, **(kwargs.pop("headers", None) or {})}
merged = CIMultiDict(self._default_headers)
merged.update(kwargs.pop("headers", None) or {})
headers = self._apply_overwrites(merged)
timeout = kwargs.get("timeout")
@@ -421,22 +430,19 @@ class ExtendedSession:
kwargs["proxy"] = resolved_proxy
debug = kwargs.pop("debug", False)
merged = {**self._default_headers, **(kwargs.get("headers") or {})}
merged = CIMultiDict(self._default_headers)
merged.update(kwargs.get("headers") or {})
kwargs["headers"] = self._apply_overwrites(merged)
kwargs["headers"] = {str(k): str(v) for k, v in kwargs["headers"].items()}
timeout = kwargs.get("timeout")
# the hang-guard deadline, captured as a bare number BEFORE the ClientTimeout
# conversion below consumes it: a per-call numeric timeout, else the session
# default. a timeout=None (omitted, or the explicit None request_with_retries
# passes) means "use the session bound" - so it guards on self._session_timeout,
# NOT unguarded. the ONLY unguarded case is a session deliberately constructed
# unbounded (Session(timeout=None) -> self._session_timeout is None): the
# documented opt-out for a genuinely unbounded call (long-poll/streaming).
if isinstance(timeout, (int, float)):
guard_secs = timeout
else:
guard_secs = self._session_timeout
# the hang-guard deadline: a per-call timeout (a number, or a ClientTimeout's
# .total), else the session default. a timeout=None (omitted, or the explicit None
# request_with_retries passes) means "use the session bound" - so it guards on
# self._session_timeout, NOT unguarded. unguarded only when the caller asked for no
# bound: a session constructed with timeout=None (the documented long-poll/streaming
# opt-out), or a per-call ClientTimeout(total=None).
guard_secs = self._session_timeout if timeout is None else getattr(timeout, "total", timeout)
if isinstance(timeout, (int, float)):
kwargs["timeout"] = aiohttp.ClientTimeout(total=timeout)
elif timeout is None and "timeout" in kwargs: