fix: copy config dicts, case-insensitive scheme/headers, guard honors ClientTimeout

T1-4: proxies/overwrites/domain_overwrites were aliased, so clear_proxies()
on one session emptied a sibling built from the same config dict, and the
proxy lookup used the raw scheme so HTTP:// missed the http key. both sent
the request direct. copy the dicts; resolve the scheme via yarl (lowercase).

T2-7: the hang-guard ignored a per-call ClientTimeout and killed the call at
the session timeout + slack. read .total off it.

T2-8: header names merged and overwritten case-sensitively, so curl_cffi
sent both User-Agent and user-agent (400 on strict servers). merge through
CIMultiDict and match overwrites case-insensitively.

Signed-off-by: disqualifier <dev@disqualifier.me>
This commit is contained in:
dsql committed 2026-10-16 04:00:33 -04:00
1 parent b009c0cf50
commit 250c9cf2eb
1 file changed
+28 -22
+28 -22
View File
@@ -15,6 +15,7 @@ import warnings
from http.cookies import SimpleCookie from http.cookies import SimpleCookie
import aiohttp import aiohttp
from multidict import CIMultiDict
from yarl import URL from yarl import URL
from commons import aretry from commons import aretry
@@ -85,12 +86,15 @@ class ExtendedSession:
kwargs: passed through to aiohttp.ClientSession kwargs: passed through to aiohttp.ClientSession
""" """
self.inject = inject self.inject = inject
self.header_overwrites = overwrites or {} # copied, not aliased: a shared config dict must not be mutated by (or leak a
self.domain_overwrites = domain_overwrites or {} # clear_proxies() into) a sibling session
self.header_overwrites = dict(overwrites or {})
self.domain_overwrites = dict(domain_overwrites or {})
self.ephemeral_headers = {} self.ephemeral_headers = {}
self.proxies = proxies or {} self.proxies = dict(proxies or {})
# own header layer, not aiohttp privates, so update_headers/clear_headers work # own header layer, not aiohttp privates, so update_headers/clear_headers work;
self._default_headers = dict(headers or {}) # case-insensitive like HTTP header names
self._default_headers = CIMultiDict(headers or {})
self._session_timeout = timeout self._session_timeout = timeout
self._session_kwargs = kwargs self._session_kwargs = kwargs
# aiohttp>=3.14 needs a running loop to build ClientSession; built lazily instead # aiohttp>=3.14 needs a running loop to build ClientSession; built lazily instead
@@ -139,8 +143,12 @@ class ExtendedSession:
self.header_overwrites[header] = value self.header_overwrites[header] = value
def _apply_overwrites(self, request_headers): def _apply_overwrites(self, request_headers):
"""apply static overwrites and ephemeral headers to a request's headers""" """apply static overwrites and ephemeral headers to a request's headers
request_headers = dict(request_headers or {})
header names match case-insensitively, so an overwrite replaces a `user-agent`
rather than sending it alongside a second `User-Agent`
"""
request_headers = CIMultiDict(request_headers or {})
# list()-snapshot so a concurrent mutation of the shared dicts can't raise # list()-snapshot so a concurrent mutation of the shared dicts can't raise
# "dict changed size during iteration" # "dict changed size during iteration"
for header, value in list(self.header_overwrites.items()): for header, value in list(self.header_overwrites.items()):
@@ -153,7 +161,7 @@ class ExtendedSession:
request_headers.update(value) request_headers.update(value)
else: else:
request_headers[header] = value request_headers[header] = value
return request_headers return dict(request_headers)
def set_ephemeral(self, header, value_callable): def set_ephemeral(self, header, value_callable):
"""register a header generated fresh (via callable) for each request""" """register a header generated fresh (via callable) for each request"""
@@ -262,7 +270,7 @@ class ExtendedSession:
falling back to them. falling back to them.
""" """
proxies = self.proxies if proxies is None else proxies proxies = self.proxies if proxies is None else proxies
scheme = url.split("://")[0] scheme = URL(url).scheme
return proxies.get(scheme) return proxies.get(scheme)
# ------------------------------------------------------------------------- # -------------------------------------------------------------------------
@@ -353,7 +361,8 @@ class ExtendedSession:
""" """
url = self._apply_domain_overwrites(url) url = self._apply_domain_overwrites(url)
proxy = self._get_proxy(url, kwargs.pop("proxies", None)) proxy = self._get_proxy(url, kwargs.pop("proxies", None))
merged = {**self._default_headers, **(kwargs.pop("headers", None) or {})} merged = CIMultiDict(self._default_headers)
merged.update(kwargs.pop("headers", None) or {})
headers = self._apply_overwrites(merged) headers = self._apply_overwrites(merged)
timeout = kwargs.get("timeout") timeout = kwargs.get("timeout")
@@ -421,22 +430,19 @@ class ExtendedSession:
kwargs["proxy"] = resolved_proxy kwargs["proxy"] = resolved_proxy
debug = kwargs.pop("debug", False) debug = kwargs.pop("debug", False)
merged = {**self._default_headers, **(kwargs.get("headers") or {})} merged = CIMultiDict(self._default_headers)
merged.update(kwargs.get("headers") or {})
kwargs["headers"] = self._apply_overwrites(merged) kwargs["headers"] = self._apply_overwrites(merged)
kwargs["headers"] = {str(k): str(v) for k, v in kwargs["headers"].items()} kwargs["headers"] = {str(k): str(v) for k, v in kwargs["headers"].items()}
timeout = kwargs.get("timeout") timeout = kwargs.get("timeout")
# the hang-guard deadline, captured as a bare number BEFORE the ClientTimeout # the hang-guard deadline: a per-call timeout (a number, or a ClientTimeout's
# conversion below consumes it: a per-call numeric timeout, else the session # .total), else the session default. a timeout=None (omitted, or the explicit None
# default. a timeout=None (omitted, or the explicit None request_with_retries # request_with_retries passes) means "use the session bound" - so it guards on
# passes) means "use the session bound" - so it guards on self._session_timeout, # self._session_timeout, NOT unguarded. unguarded only when the caller asked for no
# NOT unguarded. the ONLY unguarded case is a session deliberately constructed # bound: a session constructed with timeout=None (the documented long-poll/streaming
# unbounded (Session(timeout=None) -> self._session_timeout is None): the # opt-out), or a per-call ClientTimeout(total=None).
# documented opt-out for a genuinely unbounded call (long-poll/streaming). guard_secs = self._session_timeout if timeout is None else getattr(timeout, "total", timeout)
if isinstance(timeout, (int, float)):
guard_secs = timeout
else:
guard_secs = self._session_timeout
if isinstance(timeout, (int, float)): if isinstance(timeout, (int, float)):
kwargs["timeout"] = aiohttp.ClientTimeout(total=timeout) kwargs["timeout"] = aiohttp.ClientTimeout(total=timeout)
elif timeout is None and "timeout" in kwargs: elif timeout is None and "timeout" in kwargs: