diff --git a/README.md b/README.md index 6dc26d4..d389b43 100644 --- a/README.md +++ b/README.md @@ -11,18 +11,18 @@ and swap the HTTP client while inheriting everything else. `requirements.txt`: ``` -aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.8 +aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.9 ``` Direct: ```bash -pip install "aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.8" +pip install "aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.9" ``` Requires `aiohttp` and `yarl` (pulled transitively). -Drop the `@v0.1.8` suffix from the line above to install the latest unpinned. +Drop the `@v0.1.9` suffix from the line above to install the latest unpinned. ## Usage @@ -144,6 +144,24 @@ Two changes can't be shimmed without re-introducing the bugs they fix: ## Changelog +### v0.1.9 + +- **`_get_proxy()`/`request()` proxy resolution now checks `is None`, not + truthiness.** A per-call `proxies={}` previously fell back to the session's + configured proxies instead of disabling them for that call. `request()` also no + longer clobbers a native `proxy=` kwarg with the resolved session/`proxies=` + value (a real IP-unmasking leak) — passing both now raises `ValueError` instead + of silently picking one. +- **`Response.text(encoding=...)` no longer returns a stale cached decode.** A + second call with an explicit `encoding=` previously still returned the first + (possibly differently-encoded) cached decode; an explicit encoding now bypasses + the cache. +- **`preview()` now rewrites the url before resolving cookies**, matching what + `request()` actually sends — previously cookies were resolved against the + pre-rewrite host, which could miss or misattribute host-bound cookies. +- **`request_with_retries(attempts=0)` now floors to 1 attempt**, not silently + `DEFAULT_ATTEMPTS` (3). `attempts` is checked with `is None`, not truthiness. + ### v0.1.8 - **`request()` no longer flattens `aiohttp.ClientError` subtypes.** Every failure diff --git a/pyproject.toml b/pyproject.toml index 2a2b9ab..e4d484c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "aioweb" -version = "0.1.8" +version = "0.1.9" description = "Async HTTP session wrapper over aiohttp — proxies, header overwrites, retries, previews. Config-free, installable." requires-python = ">=3.10" dependencies = [ diff --git a/src/aioweb/preview.py b/src/aioweb/preview.py index 12447b3..8df7290 100644 --- a/src/aioweb/preview.py +++ b/src/aioweb/preview.py @@ -30,12 +30,10 @@ class RequestPreview: def as_curl(self): """equivalent cURL command for the request - every interpolated value is shell-quoted with shlex.quote, so headers, - body, url, or proxy containing quotes/spaces/metacharacters produce a - valid, non-injectable command rather than a broken or unsafe one. `params` - is merged into the url's query string (via yarl) and `timeout` is rendered - as `--max-time`, so the emitted command is faithful to what request() - actually sends rather than dropping either silently. + every interpolated value is shlex.quote'd (non-injectable even with quotes/ + spaces/metacharacters). `params` is merged into the url's query string and + `timeout` rendered as `--max-time`, so the command matches what request() + actually sends. """ parts = [f"curl -X {shlex.quote(self.details['method'])}"] for header, value in (self.details["headers"] or {}).items(): @@ -43,8 +41,7 @@ class RequestPreview: if self.details["data"] is not None: parts.append(f"--data {shlex.quote(str(self.details['data']))}") elif self.details["json"] is not None: - # is-not-None, not truthiness: an empty-but-valid body ({} / []) must still - # render rather than being dropped as falsy + # is-not-None: an empty-but-valid body ({} / []) must still render parts.append(f"--data {shlex.quote(_json.dumps(self.details['json']))}") url = self.details["url"] if self.details["params"]: diff --git a/src/aioweb/responses.py b/src/aioweb/responses.py index 53f8bd1..2f6e124 100644 --- a/src/aioweb/responses.py +++ b/src/aioweb/responses.py @@ -1,10 +1,8 @@ """ -backend-agnostic response objects for aioweb - -Response is built from primitives (status, headers, content, url, history) rather -than holding a raw aiohttp object, so any backend can produce one. FailureResponse -mirrors the same surface so callers can branch uniformly — every status/predicate is -a property on both. +backend-agnostic response objects for aioweb — Response is built from primitives +(status, headers, content, url, history), not a raw aiohttp object, so any backend can +produce one; FailureResponse mirrors the same surface (every status/predicate a +property on both) so callers can branch uniformly. """ import json as _json @@ -83,9 +81,11 @@ class Response: return self._content def text(self, encoding: Optional[str] = None) -> str: - """decoded text content (cached)""" + """decoded text content, cached only for the default encoding""" + if encoding is not None: + return self._content.decode(encoding) if self._text is None: - self._text = self._content.decode(encoding or self._encoding) + self._text = self._content.decode(self._encoding) return self._text def json(self): diff --git a/src/aioweb/session.py b/src/aioweb/session.py index 977b1e7..492b80c 100644 --- a/src/aioweb/session.py +++ b/src/aioweb/session.py @@ -1,23 +1,18 @@ """ -async HTTP session wrapper over aiohttp - -ExtendedSession adds session-level proxies, header overwrites, ephemeral -(per-request generated) headers, domain rewriting, request previews, and -retry/backoff on top of aiohttp. The actual byte-sending is isolated in -_raw_request() so a different backend (e.g. a TLS-fingerprinting client) can -subclass and override just that one method, inheriting everything else. +async HTTP session wrapper over aiohttp — proxies, header overwrites, ephemeral +headers, domain rewriting, previews, retry/backoff; byte-sending is isolated in +_raw_request() so a subclass can swap backends and inherit everything else. async with ExtendedSession(proxies={"https": "http://..."}) as s: resp = await s.request_with_retries("GET", url) if resp: # FailureResponse is falsy data = resp.json() -config-free: proxies/headers/timeouts are passed at construction or per call. the -backend HTTP session is built lazily on first use (request/cookie access/close), not -in __init__, so construction is safe before an event loop is running (e.g. attaching -to a host object at process startup). -sessions must be closed explicitly (async with, or await s.close()); there is no -__del__ auto-close (that pattern is unsafe for async resources). +config-free (proxies/headers/timeouts passed at construction or per call). the +backend session is built lazily on first use, not in __init__, so construction is +safe before an event loop is running (e.g. bot.http = ExtendedSession(...)). +sessions must be closed explicitly (async with, or await s.close()) — no __del__ +auto-close (unsafe for async resources). """ import asyncio @@ -34,12 +29,7 @@ from .responses import Response, FailureResponse def _route_body(data): - """split a body into (data=, json=) kwargs - - dict OR list bodies are valid JSON and route to json=; everything else - (str/bytes/form) routes to data=. previously only dicts went to json=, so a - JSON list was wrongly form-encoded. - """ + """split a body into (data=, json=) kwargs; dict OR list routes to json=, else data=""" if isinstance(data, (dict, list)): return None, data return data, None @@ -88,15 +78,13 @@ class ExtendedSession: self.domain_overwrites = domain_overwrites or {} self.ephemeral_headers = {} self.proxies = proxies or {} - # track our own default headers instead of touching aiohttp privates + # own header layer, not aiohttp privates, so update_headers/clear_headers work self._default_headers = dict(headers or {}) self._session_timeout = timeout self._session_kwargs = kwargs - # aiohttp.ClientSession (via _create_session) requires a running event loop - # (aiohttp >= 3.14 raises RuntimeError otherwise); building it here would - # break the common host pattern of constructing before the loop starts - # (e.g. bot.http = ExtendedSession(...) in Bot.__init__). build lazily on - # first access instead, via the `session` property / _ensure_session(). + # aiohttp>=3.14 requires a running loop to build ClientSession; build lazily + # (via `session` / _ensure_session()) so construction before the loop starts + # (e.g. bot.http = ExtendedSession(...) in Bot.__init__) doesn't crash self._session = None @property @@ -113,20 +101,13 @@ class ExtendedSession: ) def _create_session(self, headers, timeout, **kwargs): - """create the backend HTTP session — override to use a different client + """create the backend HTTP session — override to swap HTTP clients - a subclass swapping the HTTP backend (e.g. a TLS-fingerprinting client) - overrides this to return its own session object. the overwrite/domain/ - proxy/retry/preview logic in this class never touches the session object - directly (only _raw_request, the cookie methods, and close do), so those - features work unchanged on any backend. - - `headers` is the session-default header set; the default aiohttp backend - does NOT bake it into the ClientSession (which would copy it into an - immutable per-session map that update_headers/clear_headers can't touch). - instead `_default_headers` is our own mutable layer that request() and - preview() merge per call, so the mutable session-header API actually works. - a backend that needs the defaults baked at construction may use `headers`. + overwrite/domain/proxy/retry/preview logic never touches the session object + directly (only _raw_request, cookies, close do), so those work unchanged on + any backend. `headers` isn't baked in here — aiohttp would copy it into an + immutable map update_headers/clear_headers can't touch; `_default_headers` + is the mutable layer request()/preview() merge per call instead. """ return aiohttp.ClientSession( timeout=aiohttp.ClientTimeout( @@ -148,9 +129,8 @@ class ExtendedSession: def _apply_overwrites(self, request_headers): """apply static overwrites and ephemeral headers to a request's headers""" request_headers = dict(request_headers or {}) - # snapshot the shared override dicts so a concurrent mutation (e.g. a command - # editing ephemerals on a shared session) can't raise "dict changed size during - # iteration" — the loop body is sync, but the snapshot is cheap insurance + # list()-snapshot so a concurrent mutation of the shared dicts can't raise + # "dict changed size during iteration" for header, value in list(self.header_overwrites.items()): if self.inject or header in request_headers: request_headers[header] = value @@ -232,8 +212,13 @@ class ExtendedSession: self.proxies.clear() def _get_proxy(self, url, proxies=None): - """resolve the proxy for a url's scheme""" - proxies = proxies or self.proxies + """resolve the proxy for a url's scheme + + proxies is checked with `is None`, not truthiness, so an explicit + proxies={} disables session proxies for that one call instead of + falling back to them. + """ + proxies = self.proxies if proxies is None else proxies scheme = url.split("://")[0] return proxies.get(scheme) @@ -241,26 +226,20 @@ class ExtendedSession: # cookies def get_cookies(self): - """all cookies stored in the session jar, regardless of domain binding + """all cookies in the session jar, regardless of domain binding - iterates the jar directly rather than filter_cookies() (which needs a url - and, given none, returns only domain-less shared cookies — i.e. {} for any - normal domain-bound cookie). + iterates the jar directly rather than filter_cookies() (which needs a url and, + given none, returns only domain-less shared cookies — {} for any normal one). """ return {c.key: c.value for c in self.session.cookie_jar} def set_cookie(self, name, value, domain=None, path="/"): """set a cookie in the session jar - domain=None (the default) stores a truly shared cookie sent with every - request regardless of host — the jar's own "no response_url" behavior. - pass domain='example.com' (a scheme is optional and defaulted to http://) - to scope the cookie to one host; a bare hostname like 'example.com' is - normalized into a URL so the jar binds it by host instead of silently - storing another domain-less shared cookie (a schemeless domain has no - raw_host, so the jar can't tell it apart from the shared case). - `path` is honored via the morsel itself, since the jar only derives a path - from response_url when the morsel doesn't already carry one. + domain=None stores a truly shared cookie sent to every host. domain='example.com' + (scheme optional, defaulted to http://) scopes it to that host — a bare hostname + is normalized into a URL so the jar binds by host instead of silently storing + another domain-less shared cookie. `path` is honored via the morsel itself. """ cookie = SimpleCookie() cookie[name] = value @@ -277,18 +256,20 @@ class ExtendedSession: self.session.cookie_jar.clear() def _cookies_for_url(self, url): - """dict of cookies the backend would send for url — override per backend - - used by preview(). non-aiohttp backends override this (or return {}); the - rest of preview (domain rewrites, header overwrites) is backend-agnostic. - """ + """dict of cookies the backend would send for url — override per backend (used by preview())""" return {k: v.value for k, v in self.session.cookie_jar.filter_cookies(URL(url)).items()} # ------------------------------------------------------------------------- # preview def preview(self, method, url, **kwargs): - """build a RequestPreview for a request without sending it""" + """build a RequestPreview for a request without sending it + + url is domain-rewritten before cookies are resolved, matching the real + request — cookie binding is host-based, so resolving against the pre-rewrite + host could miss or misattribute cookies. + """ + url = self._apply_domain_overwrites(url) proxy = self._get_proxy(url, kwargs.pop("proxies", None)) merged = {**self._default_headers, **(kwargs.pop("headers", None) or {})} headers = self._apply_overwrites(merged) @@ -307,7 +288,7 @@ class ExtendedSession: return RequestPreview( method=method, - url=self._apply_domain_overwrites(url), + url=url, headers=headers, proxy=proxy, data=kwargs.get("data"), @@ -322,10 +303,9 @@ class ExtendedSession: async def _raw_request(self, method, url, **kwargs) -> Response: """send one request with aiohttp and adapt it into a Response - this is the backend seam: a subclass can override only this method (using a - different HTTP client, e.g. a TLS-fingerprinting one), building a Response - from that backend's primitives. everything else (overwrites, retries, - preview) is inherited. + the backend seam: a subclass overrides only this to swap HTTP clients, + building a Response from that backend's primitives; everything else + (overwrites, retries, preview) is inherited. """ response = await self.session.request(method, url, **kwargs) async with response: @@ -344,13 +324,16 @@ class ExtendedSession: async def request(self, method, url, **kwargs) -> Response: """make a request, applying overwrites/domain rewrites/proxy resolution - raises the real exception subtype on failure: a total timeout raises - aiohttp.ServerTimeoutError, and any other aiohttp.ClientError (e.g. - ClientConnectorError, ClientProxyConnectionError, ClientResponseError with - .status/.headers, TooManyRedirects) is re-raised as-is, not flattened into - the base ClientError — callers can branch by type or read subtype attributes. + raises the real exception subtype on failure (ServerTimeoutError on a total + timeout; any other aiohttp.ClientError re-raised as-is, not flattened, so + callers can branch by type or read subtype attributes like .status/.headers). + a native proxy= is never silently clobbered by resolved proxies= (that would + unmask the caller's real IP) — pass only one of them. """ - kwargs["proxy"] = self._get_proxy(url, kwargs.pop("proxies", None)) + resolved_proxy = self._get_proxy(url, kwargs.pop("proxies", None)) + if "proxy" in kwargs and kwargs["proxy"] is not None and resolved_proxy is not None: + raise ValueError("pass only one of proxy= or proxies= (session/per-call), not both") + kwargs.setdefault("proxy", resolved_proxy) debug = kwargs.pop("debug", False) merged = {**self._default_headers, **(kwargs.get("headers") or {})} @@ -361,10 +344,9 @@ class ExtendedSession: if isinstance(timeout, (int, float)): kwargs["timeout"] = aiohttp.ClientTimeout(total=timeout) elif timeout is None and "timeout" in kwargs: - # an explicit timeout=None reaches aiohttp as ClientTimeout(total=None), - # which DISABLES the timeout and overrides the session default; drop it so - # the session-level timeout applies (matters for request_with_retries, whose - # timeout kwarg defaults to None) + # explicit timeout=None would reach aiohttp as ClientTimeout(total=None) — + # infinite, disabling the session default — so drop it instead (matters for + # request_with_retries, whose timeout kwarg defaults to None) del kwargs["timeout"] url = self._apply_domain_overwrites(url) @@ -377,17 +359,13 @@ class ExtendedSession: log.info("redirect chain: %s", result.redirect_chain) return result except asyncio.TimeoutError as error: - # a total ClientTimeout raises a bare asyncio.TimeoutError, which is NOT an - # aiohttp.ClientError subclass — wrap it as ServerTimeoutError (which IS both - # a ClientError AND a TimeoutError) so direct callers get a typed failure and - # request_with_retries can still label it a timeout + # a bare asyncio.TimeoutError isn't an aiohttp.ClientError subclass — wrap it + # as ServerTimeoutError (both a ClientError AND a TimeoutError) so callers get + # a typed failure and request_with_retries can still label it a timeout raise aiohttp.ServerTimeoutError(f"timeout for {url}: {error}") from error except aiohttp.ClientError as error: - # re-raise the ORIGINAL subtype (ClientConnectorError, ClientProxyConnectionError, - # ClientResponseError with .status/.headers, TooManyRedirects, ...) instead of - # flattening into the base class — direct callers branching by type or reading - # subtype attributes need the real exception; request_with_retries still catches - # the base aiohttp.ClientError below and is unaffected + # re-raise the original subtype (not flattened) — request_with_retries still + # catches the base aiohttp.ClientError below and is unaffected log.error("client error for %s: %s", url, error) raise @@ -400,13 +378,11 @@ class ExtendedSession: returns a Response on success (or non-retryable status), or a falsy FailureResponse if every attempt fails. backoff is exponential - (backoff_base ** attempt). - - timeout defaults to None, which falls back to the session-level timeout set - at construction (aioweb pops a None timeout so it does not reach aiohttp as an - infinite ClientTimeout); pass a number to override per call. + (backoff_base ** attempt). timeout=None falls back to the session-level + timeout (see request()'s note on why explicit None is dropped, not passed + through). attempts=0 floors to 1 (via commons.aretry), not DEFAULT_ATTEMPTS. """ - attempts = attempts or DEFAULT_ATTEMPTS + attempts = DEFAULT_ATTEMPTS if attempts is None else attempts body_data, body_json = _route_body(data) if debug: @@ -438,9 +414,8 @@ class ExtendedSession: attempts, url, exhausted.response.status_code) return exhausted.response except asyncio.TimeoutError: - # request() wraps a total timeout as ServerTimeoutError (a ClientError AND a - # TimeoutError); catch the timeout case first so it's labeled a timeout rather - # than falling into the generic client-error branch below + # request() wraps timeouts as ServerTimeoutError (ClientError + TimeoutError); + # catch it first so it's labeled a timeout, not a generic client error below log.error("all %d attempts timed out for %s", attempts, url) return FailureResponse(reason="timeout", url=url) except aiohttp.ClientError as error: @@ -454,19 +429,15 @@ class ExtendedSession: # lifecycle async def close(self): - """close the backend session — override if the backend's close differs - - a no-op if the session was never built (lazy construction means a session - that made no request and was never otherwise touched has nothing to close). - """ + """close the backend session — override if the backend's close differs; no-op if never built""" if self._session is not None: await self._session.close() def _is_closed(self) -> bool: """whether the backend session is closed — override for non-aiohttp backends - an unbuilt (never-lazily-created) session counts as closed: nothing was - opened, so there is nothing to leak and __del__ should not warn. + an unbuilt session counts as closed: nothing was opened, nothing to leak, + __del__ should not warn. """ if self._session is None: return True @@ -479,8 +450,8 @@ class ExtendedSession: await self.close() def __del__(self): - # do NOT attempt async cleanup here — spinning event loops in a finalizer - # is unsafe. just warn so the leak is visible; callers must close explicitly. + # no async cleanup here — spinning event loops in a finalizer is unsafe; + # just warn so the leak is visible, callers must close explicitly try: closed = self._is_closed() except Exception: