Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0768d643b1 | ||
|
|
e1ab5d38a0 | ||
|
|
b8cd184c64 |
@@ -11,18 +11,18 @@ and swap the HTTP client while inheriting everything else.
|
|||||||
`requirements.txt`:
|
`requirements.txt`:
|
||||||
|
|
||||||
```
|
```
|
||||||
aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.5
|
aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.8
|
||||||
```
|
```
|
||||||
|
|
||||||
Direct:
|
Direct:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install "aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.5"
|
pip install "aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.8"
|
||||||
```
|
```
|
||||||
|
|
||||||
Requires `aiohttp` and `yarl` (pulled transitively).
|
Requires `aiohttp` and `yarl` (pulled transitively).
|
||||||
|
|
||||||
Drop the `@v0.1.5` suffix from the line above to install the latest unpinned.
|
Drop the `@v0.1.8` suffix from the line above to install the latest unpinned.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
@@ -65,6 +65,15 @@ resp = await s.request_with_retries(
|
|||||||
|
|
||||||
Returns a `FailureResponse` (falsy) if every attempt fails.
|
Returns a `FailureResponse` (falsy) if every attempt fails.
|
||||||
|
|
||||||
|
`request()` (the non-retrying call) raises on failure: a total timeout raises
|
||||||
|
`aiohttp.ServerTimeoutError`, and any other network/protocol failure raises its real
|
||||||
|
`aiohttp.ClientError` subtype as-is (`ClientConnectorError`, `ClientProxyConnectionError`,
|
||||||
|
`ClientResponseError` with `.status`/`.headers`, `TooManyRedirects`, ...) — it is not
|
||||||
|
flattened into the base `ClientError`, so direct callers can branch by type or read
|
||||||
|
subtype attributes. `request_with_retries` catches the base `aiohttp.ClientError` (and
|
||||||
|
`asyncio.TimeoutError`) across all attempts and returns a falsy `FailureResponse` instead
|
||||||
|
of raising.
|
||||||
|
|
||||||
## Header overwrites & ephemeral headers
|
## Header overwrites & ephemeral headers
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@@ -88,7 +97,10 @@ s.overwrite_domain("internal.local", "127.0.0.1") # host-substring rewrite
|
|||||||
print(s.preview("POST", url, json={"a": 1}).as_curl()) # equivalent cURL command
|
print(s.preview("POST", url, json={"a": 1}).as_curl()) # equivalent cURL command
|
||||||
```
|
```
|
||||||
|
|
||||||
Pass `debug=True` to `request_with_retries` to log the cURL preview and request flow.
|
`as_curl()` renders `params` (merged into the url's query string) and `timeout` (as
|
||||||
|
`--max-time`) as well as headers/body/proxy, so the emitted command is faithful to what
|
||||||
|
`request()` actually sends. Pass `debug=True` to `request_with_retries` to log the cURL
|
||||||
|
preview and request flow.
|
||||||
|
|
||||||
## Custom backends
|
## Custom backends
|
||||||
|
|
||||||
@@ -132,6 +144,46 @@ Two changes can't be shimmed without re-introducing the bugs they fix:
|
|||||||
|
|
||||||
## Changelog
|
## Changelog
|
||||||
|
|
||||||
|
### v0.1.8
|
||||||
|
|
||||||
|
- **`request()` no longer flattens `aiohttp.ClientError` subtypes.** Every failure
|
||||||
|
(connect errors, proxy errors, `raise_for_status()`-style response errors,
|
||||||
|
redirect limits, ...) was re-raised as a bare `aiohttp.ClientError`, losing the
|
||||||
|
real subtype and its attributes (`.os_error`, `.status`, `.headers`, ...) — a
|
||||||
|
direct caller doing `except ClientProxyConnectionError:` or `if e.status == 401`
|
||||||
|
would silently never match. Now the original exception is re-raised as-is (its
|
||||||
|
subtype, attributes, and `__cause__` all preserved). `request_with_retries`
|
||||||
|
still catches the base `aiohttp.ClientError` across attempts, so its behavior
|
||||||
|
(and its `FailureResponse` return on exhaustion) is unchanged.
|
||||||
|
- **`as_curl()` now renders `params` and `timeout`.** Previously a preview built
|
||||||
|
with `params=` silently omitted the query string (and a `timeout=` omitted
|
||||||
|
`--max-time`), so a `debug=True` cURL replay of a params-driven request hit a
|
||||||
|
different URL than the one actually sent. `params` are now merged into the url's
|
||||||
|
query string (via `yarl`) and `timeout` is emitted as `--max-time`.
|
||||||
|
|
||||||
|
### v0.1.7
|
||||||
|
|
||||||
|
- **`get_cookies()` now returns real cookies.** Previously called `filter_cookies()`
|
||||||
|
with no URL, which only ever returns domain-less shared cookies — every normal
|
||||||
|
domain-bound cookie (including ones set by a real `Set-Cookie` response) was
|
||||||
|
silently omitted. Now iterates the jar directly.
|
||||||
|
- **`set_cookie()` no longer leaks a shared cookie to every host.** A bare hostname
|
||||||
|
(`domain="example.com"`) built a schemeless URL, which aiohttp's jar treats as a
|
||||||
|
domain-less "shared" cookie sent with every request the session makes, including
|
||||||
|
unrelated hosts. A scheme is now added when missing so the cookie is scoped to
|
||||||
|
that host.
|
||||||
|
- **`set_cookie()` now honors `path`** (previously ignored — the cookie always
|
||||||
|
landed at `path="/"`). `domain=None` is unchanged in meaning but now stores a
|
||||||
|
truly shared cookie (sent to every host) instead of one silently bound to
|
||||||
|
`localhost` only, which made `set_cookie(name, value)` (no domain) a silent
|
||||||
|
no-op for any real request.
|
||||||
|
- **The backend session is built lazily**, not in `__init__`. Under aiohttp 3.14,
|
||||||
|
constructing `aiohttp.ClientSession` requires a running event loop; eager
|
||||||
|
construction crashed the common host pattern of attaching a session before the
|
||||||
|
loop starts (e.g. `bot.http = ExtendedSession(...)` in `Bot.__init__`). The
|
||||||
|
session (and any subclass's `_create_session` override) now builds on first
|
||||||
|
access instead.
|
||||||
|
|
||||||
### v0.1.2
|
### v0.1.2
|
||||||
|
|
||||||
- Pinned `commons` to v0.2.1 (retry `attempts` floor fix).
|
- Pinned `commons` to v0.2.1 (retry `attempts` floor fix).
|
||||||
|
|||||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "aioweb"
|
name = "aioweb"
|
||||||
version = "0.1.5"
|
version = "0.1.8"
|
||||||
description = "Async HTTP session wrapper over aiohttp — proxies, header overwrites, retries, previews. Config-free, installable."
|
description = "Async HTTP session wrapper over aiohttp — proxies, header overwrites, retries, previews. Config-free, installable."
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
|||||||
+12
-2
@@ -5,6 +5,8 @@ request preview for aioweb — format or export a request without sending it
|
|||||||
import json as _json
|
import json as _json
|
||||||
import shlex
|
import shlex
|
||||||
|
|
||||||
|
from yarl import URL
|
||||||
|
|
||||||
|
|
||||||
class RequestPreview:
|
class RequestPreview:
|
||||||
"""a formatted preview of an HTTP request (does not send)"""
|
"""a formatted preview of an HTTP request (does not send)"""
|
||||||
@@ -30,7 +32,10 @@ class RequestPreview:
|
|||||||
|
|
||||||
every interpolated value is shell-quoted with shlex.quote, so headers,
|
every interpolated value is shell-quoted with shlex.quote, so headers,
|
||||||
body, url, or proxy containing quotes/spaces/metacharacters produce a
|
body, url, or proxy containing quotes/spaces/metacharacters produce a
|
||||||
valid, non-injectable command rather than a broken or unsafe one.
|
valid, non-injectable command rather than a broken or unsafe one. `params`
|
||||||
|
is merged into the url's query string (via yarl) and `timeout` is rendered
|
||||||
|
as `--max-time`, so the emitted command is faithful to what request()
|
||||||
|
actually sends rather than dropping either silently.
|
||||||
"""
|
"""
|
||||||
parts = [f"curl -X {shlex.quote(self.details['method'])}"]
|
parts = [f"curl -X {shlex.quote(self.details['method'])}"]
|
||||||
for header, value in (self.details["headers"] or {}).items():
|
for header, value in (self.details["headers"] or {}).items():
|
||||||
@@ -41,7 +46,12 @@ class RequestPreview:
|
|||||||
# is-not-None, not truthiness: an empty-but-valid body ({} / []) must still
|
# is-not-None, not truthiness: an empty-but-valid body ({} / []) must still
|
||||||
# render rather than being dropped as falsy
|
# render rather than being dropped as falsy
|
||||||
parts.append(f"--data {shlex.quote(_json.dumps(self.details['json']))}")
|
parts.append(f"--data {shlex.quote(_json.dumps(self.details['json']))}")
|
||||||
parts.append(shlex.quote(str(self.details["url"])))
|
url = self.details["url"]
|
||||||
|
if self.details["params"]:
|
||||||
|
url = str(URL(url).update_query(self.details["params"]))
|
||||||
|
parts.append(shlex.quote(str(url)))
|
||||||
if self.details["proxy"]:
|
if self.details["proxy"]:
|
||||||
parts.append(f"--proxy {shlex.quote(str(self.details['proxy']))}")
|
parts.append(f"--proxy {shlex.quote(str(self.details['proxy']))}")
|
||||||
|
if self.details["timeout"] is not None:
|
||||||
|
parts.append(f"--max-time {shlex.quote(str(self.details['timeout']))}")
|
||||||
return " \\\n ".join(parts)
|
return " \\\n ".join(parts)
|
||||||
|
|||||||
+94
-13
@@ -12,7 +12,10 @@ subclass and override just that one method, inheriting everything else.
|
|||||||
if resp: # FailureResponse is falsy
|
if resp: # FailureResponse is falsy
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
|
|
||||||
config-free: proxies/headers/timeouts are passed at construction or per call.
|
config-free: proxies/headers/timeouts are passed at construction or per call. the
|
||||||
|
backend HTTP session is built lazily on first use (request/cookie access/close), not
|
||||||
|
in __init__, so construction is safe before an event loop is running (e.g. attaching
|
||||||
|
to a host object at process startup).
|
||||||
sessions must be closed explicitly (async with, or await s.close()); there is no
|
sessions must be closed explicitly (async with, or await s.close()); there is no
|
||||||
__del__ auto-close (that pattern is unsafe for async resources).
|
__del__ auto-close (that pattern is unsafe for async resources).
|
||||||
"""
|
"""
|
||||||
@@ -20,6 +23,7 @@ __del__ auto-close (that pattern is unsafe for async resources).
|
|||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
import warnings
|
import warnings
|
||||||
|
from http.cookies import SimpleCookie
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from yarl import URL
|
from yarl import URL
|
||||||
@@ -86,7 +90,27 @@ class ExtendedSession:
|
|||||||
self.proxies = proxies or {}
|
self.proxies = proxies or {}
|
||||||
# track our own default headers instead of touching aiohttp privates
|
# track our own default headers instead of touching aiohttp privates
|
||||||
self._default_headers = dict(headers or {})
|
self._default_headers = dict(headers or {})
|
||||||
self.session = self._create_session(self._default_headers, timeout, **kwargs)
|
self._session_timeout = timeout
|
||||||
|
self._session_kwargs = kwargs
|
||||||
|
# aiohttp.ClientSession (via _create_session) requires a running event loop
|
||||||
|
# (aiohttp >= 3.14 raises RuntimeError otherwise); building it here would
|
||||||
|
# break the common host pattern of constructing before the loop starts
|
||||||
|
# (e.g. bot.http = ExtendedSession(...) in Bot.__init__). build lazily on
|
||||||
|
# first access instead, via the `session` property / _ensure_session().
|
||||||
|
self._session = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def session(self):
|
||||||
|
"""the backend session, built lazily on first access (needs a running loop)"""
|
||||||
|
self._ensure_session()
|
||||||
|
return self._session
|
||||||
|
|
||||||
|
def _ensure_session(self):
|
||||||
|
"""build the backend session on first use — idempotent"""
|
||||||
|
if self._session is None:
|
||||||
|
self._session = self._create_session(
|
||||||
|
self._default_headers, self._session_timeout, **self._session_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
def _create_session(self, headers, timeout, **kwargs):
|
def _create_session(self, headers, timeout, **kwargs):
|
||||||
"""create the backend HTTP session — override to use a different client
|
"""create the backend HTTP session — override to use a different client
|
||||||
@@ -217,13 +241,36 @@ class ExtendedSession:
|
|||||||
# cookies
|
# cookies
|
||||||
|
|
||||||
def get_cookies(self):
|
def get_cookies(self):
|
||||||
"""cookies stored in the session jar"""
|
"""all cookies stored in the session jar, regardless of domain binding
|
||||||
return self.session.cookie_jar.filter_cookies()
|
|
||||||
|
iterates the jar directly rather than filter_cookies() (which needs a url
|
||||||
|
and, given none, returns only domain-less shared cookies — i.e. {} for any
|
||||||
|
normal domain-bound cookie).
|
||||||
|
"""
|
||||||
|
return {c.key: c.value for c in self.session.cookie_jar}
|
||||||
|
|
||||||
def set_cookie(self, name, value, domain=None, path="/"):
|
def set_cookie(self, name, value, domain=None, path="/"):
|
||||||
"""set a cookie in the session jar"""
|
"""set a cookie in the session jar
|
||||||
response_url = URL(domain or "http://localhost")
|
|
||||||
self.session.cookie_jar.update_cookies({name: value}, response_url=response_url)
|
domain=None (the default) stores a truly shared cookie sent with every
|
||||||
|
request regardless of host — the jar's own "no response_url" behavior.
|
||||||
|
pass domain='example.com' (a scheme is optional and defaulted to http://)
|
||||||
|
to scope the cookie to one host; a bare hostname like 'example.com' is
|
||||||
|
normalized into a URL so the jar binds it by host instead of silently
|
||||||
|
storing another domain-less shared cookie (a schemeless domain has no
|
||||||
|
raw_host, so the jar can't tell it apart from the shared case).
|
||||||
|
`path` is honored via the morsel itself, since the jar only derives a path
|
||||||
|
from response_url when the morsel doesn't already carry one.
|
||||||
|
"""
|
||||||
|
cookie = SimpleCookie()
|
||||||
|
cookie[name] = value
|
||||||
|
if domain is None:
|
||||||
|
self.session.cookie_jar.update_cookies(cookie)
|
||||||
|
return
|
||||||
|
if "://" not in domain:
|
||||||
|
domain = "http://" + domain
|
||||||
|
cookie[name]["path"] = path
|
||||||
|
self.session.cookie_jar.update_cookies(cookie, response_url=URL(domain))
|
||||||
|
|
||||||
def clear_cookies(self):
|
def clear_cookies(self):
|
||||||
"""clear the session cookie jar"""
|
"""clear the session cookie jar"""
|
||||||
@@ -295,7 +342,14 @@ class ExtendedSession:
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def request(self, method, url, **kwargs) -> Response:
|
async def request(self, method, url, **kwargs) -> Response:
|
||||||
"""make a request, applying overwrites/domain rewrites/proxy resolution"""
|
"""make a request, applying overwrites/domain rewrites/proxy resolution
|
||||||
|
|
||||||
|
raises the real exception subtype on failure: a total timeout raises
|
||||||
|
aiohttp.ServerTimeoutError, and any other aiohttp.ClientError (e.g.
|
||||||
|
ClientConnectorError, ClientProxyConnectionError, ClientResponseError with
|
||||||
|
.status/.headers, TooManyRedirects) is re-raised as-is, not flattened into
|
||||||
|
the base ClientError — callers can branch by type or read subtype attributes.
|
||||||
|
"""
|
||||||
kwargs["proxy"] = self._get_proxy(url, kwargs.pop("proxies", None))
|
kwargs["proxy"] = self._get_proxy(url, kwargs.pop("proxies", None))
|
||||||
debug = kwargs.pop("debug", False)
|
debug = kwargs.pop("debug", False)
|
||||||
|
|
||||||
@@ -306,6 +360,12 @@ class ExtendedSession:
|
|||||||
timeout = kwargs.get("timeout")
|
timeout = kwargs.get("timeout")
|
||||||
if isinstance(timeout, (int, float)):
|
if isinstance(timeout, (int, float)):
|
||||||
kwargs["timeout"] = aiohttp.ClientTimeout(total=timeout)
|
kwargs["timeout"] = aiohttp.ClientTimeout(total=timeout)
|
||||||
|
elif timeout is None and "timeout" in kwargs:
|
||||||
|
# an explicit timeout=None reaches aiohttp as ClientTimeout(total=None),
|
||||||
|
# which DISABLES the timeout and overrides the session default; drop it so
|
||||||
|
# the session-level timeout applies (matters for request_with_retries, whose
|
||||||
|
# timeout kwarg defaults to None)
|
||||||
|
del kwargs["timeout"]
|
||||||
|
|
||||||
url = self._apply_domain_overwrites(url)
|
url = self._apply_domain_overwrites(url)
|
||||||
if debug:
|
if debug:
|
||||||
@@ -323,7 +383,13 @@ class ExtendedSession:
|
|||||||
# request_with_retries can still label it a timeout
|
# request_with_retries can still label it a timeout
|
||||||
raise aiohttp.ServerTimeoutError(f"timeout for {url}: {error}") from error
|
raise aiohttp.ServerTimeoutError(f"timeout for {url}: {error}") from error
|
||||||
except aiohttp.ClientError as error:
|
except aiohttp.ClientError as error:
|
||||||
raise aiohttp.ClientError(f"client error for {url}: {error}") from error
|
# re-raise the ORIGINAL subtype (ClientConnectorError, ClientProxyConnectionError,
|
||||||
|
# ClientResponseError with .status/.headers, TooManyRedirects, ...) instead of
|
||||||
|
# flattening into the base class — direct callers branching by type or reading
|
||||||
|
# subtype attributes need the real exception; request_with_retries still catches
|
||||||
|
# the base aiohttp.ClientError below and is unaffected
|
||||||
|
log.error("client error for %s: %s", url, error)
|
||||||
|
raise
|
||||||
|
|
||||||
async def request_with_retries(
|
async def request_with_retries(
|
||||||
self, method, url, *, data=None, proxies=None, timeout=None, attempts=None,
|
self, method, url, *, data=None, proxies=None, timeout=None, attempts=None,
|
||||||
@@ -335,6 +401,10 @@ class ExtendedSession:
|
|||||||
returns a Response on success (or non-retryable status), or a falsy
|
returns a Response on success (or non-retryable status), or a falsy
|
||||||
FailureResponse if every attempt fails. backoff is exponential
|
FailureResponse if every attempt fails. backoff is exponential
|
||||||
(backoff_base ** attempt).
|
(backoff_base ** attempt).
|
||||||
|
|
||||||
|
timeout defaults to None, which falls back to the session-level timeout set
|
||||||
|
at construction (aioweb pops a None timeout so it does not reach aiohttp as an
|
||||||
|
infinite ClientTimeout); pass a number to override per call.
|
||||||
"""
|
"""
|
||||||
attempts = attempts or DEFAULT_ATTEMPTS
|
attempts = attempts or DEFAULT_ATTEMPTS
|
||||||
body_data, body_json = _route_body(data)
|
body_data, body_json = _route_body(data)
|
||||||
@@ -384,12 +454,23 @@ class ExtendedSession:
|
|||||||
# lifecycle
|
# lifecycle
|
||||||
|
|
||||||
async def close(self):
|
async def close(self):
|
||||||
"""close the backend session — override if the backend's close differs"""
|
"""close the backend session — override if the backend's close differs
|
||||||
await self.session.close()
|
|
||||||
|
a no-op if the session was never built (lazy construction means a session
|
||||||
|
that made no request and was never otherwise touched has nothing to close).
|
||||||
|
"""
|
||||||
|
if self._session is not None:
|
||||||
|
await self._session.close()
|
||||||
|
|
||||||
def _is_closed(self) -> bool:
|
def _is_closed(self) -> bool:
|
||||||
"""whether the backend session is closed — override for non-aiohttp backends"""
|
"""whether the backend session is closed — override for non-aiohttp backends
|
||||||
return self.session.closed
|
|
||||||
|
an unbuilt (never-lazily-created) session counts as closed: nothing was
|
||||||
|
opened, so there is nothing to leak and __del__ should not warn.
|
||||||
|
"""
|
||||||
|
if self._session is None:
|
||||||
|
return True
|
||||||
|
return self._session.closed
|
||||||
|
|
||||||
async def __aenter__(self):
|
async def __aenter__(self):
|
||||||
return self
|
return self
|
||||||
|
|||||||
Reference in New Issue
Block a user