Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ca23099e06 | ||
|
|
0768d643b1 | ||
|
|
e1ab5d38a0 | ||
|
|
b8cd184c64 | ||
|
|
74ed83cf73 | ||
|
|
14a3ee1456 | ||
|
|
3737af0cf5 | ||
|
|
d3f2bed7fe | ||
|
|
849200985c | ||
|
|
7da06443c8 | ||
|
|
382b8aa632 | ||
|
|
d527174a2b | ||
|
|
bad3ea2677 | ||
|
|
dc3fb70a1e | ||
|
|
7a2f24be9e | ||
|
|
7779d0b050 |
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
# claude
|
# claude
|
||||||
CLAUDE.md
|
.claude/
|
||||||
|
|
||||||
# python
|
# python
|
||||||
__pycache__/
|
__pycache__/
|
||||||
|
|||||||
@@ -11,17 +11,19 @@ and swap the HTTP client while inheriting everything else.
|
|||||||
`requirements.txt`:
|
`requirements.txt`:
|
||||||
|
|
||||||
```
|
```
|
||||||
aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.0
|
aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.9
|
||||||
```
|
```
|
||||||
|
|
||||||
Direct:
|
Direct:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install "aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.0"
|
pip install "aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.9"
|
||||||
```
|
```
|
||||||
|
|
||||||
Requires `aiohttp` and `yarl` (pulled transitively).
|
Requires `aiohttp` and `yarl` (pulled transitively).
|
||||||
|
|
||||||
|
Drop the `@v0.1.9` suffix from the line above to install the latest unpinned.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@@ -63,6 +65,15 @@ resp = await s.request_with_retries(
|
|||||||
|
|
||||||
Returns a `FailureResponse` (falsy) if every attempt fails.
|
Returns a `FailureResponse` (falsy) if every attempt fails.
|
||||||
|
|
||||||
|
`request()` (the non-retrying call) raises on failure: a total timeout raises
|
||||||
|
`aiohttp.ServerTimeoutError`, and any other network/protocol failure raises its real
|
||||||
|
`aiohttp.ClientError` subtype as-is (`ClientConnectorError`, `ClientProxyConnectionError`,
|
||||||
|
`ClientResponseError` with `.status`/`.headers`, `TooManyRedirects`, ...) — it is not
|
||||||
|
flattened into the base `ClientError`, so direct callers can branch by type or read
|
||||||
|
subtype attributes. `request_with_retries` catches the base `aiohttp.ClientError` (and
|
||||||
|
`asyncio.TimeoutError`) across all attempts and returns a falsy `FailureResponse` instead
|
||||||
|
of raising.
|
||||||
|
|
||||||
## Header overwrites & ephemeral headers
|
## Header overwrites & ephemeral headers
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@@ -86,7 +97,10 @@ s.overwrite_domain("internal.local", "127.0.0.1") # host-substring rewrite
|
|||||||
print(s.preview("POST", url, json={"a": 1}).as_curl()) # equivalent cURL command
|
print(s.preview("POST", url, json={"a": 1}).as_curl()) # equivalent cURL command
|
||||||
```
|
```
|
||||||
|
|
||||||
Pass `debug=True` to `request_with_retries` to log the cURL preview and request flow.
|
`as_curl()` renders `params` (merged into the url's query string) and `timeout` (as
|
||||||
|
`--max-time`) as well as headers/body/proxy, so the emitted command is faithful to what
|
||||||
|
`request()` actually sends. Pass `debug=True` to `request_with_retries` to log the cURL
|
||||||
|
preview and request flow.
|
||||||
|
|
||||||
## Custom backends
|
## Custom backends
|
||||||
|
|
||||||
@@ -128,6 +142,81 @@ Two changes can't be shimmed without re-introducing the bugs they fix:
|
|||||||
`await s.close()`; a leaked session emits a `ResourceWarning`. The old finalizer-based
|
`await s.close()`; a leaked session emits a `ResourceWarning`. The old finalizer-based
|
||||||
auto-close was unsafe and was removed.
|
auto-close was unsafe and was removed.
|
||||||
|
|
||||||
|
## Changelog
|
||||||
|
|
||||||
|
### v0.1.9
|
||||||
|
|
||||||
|
- **`_get_proxy()`/`request()` proxy resolution now checks `is None`, not
|
||||||
|
truthiness.** A per-call `proxies={}` previously fell back to the session's
|
||||||
|
configured proxies instead of disabling them for that call. `request()` also no
|
||||||
|
longer clobbers a native `proxy=` kwarg with the resolved session/`proxies=`
|
||||||
|
value (a real IP-unmasking leak) — passing both now raises `ValueError` instead
|
||||||
|
of silently picking one.
|
||||||
|
- **`Response.text(encoding=...)` no longer returns a stale cached decode.** A
|
||||||
|
second call with an explicit `encoding=` previously still returned the first
|
||||||
|
(possibly differently-encoded) cached decode; an explicit encoding now bypasses
|
||||||
|
the cache.
|
||||||
|
- **`preview()` now rewrites the url before resolving cookies**, matching what
|
||||||
|
`request()` actually sends — previously cookies were resolved against the
|
||||||
|
pre-rewrite host, which could miss or misattribute host-bound cookies.
|
||||||
|
- **`request_with_retries(attempts=0)` now floors to 1 attempt**, not silently
|
||||||
|
`DEFAULT_ATTEMPTS` (3). `attempts` is checked with `is None`, not truthiness.
|
||||||
|
|
||||||
|
### v0.1.8
|
||||||
|
|
||||||
|
- **`request()` no longer flattens `aiohttp.ClientError` subtypes.** Every failure
|
||||||
|
(connect errors, proxy errors, `raise_for_status()`-style response errors,
|
||||||
|
redirect limits, ...) was re-raised as a bare `aiohttp.ClientError`, losing the
|
||||||
|
real subtype and its attributes (`.os_error`, `.status`, `.headers`, ...) — a
|
||||||
|
direct caller doing `except ClientProxyConnectionError:` or `if e.status == 401`
|
||||||
|
would silently never match. Now the original exception is re-raised as-is (its
|
||||||
|
subtype, attributes, and `__cause__` all preserved). `request_with_retries`
|
||||||
|
still catches the base `aiohttp.ClientError` across attempts, so its behavior
|
||||||
|
(and its `FailureResponse` return on exhaustion) is unchanged.
|
||||||
|
- **`as_curl()` now renders `params` and `timeout`.** Previously a preview built
|
||||||
|
with `params=` silently omitted the query string (and a `timeout=` omitted
|
||||||
|
`--max-time`), so a `debug=True` cURL replay of a params-driven request hit a
|
||||||
|
different URL than the one actually sent. `params` are now merged into the url's
|
||||||
|
query string (via `yarl`) and `timeout` is emitted as `--max-time`.
|
||||||
|
|
||||||
|
### v0.1.7
|
||||||
|
|
||||||
|
- **`get_cookies()` now returns real cookies.** Previously called `filter_cookies()`
|
||||||
|
with no URL, which only ever returns domain-less shared cookies — every normal
|
||||||
|
domain-bound cookie (including ones set by a real `Set-Cookie` response) was
|
||||||
|
silently omitted. Now iterates the jar directly.
|
||||||
|
- **`set_cookie()` no longer leaks a shared cookie to every host.** A bare hostname
|
||||||
|
(`domain="example.com"`) built a schemeless URL, which aiohttp's jar treats as a
|
||||||
|
domain-less "shared" cookie sent with every request the session makes, including
|
||||||
|
unrelated hosts. A scheme is now added when missing so the cookie is scoped to
|
||||||
|
that host.
|
||||||
|
- **`set_cookie()` now honors `path`** (previously ignored — the cookie always
|
||||||
|
landed at `path="/"`). `domain=None` is unchanged in meaning but now stores a
|
||||||
|
truly shared cookie (sent to every host) instead of one silently bound to
|
||||||
|
`localhost` only, which made `set_cookie(name, value)` (no domain) a silent
|
||||||
|
no-op for any real request.
|
||||||
|
- **The backend session is built lazily**, not in `__init__`. Under aiohttp 3.14,
|
||||||
|
constructing `aiohttp.ClientSession` requires a running event loop; eager
|
||||||
|
construction crashed the common host pattern of attaching a session before the
|
||||||
|
loop starts (e.g. `bot.http = ExtendedSession(...)` in `Bot.__init__`). The
|
||||||
|
session (and any subclass's `_create_session` override) now builds on first
|
||||||
|
access instead.
|
||||||
|
|
||||||
|
### v0.1.2
|
||||||
|
|
||||||
|
- Pinned `commons` to v0.2.1 (retry `attempts` floor fix).
|
||||||
|
|
||||||
|
### v0.1.1
|
||||||
|
|
||||||
|
- **JSON list bodies** now route to `json=` (were wrongly form-encoded via `data=` —
|
||||||
|
only dicts went to `json=` before).
|
||||||
|
- **Exhausted retries return the real last response.** When every attempt hit a
|
||||||
|
retryable status (429/5xx), the loop discarded it and returned a synthetic
|
||||||
|
`FailureResponse` (status 0); now the real last 4xx/5xx `Response` is returned (only a
|
||||||
|
pure-exception failure yields `FailureResponse`).
|
||||||
|
- Retry/backoff moved onto `commons.aretry` (shared engine); backoff schedule unchanged.
|
||||||
|
Adds a `commons` dependency.
|
||||||
|
|
||||||
## Versioning
|
## Versioning
|
||||||
|
|
||||||
Tagged `vX.Y.Z`. Pin the tag in `requirements.txt`.
|
Releases are tagged `vX.Y.Z`. The install line above pins a release; drop the `@vX.Y.Z` suffix to install the latest unpinned. Pin deliberately for reproducible installs.
|
||||||
|
|||||||
+5
-1
@@ -4,13 +4,17 @@ build-backend = "hatchling.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "aioweb"
|
name = "aioweb"
|
||||||
version = "0.1.0"
|
version = "0.1.9"
|
||||||
description = "Async HTTP session wrapper over aiohttp — proxies, header overwrites, retries, previews. Config-free, installable."
|
description = "Async HTTP session wrapper over aiohttp — proxies, header overwrites, retries, previews. Config-free, installable."
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aiohttp>=3.9",
|
"aiohttp>=3.9",
|
||||||
"yarl>=1.9",
|
"yarl>=1.9",
|
||||||
|
"commons @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.2.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[tool.hatch.metadata]
|
||||||
|
allow-direct-references = true
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
[tool.hatch.build.targets.wheel]
|
||||||
packages = ["src/aioweb"]
|
packages = ["src/aioweb"]
|
||||||
|
|||||||
+24
-9
@@ -3,6 +3,9 @@ request preview for aioweb — format or export a request without sending it
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import json as _json
|
import json as _json
|
||||||
|
import shlex
|
||||||
|
|
||||||
|
from yarl import URL
|
||||||
|
|
||||||
|
|
||||||
class RequestPreview:
|
class RequestPreview:
|
||||||
@@ -25,15 +28,27 @@ class RequestPreview:
|
|||||||
return "\n".join(f"{key}: {value}" for key, value in self.details.items())
|
return "\n".join(f"{key}: {value}" for key, value in self.details.items())
|
||||||
|
|
||||||
def as_curl(self):
|
def as_curl(self):
|
||||||
"""equivalent cURL command for the request"""
|
"""equivalent cURL command for the request
|
||||||
parts = [f"curl -X {self.details['method']}"]
|
|
||||||
|
every interpolated value is shlex.quote'd (non-injectable even with quotes/
|
||||||
|
spaces/metacharacters). `params` is merged into the url's query string and
|
||||||
|
`timeout` rendered as `--max-time`, so the command matches what request()
|
||||||
|
actually sends.
|
||||||
|
"""
|
||||||
|
parts = [f"curl -X {shlex.quote(self.details['method'])}"]
|
||||||
for header, value in (self.details["headers"] or {}).items():
|
for header, value in (self.details["headers"] or {}).items():
|
||||||
parts.append(f"-H '{header}: {value}'")
|
parts.append(f"-H {shlex.quote(f'{header}: {value}')}")
|
||||||
if self.details["data"]:
|
if self.details["data"] is not None:
|
||||||
parts.append(f"--data '{self.details['data']}'")
|
parts.append(f"--data {shlex.quote(str(self.details['data']))}")
|
||||||
elif self.details["json"]:
|
elif self.details["json"] is not None:
|
||||||
parts.append(f"--data '{_json.dumps(self.details['json'])}'")
|
# is-not-None: an empty-but-valid body ({} / []) must still render
|
||||||
parts.append(f"'{self.details['url']}'")
|
parts.append(f"--data {shlex.quote(_json.dumps(self.details['json']))}")
|
||||||
|
url = self.details["url"]
|
||||||
|
if self.details["params"]:
|
||||||
|
url = str(URL(url).update_query(self.details["params"]))
|
||||||
|
parts.append(shlex.quote(str(url)))
|
||||||
if self.details["proxy"]:
|
if self.details["proxy"]:
|
||||||
parts.append(f"--proxy '{self.details['proxy']}'")
|
parts.append(f"--proxy {shlex.quote(str(self.details['proxy']))}")
|
||||||
|
if self.details["timeout"] is not None:
|
||||||
|
parts.append(f"--max-time {shlex.quote(str(self.details['timeout']))}")
|
||||||
return " \\\n ".join(parts)
|
return " \\\n ".join(parts)
|
||||||
|
|||||||
+11
-9
@@ -1,10 +1,8 @@
|
|||||||
"""
|
"""
|
||||||
backend-agnostic response objects for aioweb
|
backend-agnostic response objects for aioweb — Response is built from primitives
|
||||||
|
(status, headers, content, url, history), not a raw aiohttp object, so any backend can
|
||||||
Response is built from primitives (status, headers, content, url, history) rather
|
produce one; FailureResponse mirrors the same surface (every status/predicate a
|
||||||
than holding a raw aiohttp object, so any backend can produce one. FailureResponse
|
property on both) so callers can branch uniformly.
|
||||||
mirrors the same surface so callers can branch uniformly — every status/predicate is
|
|
||||||
a property on both.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json as _json
|
import json as _json
|
||||||
@@ -83,16 +81,20 @@ class Response:
|
|||||||
return self._content
|
return self._content
|
||||||
|
|
||||||
def text(self, encoding: Optional[str] = None) -> str:
|
def text(self, encoding: Optional[str] = None) -> str:
|
||||||
"""decoded text content (cached)"""
|
"""decoded text content, cached only for the default encoding"""
|
||||||
|
if encoding is not None:
|
||||||
|
return self._content.decode(encoding)
|
||||||
if self._text is None:
|
if self._text is None:
|
||||||
self._text = self._content.decode(encoding or self._encoding)
|
self._text = self._content.decode(self._encoding)
|
||||||
return self._text
|
return self._text
|
||||||
|
|
||||||
def json(self):
|
def json(self):
|
||||||
"""parsed JSON content, or None if not valid JSON"""
|
"""parsed JSON content, or None if not valid JSON"""
|
||||||
try:
|
try:
|
||||||
return _json.loads(self.text())
|
return _json.loads(self.text())
|
||||||
except _json.JSONDecodeError:
|
except (_json.JSONDecodeError, UnicodeDecodeError):
|
||||||
|
# text() decodes the body and can raise UnicodeDecodeError on a non-UTF-8
|
||||||
|
# payload — that's a "not valid JSON" outcome, not an error to propagate
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def raise_for_status(self):
|
def raise_for_status(self):
|
||||||
|
|||||||
+173
-75
@@ -1,32 +1,52 @@
|
|||||||
"""
|
"""
|
||||||
async HTTP session wrapper over aiohttp
|
async HTTP session wrapper over aiohttp — proxies, header overwrites, ephemeral
|
||||||
|
headers, domain rewriting, previews, retry/backoff; byte-sending is isolated in
|
||||||
ExtendedSession adds session-level proxies, header overwrites, ephemeral
|
_raw_request() so a subclass can swap backends and inherit everything else.
|
||||||
(per-request generated) headers, domain rewriting, request previews, and
|
|
||||||
retry/backoff on top of aiohttp. The actual byte-sending is isolated in
|
|
||||||
_raw_request() so a different backend (e.g. a TLS-fingerprinting client) can
|
|
||||||
subclass and override just that one method, inheriting everything else.
|
|
||||||
|
|
||||||
async with ExtendedSession(proxies={"https": "http://..."}) as s:
|
async with ExtendedSession(proxies={"https": "http://..."}) as s:
|
||||||
resp = await s.request_with_retries("GET", url)
|
resp = await s.request_with_retries("GET", url)
|
||||||
if resp: # FailureResponse is falsy
|
if resp: # FailureResponse is falsy
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
|
|
||||||
config-free: proxies/headers/timeouts are passed at construction or per call.
|
config-free (proxies/headers/timeouts passed at construction or per call). the
|
||||||
sessions must be closed explicitly (async with, or await s.close()); there is no
|
backend session is built lazily on first use, not in __init__, so construction is
|
||||||
__del__ auto-close (that pattern is unsafe for async resources).
|
safe before an event loop is running (e.g. bot.http = ExtendedSession(...)).
|
||||||
|
sessions must be closed explicitly (async with, or await s.close()) — no __del__
|
||||||
|
auto-close (unsafe for async resources).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
import warnings
|
import warnings
|
||||||
|
from http.cookies import SimpleCookie
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from yarl import URL
|
from yarl import URL
|
||||||
|
from commons import aretry
|
||||||
|
|
||||||
from .preview import RequestPreview
|
from .preview import RequestPreview
|
||||||
from .responses import Response, FailureResponse
|
from .responses import Response, FailureResponse
|
||||||
|
|
||||||
|
|
||||||
|
def _route_body(data):
|
||||||
|
"""split a body into (data=, json=) kwargs; dict OR list routes to json=, else data="""
|
||||||
|
if isinstance(data, (dict, list)):
|
||||||
|
return None, data
|
||||||
|
return data, None
|
||||||
|
|
||||||
|
|
||||||
|
class _RetryStatus(Exception):
|
||||||
|
"""internal signal: a retryable HTTP status; carries the real Response
|
||||||
|
|
||||||
|
raised inside an attempt so commons.aretry drives the backoff + cap; the caller
|
||||||
|
catches the final one to return the REAL last response, not a synthetic failure.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, response):
|
||||||
|
super().__init__(f"retryable status {response.status_code}")
|
||||||
|
self.response = response
|
||||||
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
DEFAULT_ATTEMPTS = 3
|
DEFAULT_ATTEMPTS = 3
|
||||||
@@ -58,21 +78,38 @@ class ExtendedSession:
|
|||||||
self.domain_overwrites = domain_overwrites or {}
|
self.domain_overwrites = domain_overwrites or {}
|
||||||
self.ephemeral_headers = {}
|
self.ephemeral_headers = {}
|
||||||
self.proxies = proxies or {}
|
self.proxies = proxies or {}
|
||||||
# track our own default headers instead of touching aiohttp privates
|
# own header layer, not aiohttp privates, so update_headers/clear_headers work
|
||||||
self._default_headers = dict(headers or {})
|
self._default_headers = dict(headers or {})
|
||||||
self.session = self._create_session(self._default_headers, timeout, **kwargs)
|
self._session_timeout = timeout
|
||||||
|
self._session_kwargs = kwargs
|
||||||
|
# aiohttp>=3.14 requires a running loop to build ClientSession; build lazily
|
||||||
|
# (via `session` / _ensure_session()) so construction before the loop starts
|
||||||
|
# (e.g. bot.http = ExtendedSession(...) in Bot.__init__) doesn't crash
|
||||||
|
self._session = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def session(self):
|
||||||
|
"""the backend session, built lazily on first access (needs a running loop)"""
|
||||||
|
self._ensure_session()
|
||||||
|
return self._session
|
||||||
|
|
||||||
|
def _ensure_session(self):
|
||||||
|
"""build the backend session on first use — idempotent"""
|
||||||
|
if self._session is None:
|
||||||
|
self._session = self._create_session(
|
||||||
|
self._default_headers, self._session_timeout, **self._session_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
def _create_session(self, headers, timeout, **kwargs):
|
def _create_session(self, headers, timeout, **kwargs):
|
||||||
"""create the backend HTTP session — override to use a different client
|
"""create the backend HTTP session — override to swap HTTP clients
|
||||||
|
|
||||||
a subclass swapping the HTTP backend (e.g. a TLS-fingerprinting client)
|
overwrite/domain/proxy/retry/preview logic never touches the session object
|
||||||
overrides this to return its own session object. the overwrite/domain/
|
directly (only _raw_request, cookies, close do), so those work unchanged on
|
||||||
proxy/retry/preview logic in this class never touches the session object
|
any backend. `headers` isn't baked in here — aiohttp would copy it into an
|
||||||
directly (only _raw_request, the cookie methods, and close do), so those
|
immutable map update_headers/clear_headers can't touch; `_default_headers`
|
||||||
features work unchanged on any backend.
|
is the mutable layer request()/preview() merge per call instead.
|
||||||
"""
|
"""
|
||||||
return aiohttp.ClientSession(
|
return aiohttp.ClientSession(
|
||||||
headers=headers,
|
|
||||||
timeout=aiohttp.ClientTimeout(
|
timeout=aiohttp.ClientTimeout(
|
||||||
total=timeout,
|
total=timeout,
|
||||||
connect=timeout / 2,
|
connect=timeout / 2,
|
||||||
@@ -92,10 +129,12 @@ class ExtendedSession:
|
|||||||
def _apply_overwrites(self, request_headers):
|
def _apply_overwrites(self, request_headers):
|
||||||
"""apply static overwrites and ephemeral headers to a request's headers"""
|
"""apply static overwrites and ephemeral headers to a request's headers"""
|
||||||
request_headers = dict(request_headers or {})
|
request_headers = dict(request_headers or {})
|
||||||
for header, value in self.header_overwrites.items():
|
# list()-snapshot so a concurrent mutation of the shared dicts can't raise
|
||||||
|
# "dict changed size during iteration"
|
||||||
|
for header, value in list(self.header_overwrites.items()):
|
||||||
if self.inject or header in request_headers:
|
if self.inject or header in request_headers:
|
||||||
request_headers[header] = value
|
request_headers[header] = value
|
||||||
for header, value_callable in self.ephemeral_headers.items():
|
for header, value_callable in list(self.ephemeral_headers.items()):
|
||||||
if self.inject or header in request_headers:
|
if self.inject or header in request_headers:
|
||||||
value = value_callable()
|
value = value_callable()
|
||||||
if isinstance(value, dict):
|
if isinstance(value, dict):
|
||||||
@@ -173,8 +212,13 @@ class ExtendedSession:
|
|||||||
self.proxies.clear()
|
self.proxies.clear()
|
||||||
|
|
||||||
def _get_proxy(self, url, proxies=None):
|
def _get_proxy(self, url, proxies=None):
|
||||||
"""resolve the proxy for a url's scheme"""
|
"""resolve the proxy for a url's scheme
|
||||||
proxies = proxies or self.proxies
|
|
||||||
|
proxies is checked with `is None`, not truthiness, so an explicit
|
||||||
|
proxies={} disables session proxies for that one call instead of
|
||||||
|
falling back to them.
|
||||||
|
"""
|
||||||
|
proxies = self.proxies if proxies is None else proxies
|
||||||
scheme = url.split("://")[0]
|
scheme = url.split("://")[0]
|
||||||
return proxies.get(scheme)
|
return proxies.get(scheme)
|
||||||
|
|
||||||
@@ -182,36 +226,53 @@ class ExtendedSession:
|
|||||||
# cookies
|
# cookies
|
||||||
|
|
||||||
def get_cookies(self):
|
def get_cookies(self):
|
||||||
"""cookies stored in the session jar"""
|
"""all cookies in the session jar, regardless of domain binding
|
||||||
return self.session.cookie_jar.filter_cookies()
|
|
||||||
|
iterates the jar directly rather than filter_cookies() (which needs a url and,
|
||||||
|
given none, returns only domain-less shared cookies — {} for any normal one).
|
||||||
|
"""
|
||||||
|
return {c.key: c.value for c in self.session.cookie_jar}
|
||||||
|
|
||||||
def set_cookie(self, name, value, domain=None, path="/"):
|
def set_cookie(self, name, value, domain=None, path="/"):
|
||||||
"""set a cookie in the session jar"""
|
"""set a cookie in the session jar
|
||||||
response_url = URL(domain or "http://localhost")
|
|
||||||
self.session.cookie_jar.update_cookies({name: value}, response_url=response_url)
|
domain=None stores a truly shared cookie sent to every host. domain='example.com'
|
||||||
|
(scheme optional, defaulted to http://) scopes it to that host — a bare hostname
|
||||||
|
is normalized into a URL so the jar binds by host instead of silently storing
|
||||||
|
another domain-less shared cookie. `path` is honored via the morsel itself.
|
||||||
|
"""
|
||||||
|
cookie = SimpleCookie()
|
||||||
|
cookie[name] = value
|
||||||
|
if domain is None:
|
||||||
|
self.session.cookie_jar.update_cookies(cookie)
|
||||||
|
return
|
||||||
|
if "://" not in domain:
|
||||||
|
domain = "http://" + domain
|
||||||
|
cookie[name]["path"] = path
|
||||||
|
self.session.cookie_jar.update_cookies(cookie, response_url=URL(domain))
|
||||||
|
|
||||||
def clear_cookies(self):
|
def clear_cookies(self):
|
||||||
"""clear the session cookie jar"""
|
"""clear the session cookie jar"""
|
||||||
self.session.cookie_jar.clear()
|
self.session.cookie_jar.clear()
|
||||||
|
|
||||||
def _cookies_for_url(self, url):
|
def _cookies_for_url(self, url):
|
||||||
"""dict of cookies the backend would send for url — override per backend
|
"""dict of cookies the backend would send for url — override per backend (used by preview())"""
|
||||||
|
|
||||||
used by preview(). non-aiohttp backends override this (or return {}); the
|
|
||||||
rest of preview (domain rewrites, header overwrites) is backend-agnostic.
|
|
||||||
"""
|
|
||||||
return {k: v.value for k, v in self.session.cookie_jar.filter_cookies(URL(url)).items()}
|
return {k: v.value for k, v in self.session.cookie_jar.filter_cookies(URL(url)).items()}
|
||||||
|
|
||||||
# -------------------------------------------------------------------------
|
# -------------------------------------------------------------------------
|
||||||
# preview
|
# preview
|
||||||
|
|
||||||
def preview(self, method, url, **kwargs):
|
def preview(self, method, url, **kwargs):
|
||||||
"""build a RequestPreview for a request without sending it"""
|
"""build a RequestPreview for a request without sending it
|
||||||
|
|
||||||
|
url is domain-rewritten before cookies are resolved, matching the real
|
||||||
|
request — cookie binding is host-based, so resolving against the pre-rewrite
|
||||||
|
host could miss or misattribute cookies.
|
||||||
|
"""
|
||||||
|
url = self._apply_domain_overwrites(url)
|
||||||
proxy = self._get_proxy(url, kwargs.pop("proxies", None))
|
proxy = self._get_proxy(url, kwargs.pop("proxies", None))
|
||||||
if kwargs.get("headers"):
|
merged = {**self._default_headers, **(kwargs.pop("headers", None) or {})}
|
||||||
headers = self._apply_overwrites(kwargs.pop("headers"))
|
headers = self._apply_overwrites(merged)
|
||||||
else:
|
|
||||||
headers = dict(self.get_headers())
|
|
||||||
|
|
||||||
timeout = kwargs.get("timeout")
|
timeout = kwargs.get("timeout")
|
||||||
timeout_total = timeout if isinstance(timeout, (int, float)) else None
|
timeout_total = timeout if isinstance(timeout, (int, float)) else None
|
||||||
@@ -227,7 +288,7 @@ class ExtendedSession:
|
|||||||
|
|
||||||
return RequestPreview(
|
return RequestPreview(
|
||||||
method=method,
|
method=method,
|
||||||
url=self._apply_domain_overwrites(url),
|
url=url,
|
||||||
headers=headers,
|
headers=headers,
|
||||||
proxy=proxy,
|
proxy=proxy,
|
||||||
data=kwargs.get("data"),
|
data=kwargs.get("data"),
|
||||||
@@ -242,10 +303,9 @@ class ExtendedSession:
|
|||||||
async def _raw_request(self, method, url, **kwargs) -> Response:
|
async def _raw_request(self, method, url, **kwargs) -> Response:
|
||||||
"""send one request with aiohttp and adapt it into a Response
|
"""send one request with aiohttp and adapt it into a Response
|
||||||
|
|
||||||
this is the backend seam: a subclass can override only this method (using a
|
the backend seam: a subclass overrides only this to swap HTTP clients,
|
||||||
different HTTP client, e.g. a TLS-fingerprinting one), building a Response
|
building a Response from that backend's primitives; everything else
|
||||||
from that backend's primitives. everything else (overwrites, retries,
|
(overwrites, retries, preview) is inherited.
|
||||||
preview) is inherited.
|
|
||||||
"""
|
"""
|
||||||
response = await self.session.request(method, url, **kwargs)
|
response = await self.session.request(method, url, **kwargs)
|
||||||
async with response:
|
async with response:
|
||||||
@@ -262,16 +322,32 @@ class ExtendedSession:
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def request(self, method, url, **kwargs) -> Response:
|
async def request(self, method, url, **kwargs) -> Response:
|
||||||
"""make a request, applying overwrites/domain rewrites/proxy resolution"""
|
"""make a request, applying overwrites/domain rewrites/proxy resolution
|
||||||
kwargs["proxy"] = self._get_proxy(url, kwargs.pop("proxies", None))
|
|
||||||
|
raises the real exception subtype on failure (ServerTimeoutError on a total
|
||||||
|
timeout; any other aiohttp.ClientError re-raised as-is, not flattened, so
|
||||||
|
callers can branch by type or read subtype attributes like .status/.headers).
|
||||||
|
a native proxy= is never silently clobbered by resolved proxies= (that would
|
||||||
|
unmask the caller's real IP) — pass only one of them.
|
||||||
|
"""
|
||||||
|
resolved_proxy = self._get_proxy(url, kwargs.pop("proxies", None))
|
||||||
|
if "proxy" in kwargs and kwargs["proxy"] is not None and resolved_proxy is not None:
|
||||||
|
raise ValueError("pass only one of proxy= or proxies= (session/per-call), not both")
|
||||||
|
kwargs.setdefault("proxy", resolved_proxy)
|
||||||
debug = kwargs.pop("debug", False)
|
debug = kwargs.pop("debug", False)
|
||||||
|
|
||||||
kwargs["headers"] = self._apply_overwrites(kwargs.get("headers"))
|
merged = {**self._default_headers, **(kwargs.get("headers") or {})}
|
||||||
|
kwargs["headers"] = self._apply_overwrites(merged)
|
||||||
kwargs["headers"] = {str(k): str(v) for k, v in kwargs["headers"].items()}
|
kwargs["headers"] = {str(k): str(v) for k, v in kwargs["headers"].items()}
|
||||||
|
|
||||||
timeout = kwargs.get("timeout")
|
timeout = kwargs.get("timeout")
|
||||||
if isinstance(timeout, (int, float)):
|
if isinstance(timeout, (int, float)):
|
||||||
kwargs["timeout"] = aiohttp.ClientTimeout(total=timeout)
|
kwargs["timeout"] = aiohttp.ClientTimeout(total=timeout)
|
||||||
|
elif timeout is None and "timeout" in kwargs:
|
||||||
|
# explicit timeout=None would reach aiohttp as ClientTimeout(total=None) —
|
||||||
|
# infinite, disabling the session default — so drop it instead (matters for
|
||||||
|
# request_with_retries, whose timeout kwarg defaults to None)
|
||||||
|
del kwargs["timeout"]
|
||||||
|
|
||||||
url = self._apply_domain_overwrites(url)
|
url = self._apply_domain_overwrites(url)
|
||||||
if debug:
|
if debug:
|
||||||
@@ -282,8 +358,16 @@ class ExtendedSession:
|
|||||||
if debug and result.redirect_chain:
|
if debug and result.redirect_chain:
|
||||||
log.info("redirect chain: %s", result.redirect_chain)
|
log.info("redirect chain: %s", result.redirect_chain)
|
||||||
return result
|
return result
|
||||||
|
except asyncio.TimeoutError as error:
|
||||||
|
# a bare asyncio.TimeoutError isn't an aiohttp.ClientError subclass — wrap it
|
||||||
|
# as ServerTimeoutError (both a ClientError AND a TimeoutError) so callers get
|
||||||
|
# a typed failure and request_with_retries can still label it a timeout
|
||||||
|
raise aiohttp.ServerTimeoutError(f"timeout for {url}: {error}") from error
|
||||||
except aiohttp.ClientError as error:
|
except aiohttp.ClientError as error:
|
||||||
raise aiohttp.ClientError(f"client error for {url}: {error}") from error
|
# re-raise the original subtype (not flattened) — request_with_retries still
|
||||||
|
# catches the base aiohttp.ClientError below and is unaffected
|
||||||
|
log.error("client error for %s: %s", url, error)
|
||||||
|
raise
|
||||||
|
|
||||||
async def request_with_retries(
|
async def request_with_retries(
|
||||||
self, method, url, *, data=None, proxies=None, timeout=None, attempts=None,
|
self, method, url, *, data=None, proxies=None, timeout=None, attempts=None,
|
||||||
@@ -294,56 +378,70 @@ class ExtendedSession:
|
|||||||
|
|
||||||
returns a Response on success (or non-retryable status), or a falsy
|
returns a Response on success (or non-retryable status), or a falsy
|
||||||
FailureResponse if every attempt fails. backoff is exponential
|
FailureResponse if every attempt fails. backoff is exponential
|
||||||
(backoff_base ** attempt).
|
(backoff_base ** attempt). timeout=None falls back to the session-level
|
||||||
|
timeout (see request()'s note on why explicit None is dropped, not passed
|
||||||
|
through). attempts=0 floors to 1 (via commons.aretry), not DEFAULT_ATTEMPTS.
|
||||||
"""
|
"""
|
||||||
attempts = attempts or DEFAULT_ATTEMPTS
|
attempts = DEFAULT_ATTEMPTS if attempts is None else attempts
|
||||||
last_error = None
|
body_data, body_json = _route_body(data)
|
||||||
|
|
||||||
if debug:
|
if debug:
|
||||||
preview = self.preview(
|
preview = self.preview(
|
||||||
method=method, url=url, params=params,
|
method=method, url=url, params=params,
|
||||||
data=None if isinstance(data, dict) else data,
|
data=body_data, json=body_json,
|
||||||
json=data if isinstance(data, dict) else None,
|
|
||||||
headers=headers, proxies=proxies, timeout=timeout,
|
headers=headers, proxies=proxies, timeout=timeout,
|
||||||
).as_curl()
|
).as_curl()
|
||||||
log.info("[aioweb.debug]\n%s\nproxies: %s inject: %s", preview, self.proxies, self.inject)
|
log.info("[aioweb.debug]\n%s\nproxies: %s inject: %s", preview, self.proxies, self.inject)
|
||||||
|
|
||||||
for attempt in range(attempts):
|
async def attempt():
|
||||||
try:
|
|
||||||
response = await self.request(
|
response = await self.request(
|
||||||
method=method, url=url, params=params,
|
method=method, url=url, params=params,
|
||||||
data=None if isinstance(data, dict) else data,
|
data=body_data, json=body_json,
|
||||||
json=data if isinstance(data, dict) else None,
|
|
||||||
headers=headers, proxies=proxies, timeout=timeout, debug=debug,
|
headers=headers, proxies=proxies, timeout=timeout, debug=debug,
|
||||||
)
|
)
|
||||||
if response.status_code in retry_statuses:
|
if response.status_code in retry_statuses:
|
||||||
last_error = f"retryable status {response.status_code}"
|
log.warning("retryable status %s for %s", response.status_code, url)
|
||||||
log.warning("attempt %d: %s for %s", attempt + 1, last_error, url)
|
raise _RetryStatus(response)
|
||||||
else:
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
try:
|
||||||
|
return await aretry(
|
||||||
|
attempt, attempts=attempts, backoff=1.0, factor=backoff_base,
|
||||||
|
jitter=False, on=(Exception,),
|
||||||
|
)
|
||||||
|
except _RetryStatus as exhausted:
|
||||||
|
log.error("all %d attempts failed for %s (last status %s)",
|
||||||
|
attempts, url, exhausted.response.status_code)
|
||||||
|
return exhausted.response
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
# request() wraps timeouts as ServerTimeoutError (ClientError + TimeoutError);
|
||||||
|
# catch it first so it's labeled a timeout, not a generic client error below
|
||||||
|
log.error("all %d attempts timed out for %s", attempts, url)
|
||||||
|
return FailureResponse(reason="timeout", url=url)
|
||||||
except aiohttp.ClientError as error:
|
except aiohttp.ClientError as error:
|
||||||
last_error = f"client error: {error}"
|
log.error("all %d attempts failed for %s (client error: %s)", attempts, url, error)
|
||||||
log.warning("attempt %d: %s, retrying", attempt + 1, last_error)
|
return FailureResponse(reason=f"client error: {error}", url=url)
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
last_error = f"unexpected error: {error}"
|
log.error("all %d attempts failed for %s (unexpected: %s)", attempts, url, error)
|
||||||
log.exception("attempt %d: %s, retrying", attempt + 1, last_error)
|
return FailureResponse(reason=f"unexpected error: {error}", url=url)
|
||||||
|
|
||||||
if attempt < attempts - 1:
|
|
||||||
await asyncio.sleep(backoff_base ** attempt)
|
|
||||||
|
|
||||||
log.error("all %d attempts failed for %s", attempts, url)
|
|
||||||
return FailureResponse(reason=last_error, url=url)
|
|
||||||
|
|
||||||
# -------------------------------------------------------------------------
|
# -------------------------------------------------------------------------
|
||||||
# lifecycle
|
# lifecycle
|
||||||
|
|
||||||
async def close(self):
|
async def close(self):
|
||||||
"""close the backend session — override if the backend's close differs"""
|
"""close the backend session — override if the backend's close differs; no-op if never built"""
|
||||||
await self.session.close()
|
if self._session is not None:
|
||||||
|
await self._session.close()
|
||||||
|
|
||||||
def _is_closed(self) -> bool:
|
def _is_closed(self) -> bool:
|
||||||
"""whether the backend session is closed — override for non-aiohttp backends"""
|
"""whether the backend session is closed — override for non-aiohttp backends
|
||||||
return self.session.closed
|
|
||||||
|
an unbuilt session counts as closed: nothing was opened, nothing to leak,
|
||||||
|
__del__ should not warn.
|
||||||
|
"""
|
||||||
|
if self._session is None:
|
||||||
|
return True
|
||||||
|
return self._session.closed
|
||||||
|
|
||||||
async def __aenter__(self):
|
async def __aenter__(self):
|
||||||
return self
|
return self
|
||||||
@@ -352,8 +450,8 @@ class ExtendedSession:
|
|||||||
await self.close()
|
await self.close()
|
||||||
|
|
||||||
def __del__(self):
|
def __del__(self):
|
||||||
# do NOT attempt async cleanup here — spinning event loops in a finalizer
|
# no async cleanup here — spinning event loops in a finalizer is unsafe;
|
||||||
# is unsafe. just warn so the leak is visible; callers must close explicitly.
|
# just warn so the leak is visible, callers must close explicitly
|
||||||
try:
|
try:
|
||||||
closed = self._is_closed()
|
closed = self._is_closed()
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
Reference in New Issue
Block a user