Compare commits

..

No commits in common. "main" and "v0.1.0" have entirely different histories.
main ... v0.1.0

6 changed files with 50 additions and 129 deletions

2
.gitignore vendored
View File

@ -1,5 +1,5 @@
# claude # claude
.claude/ CLAUDE.md
# python # python
__pycache__/ __pycache__/

View File

@ -11,19 +11,17 @@ and swap the HTTP client while inheriting everything else.
`requirements.txt`: `requirements.txt`:
``` ```
aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.5 aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.0
``` ```
Direct: Direct:
```bash ```bash
pip install "aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.5" pip install "aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.0"
``` ```
Requires `aiohttp` and `yarl` (pulled transitively). Requires `aiohttp` and `yarl` (pulled transitively).
Drop the `@v0.1.5` suffix from the line above to install the latest unpinned.
## Usage ## Usage
```python ```python
@ -130,23 +128,6 @@ Two changes can't be shimmed without re-introducing the bugs they fix:
`await s.close()`; a leaked session emits a `ResourceWarning`. The old finalizer-based `await s.close()`; a leaked session emits a `ResourceWarning`. The old finalizer-based
auto-close was unsafe and was removed. auto-close was unsafe and was removed.
## Changelog
### v0.1.2
- Pinned `commons` to v0.2.1 (retry `attempts` floor fix).
### v0.1.1
- **JSON list bodies** now route to `json=` (were wrongly form-encoded via `data=`
only dicts went to `json=` before).
- **Exhausted retries return the real last response.** When every attempt hit a
retryable status (429/5xx), the loop discarded it and returned a synthetic
`FailureResponse` (status 0); now the real last 4xx/5xx `Response` is returned (only a
pure-exception failure yields `FailureResponse`).
- Retry/backoff moved onto `commons.aretry` (shared engine); backoff schedule unchanged.
Adds a `commons` dependency.
## Versioning ## Versioning
Releases are tagged `vX.Y.Z`. The install line above pins a release; drop the `@vX.Y.Z` suffix to install the latest unpinned. Pin deliberately for reproducible installs. Tagged `vX.Y.Z`. Pin the tag in `requirements.txt`.

View File

@ -4,17 +4,13 @@ build-backend = "hatchling.build"
[project] [project]
name = "aioweb" name = "aioweb"
version = "0.1.5" version = "0.1.0"
description = "Async HTTP session wrapper over aiohttp — proxies, header overwrites, retries, previews. Config-free, installable." description = "Async HTTP session wrapper over aiohttp — proxies, header overwrites, retries, previews. Config-free, installable."
requires-python = ">=3.10" requires-python = ">=3.10"
dependencies = [ dependencies = [
"aiohttp>=3.9", "aiohttp>=3.9",
"yarl>=1.9", "yarl>=1.9",
"commons @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.2.1",
] ]
[tool.hatch.metadata]
allow-direct-references = true
[tool.hatch.build.targets.wheel] [tool.hatch.build.targets.wheel]
packages = ["src/aioweb"] packages = ["src/aioweb"]

View File

@ -3,7 +3,6 @@ request preview for aioweb — format or export a request without sending it
""" """
import json as _json import json as _json
import shlex
class RequestPreview: class RequestPreview:
@ -26,22 +25,15 @@ class RequestPreview:
return "\n".join(f"{key}: {value}" for key, value in self.details.items()) return "\n".join(f"{key}: {value}" for key, value in self.details.items())
def as_curl(self): def as_curl(self):
"""equivalent cURL command for the request """equivalent cURL command for the request"""
parts = [f"curl -X {self.details['method']}"]
every interpolated value is shell-quoted with shlex.quote, so headers,
body, url, or proxy containing quotes/spaces/metacharacters produce a
valid, non-injectable command rather than a broken or unsafe one.
"""
parts = [f"curl -X {shlex.quote(self.details['method'])}"]
for header, value in (self.details["headers"] or {}).items(): for header, value in (self.details["headers"] or {}).items():
parts.append(f"-H {shlex.quote(f'{header}: {value}')}") parts.append(f"-H '{header}: {value}'")
if self.details["data"] is not None: if self.details["data"]:
parts.append(f"--data {shlex.quote(str(self.details['data']))}") parts.append(f"--data '{self.details['data']}'")
elif self.details["json"] is not None: elif self.details["json"]:
# is-not-None, not truthiness: an empty-but-valid body ({} / []) must still parts.append(f"--data '{_json.dumps(self.details['json'])}'")
# render rather than being dropped as falsy parts.append(f"'{self.details['url']}'")
parts.append(f"--data {shlex.quote(_json.dumps(self.details['json']))}")
parts.append(shlex.quote(str(self.details["url"])))
if self.details["proxy"]: if self.details["proxy"]:
parts.append(f"--proxy {shlex.quote(str(self.details['proxy']))}") parts.append(f"--proxy '{self.details['proxy']}'")
return " \\\n ".join(parts) return " \\\n ".join(parts)

View File

@ -92,9 +92,7 @@ class Response:
"""parsed JSON content, or None if not valid JSON""" """parsed JSON content, or None if not valid JSON"""
try: try:
return _json.loads(self.text()) return _json.loads(self.text())
except (_json.JSONDecodeError, UnicodeDecodeError): except _json.JSONDecodeError:
# text() decodes the body and can raise UnicodeDecodeError on a non-UTF-8
# payload — that's a "not valid JSON" outcome, not an error to propagate
return None return None
def raise_for_status(self): def raise_for_status(self):

View File

@ -23,36 +23,10 @@ import warnings
import aiohttp import aiohttp
from yarl import URL from yarl import URL
from commons import aretry
from .preview import RequestPreview from .preview import RequestPreview
from .responses import Response, FailureResponse from .responses import Response, FailureResponse
def _route_body(data):
"""split a body into (data=, json=) kwargs
dict OR list bodies are valid JSON and route to json=; everything else
(str/bytes/form) routes to data=. previously only dicts went to json=, so a
JSON list was wrongly form-encoded.
"""
if isinstance(data, (dict, list)):
return None, data
return data, None
class _RetryStatus(Exception):
"""internal signal: a retryable HTTP status; carries the real Response
raised inside an attempt so commons.aretry drives the backoff + cap; the caller
catches the final one to return the REAL last response, not a synthetic failure.
"""
def __init__(self, response):
super().__init__(f"retryable status {response.status_code}")
self.response = response
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
DEFAULT_ATTEMPTS = 3 DEFAULT_ATTEMPTS = 3
@ -96,15 +70,9 @@ class ExtendedSession:
proxy/retry/preview logic in this class never touches the session object proxy/retry/preview logic in this class never touches the session object
directly (only _raw_request, the cookie methods, and close do), so those directly (only _raw_request, the cookie methods, and close do), so those
features work unchanged on any backend. features work unchanged on any backend.
`headers` is the session-default header set; the default aiohttp backend
does NOT bake it into the ClientSession (which would copy it into an
immutable per-session map that update_headers/clear_headers can't touch).
instead `_default_headers` is our own mutable layer that request() and
preview() merge per call, so the mutable session-header API actually works.
a backend that needs the defaults baked at construction may use `headers`.
""" """
return aiohttp.ClientSession( return aiohttp.ClientSession(
headers=headers,
timeout=aiohttp.ClientTimeout( timeout=aiohttp.ClientTimeout(
total=timeout, total=timeout,
connect=timeout / 2, connect=timeout / 2,
@ -124,13 +92,10 @@ class ExtendedSession:
def _apply_overwrites(self, request_headers): def _apply_overwrites(self, request_headers):
"""apply static overwrites and ephemeral headers to a request's headers""" """apply static overwrites and ephemeral headers to a request's headers"""
request_headers = dict(request_headers or {}) request_headers = dict(request_headers or {})
# snapshot the shared override dicts so a concurrent mutation (e.g. a command for header, value in self.header_overwrites.items():
# editing ephemerals on a shared session) can't raise "dict changed size during
# iteration" — the loop body is sync, but the snapshot is cheap insurance
for header, value in list(self.header_overwrites.items()):
if self.inject or header in request_headers: if self.inject or header in request_headers:
request_headers[header] = value request_headers[header] = value
for header, value_callable in list(self.ephemeral_headers.items()): for header, value_callable in self.ephemeral_headers.items():
if self.inject or header in request_headers: if self.inject or header in request_headers:
value = value_callable() value = value_callable()
if isinstance(value, dict): if isinstance(value, dict):
@ -243,8 +208,10 @@ class ExtendedSession:
def preview(self, method, url, **kwargs): def preview(self, method, url, **kwargs):
"""build a RequestPreview for a request without sending it""" """build a RequestPreview for a request without sending it"""
proxy = self._get_proxy(url, kwargs.pop("proxies", None)) proxy = self._get_proxy(url, kwargs.pop("proxies", None))
merged = {**self._default_headers, **(kwargs.pop("headers", None) or {})} if kwargs.get("headers"):
headers = self._apply_overwrites(merged) headers = self._apply_overwrites(kwargs.pop("headers"))
else:
headers = dict(self.get_headers())
timeout = kwargs.get("timeout") timeout = kwargs.get("timeout")
timeout_total = timeout if isinstance(timeout, (int, float)) else None timeout_total = timeout if isinstance(timeout, (int, float)) else None
@ -299,8 +266,7 @@ class ExtendedSession:
kwargs["proxy"] = self._get_proxy(url, kwargs.pop("proxies", None)) kwargs["proxy"] = self._get_proxy(url, kwargs.pop("proxies", None))
debug = kwargs.pop("debug", False) debug = kwargs.pop("debug", False)
merged = {**self._default_headers, **(kwargs.get("headers") or {})} kwargs["headers"] = self._apply_overwrites(kwargs.get("headers"))
kwargs["headers"] = self._apply_overwrites(merged)
kwargs["headers"] = {str(k): str(v) for k, v in kwargs["headers"].items()} kwargs["headers"] = {str(k): str(v) for k, v in kwargs["headers"].items()}
timeout = kwargs.get("timeout") timeout = kwargs.get("timeout")
@ -316,12 +282,6 @@ class ExtendedSession:
if debug and result.redirect_chain: if debug and result.redirect_chain:
log.info("redirect chain: %s", result.redirect_chain) log.info("redirect chain: %s", result.redirect_chain)
return result return result
except asyncio.TimeoutError as error:
# a total ClientTimeout raises a bare asyncio.TimeoutError, which is NOT an
# aiohttp.ClientError subclass — wrap it as ServerTimeoutError (which IS both
# a ClientError AND a TimeoutError) so direct callers get a typed failure and
# request_with_retries can still label it a timeout
raise aiohttp.ServerTimeoutError(f"timeout for {url}: {error}") from error
except aiohttp.ClientError as error: except aiohttp.ClientError as error:
raise aiohttp.ClientError(f"client error for {url}: {error}") from error raise aiohttp.ClientError(f"client error for {url}: {error}") from error
@ -337,48 +297,42 @@ class ExtendedSession:
(backoff_base ** attempt). (backoff_base ** attempt).
""" """
attempts = attempts or DEFAULT_ATTEMPTS attempts = attempts or DEFAULT_ATTEMPTS
body_data, body_json = _route_body(data) last_error = None
if debug: if debug:
preview = self.preview( preview = self.preview(
method=method, url=url, params=params, method=method, url=url, params=params,
data=body_data, json=body_json, data=None if isinstance(data, dict) else data,
json=data if isinstance(data, dict) else None,
headers=headers, proxies=proxies, timeout=timeout, headers=headers, proxies=proxies, timeout=timeout,
).as_curl() ).as_curl()
log.info("[aioweb.debug]\n%s\nproxies: %s inject: %s", preview, self.proxies, self.inject) log.info("[aioweb.debug]\n%s\nproxies: %s inject: %s", preview, self.proxies, self.inject)
async def attempt(): for attempt in range(attempts):
try:
response = await self.request( response = await self.request(
method=method, url=url, params=params, method=method, url=url, params=params,
data=body_data, json=body_json, data=None if isinstance(data, dict) else data,
json=data if isinstance(data, dict) else None,
headers=headers, proxies=proxies, timeout=timeout, debug=debug, headers=headers, proxies=proxies, timeout=timeout, debug=debug,
) )
if response.status_code in retry_statuses: if response.status_code in retry_statuses:
log.warning("retryable status %s for %s", response.status_code, url) last_error = f"retryable status {response.status_code}"
raise _RetryStatus(response) log.warning("attempt %d: %s for %s", attempt + 1, last_error, url)
else:
return response return response
try:
return await aretry(
attempt, attempts=attempts, backoff=1.0, factor=backoff_base,
jitter=False, on=(Exception,),
)
except _RetryStatus as exhausted:
log.error("all %d attempts failed for %s (last status %s)",
attempts, url, exhausted.response.status_code)
return exhausted.response
except asyncio.TimeoutError:
# request() wraps a total timeout as ServerTimeoutError (a ClientError AND a
# TimeoutError); catch the timeout case first so it's labeled a timeout rather
# than falling into the generic client-error branch below
log.error("all %d attempts timed out for %s", attempts, url)
return FailureResponse(reason="timeout", url=url)
except aiohttp.ClientError as error: except aiohttp.ClientError as error:
log.error("all %d attempts failed for %s (client error: %s)", attempts, url, error) last_error = f"client error: {error}"
return FailureResponse(reason=f"client error: {error}", url=url) log.warning("attempt %d: %s, retrying", attempt + 1, last_error)
except Exception as error: except Exception as error:
log.error("all %d attempts failed for %s (unexpected: %s)", attempts, url, error) last_error = f"unexpected error: {error}"
return FailureResponse(reason=f"unexpected error: {error}", url=url) log.exception("attempt %d: %s, retrying", attempt + 1, last_error)
if attempt < attempts - 1:
await asyncio.sleep(backoff_base ** attempt)
log.error("all %d attempts failed for %s", attempts, url)
return FailureResponse(reason=last_error, url=url)
# ------------------------------------------------------------------------- # -------------------------------------------------------------------------
# lifecycle # lifecycle