Compare commits
No commits in common. "main" and "v0.1.1" have entirely different histories.
2
.gitignore
vendored
2
.gitignore
vendored
@ -1,5 +1,5 @@
|
|||||||
# claude
|
# claude
|
||||||
.claude/
|
CLAUDE.md
|
||||||
|
|
||||||
# python
|
# python
|
||||||
__pycache__/
|
__pycache__/
|
||||||
|
|||||||
25
README.md
25
README.md
@ -11,19 +11,17 @@ and swap the HTTP client while inheriting everything else.
|
|||||||
`requirements.txt`:
|
`requirements.txt`:
|
||||||
|
|
||||||
```
|
```
|
||||||
aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.5
|
aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.0
|
||||||
```
|
```
|
||||||
|
|
||||||
Direct:
|
Direct:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install "aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.5"
|
pip install "aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.0"
|
||||||
```
|
```
|
||||||
|
|
||||||
Requires `aiohttp` and `yarl` (pulled transitively).
|
Requires `aiohttp` and `yarl` (pulled transitively).
|
||||||
|
|
||||||
Drop the `@v0.1.5` suffix from the line above to install the latest unpinned.
|
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@ -130,23 +128,6 @@ Two changes can't be shimmed without re-introducing the bugs they fix:
|
|||||||
`await s.close()`; a leaked session emits a `ResourceWarning`. The old finalizer-based
|
`await s.close()`; a leaked session emits a `ResourceWarning`. The old finalizer-based
|
||||||
auto-close was unsafe and was removed.
|
auto-close was unsafe and was removed.
|
||||||
|
|
||||||
## Changelog
|
|
||||||
|
|
||||||
### v0.1.2
|
|
||||||
|
|
||||||
- Pinned `commons` to v0.2.1 (retry `attempts` floor fix).
|
|
||||||
|
|
||||||
### v0.1.1
|
|
||||||
|
|
||||||
- **JSON list bodies** now route to `json=` (were wrongly form-encoded via `data=` —
|
|
||||||
only dicts went to `json=` before).
|
|
||||||
- **Exhausted retries return the real last response.** When every attempt hit a
|
|
||||||
retryable status (429/5xx), the loop discarded it and returned a synthetic
|
|
||||||
`FailureResponse` (status 0); now the real last 4xx/5xx `Response` is returned (only a
|
|
||||||
pure-exception failure yields `FailureResponse`).
|
|
||||||
- Retry/backoff moved onto `commons.aretry` (shared engine); backoff schedule unchanged.
|
|
||||||
Adds a `commons` dependency.
|
|
||||||
|
|
||||||
## Versioning
|
## Versioning
|
||||||
|
|
||||||
Releases are tagged `vX.Y.Z`. The install line above pins a release; drop the `@vX.Y.Z` suffix to install the latest unpinned. Pin deliberately for reproducible installs.
|
Tagged `vX.Y.Z`. Pin the tag in `requirements.txt`.
|
||||||
|
|||||||
@ -4,13 +4,13 @@ build-backend = "hatchling.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "aioweb"
|
name = "aioweb"
|
||||||
version = "0.1.5"
|
version = "0.1.1"
|
||||||
description = "Async HTTP session wrapper over aiohttp — proxies, header overwrites, retries, previews. Config-free, installable."
|
description = "Async HTTP session wrapper over aiohttp — proxies, header overwrites, retries, previews. Config-free, installable."
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aiohttp>=3.9",
|
"aiohttp>=3.9",
|
||||||
"yarl>=1.9",
|
"yarl>=1.9",
|
||||||
"commons @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.2.1",
|
"commons @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.2.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[tool.hatch.metadata]
|
[tool.hatch.metadata]
|
||||||
|
|||||||
@ -3,7 +3,6 @@ request preview for aioweb — format or export a request without sending it
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import json as _json
|
import json as _json
|
||||||
import shlex
|
|
||||||
|
|
||||||
|
|
||||||
class RequestPreview:
|
class RequestPreview:
|
||||||
@ -26,22 +25,15 @@ class RequestPreview:
|
|||||||
return "\n".join(f"{key}: {value}" for key, value in self.details.items())
|
return "\n".join(f"{key}: {value}" for key, value in self.details.items())
|
||||||
|
|
||||||
def as_curl(self):
|
def as_curl(self):
|
||||||
"""equivalent cURL command for the request
|
"""equivalent cURL command for the request"""
|
||||||
|
parts = [f"curl -X {self.details['method']}"]
|
||||||
every interpolated value is shell-quoted with shlex.quote, so headers,
|
|
||||||
body, url, or proxy containing quotes/spaces/metacharacters produce a
|
|
||||||
valid, non-injectable command rather than a broken or unsafe one.
|
|
||||||
"""
|
|
||||||
parts = [f"curl -X {shlex.quote(self.details['method'])}"]
|
|
||||||
for header, value in (self.details["headers"] or {}).items():
|
for header, value in (self.details["headers"] or {}).items():
|
||||||
parts.append(f"-H {shlex.quote(f'{header}: {value}')}")
|
parts.append(f"-H '{header}: {value}'")
|
||||||
if self.details["data"] is not None:
|
if self.details["data"]:
|
||||||
parts.append(f"--data {shlex.quote(str(self.details['data']))}")
|
parts.append(f"--data '{self.details['data']}'")
|
||||||
elif self.details["json"] is not None:
|
elif self.details["json"]:
|
||||||
# is-not-None, not truthiness: an empty-but-valid body ({} / []) must still
|
parts.append(f"--data '{_json.dumps(self.details['json'])}'")
|
||||||
# render rather than being dropped as falsy
|
parts.append(f"'{self.details['url']}'")
|
||||||
parts.append(f"--data {shlex.quote(_json.dumps(self.details['json']))}")
|
|
||||||
parts.append(shlex.quote(str(self.details["url"])))
|
|
||||||
if self.details["proxy"]:
|
if self.details["proxy"]:
|
||||||
parts.append(f"--proxy {shlex.quote(str(self.details['proxy']))}")
|
parts.append(f"--proxy '{self.details['proxy']}'")
|
||||||
return " \\\n ".join(parts)
|
return " \\\n ".join(parts)
|
||||||
|
|||||||
@ -92,9 +92,7 @@ class Response:
|
|||||||
"""parsed JSON content, or None if not valid JSON"""
|
"""parsed JSON content, or None if not valid JSON"""
|
||||||
try:
|
try:
|
||||||
return _json.loads(self.text())
|
return _json.loads(self.text())
|
||||||
except (_json.JSONDecodeError, UnicodeDecodeError):
|
except _json.JSONDecodeError:
|
||||||
# text() decodes the body and can raise UnicodeDecodeError on a non-UTF-8
|
|
||||||
# payload — that's a "not valid JSON" outcome, not an error to propagate
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def raise_for_status(self):
|
def raise_for_status(self):
|
||||||
|
|||||||
@ -17,7 +17,6 @@ sessions must be closed explicitly (async with, or await s.close()); there is no
|
|||||||
__del__ auto-close (that pattern is unsafe for async resources).
|
__del__ auto-close (that pattern is unsafe for async resources).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import logging
|
import logging
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
@ -96,15 +95,9 @@ class ExtendedSession:
|
|||||||
proxy/retry/preview logic in this class never touches the session object
|
proxy/retry/preview logic in this class never touches the session object
|
||||||
directly (only _raw_request, the cookie methods, and close do), so those
|
directly (only _raw_request, the cookie methods, and close do), so those
|
||||||
features work unchanged on any backend.
|
features work unchanged on any backend.
|
||||||
|
|
||||||
`headers` is the session-default header set; the default aiohttp backend
|
|
||||||
does NOT bake it into the ClientSession (which would copy it into an
|
|
||||||
immutable per-session map that update_headers/clear_headers can't touch).
|
|
||||||
instead `_default_headers` is our own mutable layer that request() and
|
|
||||||
preview() merge per call, so the mutable session-header API actually works.
|
|
||||||
a backend that needs the defaults baked at construction may use `headers`.
|
|
||||||
"""
|
"""
|
||||||
return aiohttp.ClientSession(
|
return aiohttp.ClientSession(
|
||||||
|
headers=headers,
|
||||||
timeout=aiohttp.ClientTimeout(
|
timeout=aiohttp.ClientTimeout(
|
||||||
total=timeout,
|
total=timeout,
|
||||||
connect=timeout / 2,
|
connect=timeout / 2,
|
||||||
@ -124,13 +117,10 @@ class ExtendedSession:
|
|||||||
def _apply_overwrites(self, request_headers):
|
def _apply_overwrites(self, request_headers):
|
||||||
"""apply static overwrites and ephemeral headers to a request's headers"""
|
"""apply static overwrites and ephemeral headers to a request's headers"""
|
||||||
request_headers = dict(request_headers or {})
|
request_headers = dict(request_headers or {})
|
||||||
# snapshot the shared override dicts so a concurrent mutation (e.g. a command
|
for header, value in self.header_overwrites.items():
|
||||||
# editing ephemerals on a shared session) can't raise "dict changed size during
|
|
||||||
# iteration" — the loop body is sync, but the snapshot is cheap insurance
|
|
||||||
for header, value in list(self.header_overwrites.items()):
|
|
||||||
if self.inject or header in request_headers:
|
if self.inject or header in request_headers:
|
||||||
request_headers[header] = value
|
request_headers[header] = value
|
||||||
for header, value_callable in list(self.ephemeral_headers.items()):
|
for header, value_callable in self.ephemeral_headers.items():
|
||||||
if self.inject or header in request_headers:
|
if self.inject or header in request_headers:
|
||||||
value = value_callable()
|
value = value_callable()
|
||||||
if isinstance(value, dict):
|
if isinstance(value, dict):
|
||||||
@ -243,8 +233,10 @@ class ExtendedSession:
|
|||||||
def preview(self, method, url, **kwargs):
|
def preview(self, method, url, **kwargs):
|
||||||
"""build a RequestPreview for a request without sending it"""
|
"""build a RequestPreview for a request without sending it"""
|
||||||
proxy = self._get_proxy(url, kwargs.pop("proxies", None))
|
proxy = self._get_proxy(url, kwargs.pop("proxies", None))
|
||||||
merged = {**self._default_headers, **(kwargs.pop("headers", None) or {})}
|
if kwargs.get("headers"):
|
||||||
headers = self._apply_overwrites(merged)
|
headers = self._apply_overwrites(kwargs.pop("headers"))
|
||||||
|
else:
|
||||||
|
headers = dict(self.get_headers())
|
||||||
|
|
||||||
timeout = kwargs.get("timeout")
|
timeout = kwargs.get("timeout")
|
||||||
timeout_total = timeout if isinstance(timeout, (int, float)) else None
|
timeout_total = timeout if isinstance(timeout, (int, float)) else None
|
||||||
@ -299,8 +291,7 @@ class ExtendedSession:
|
|||||||
kwargs["proxy"] = self._get_proxy(url, kwargs.pop("proxies", None))
|
kwargs["proxy"] = self._get_proxy(url, kwargs.pop("proxies", None))
|
||||||
debug = kwargs.pop("debug", False)
|
debug = kwargs.pop("debug", False)
|
||||||
|
|
||||||
merged = {**self._default_headers, **(kwargs.get("headers") or {})}
|
kwargs["headers"] = self._apply_overwrites(kwargs.get("headers"))
|
||||||
kwargs["headers"] = self._apply_overwrites(merged)
|
|
||||||
kwargs["headers"] = {str(k): str(v) for k, v in kwargs["headers"].items()}
|
kwargs["headers"] = {str(k): str(v) for k, v in kwargs["headers"].items()}
|
||||||
|
|
||||||
timeout = kwargs.get("timeout")
|
timeout = kwargs.get("timeout")
|
||||||
@ -316,12 +307,6 @@ class ExtendedSession:
|
|||||||
if debug and result.redirect_chain:
|
if debug and result.redirect_chain:
|
||||||
log.info("redirect chain: %s", result.redirect_chain)
|
log.info("redirect chain: %s", result.redirect_chain)
|
||||||
return result
|
return result
|
||||||
except asyncio.TimeoutError as error:
|
|
||||||
# a total ClientTimeout raises a bare asyncio.TimeoutError, which is NOT an
|
|
||||||
# aiohttp.ClientError subclass — wrap it as ServerTimeoutError (which IS both
|
|
||||||
# a ClientError AND a TimeoutError) so direct callers get a typed failure and
|
|
||||||
# request_with_retries can still label it a timeout
|
|
||||||
raise aiohttp.ServerTimeoutError(f"timeout for {url}: {error}") from error
|
|
||||||
except aiohttp.ClientError as error:
|
except aiohttp.ClientError as error:
|
||||||
raise aiohttp.ClientError(f"client error for {url}: {error}") from error
|
raise aiohttp.ClientError(f"client error for {url}: {error}") from error
|
||||||
|
|
||||||
@ -367,12 +352,6 @@ class ExtendedSession:
|
|||||||
log.error("all %d attempts failed for %s (last status %s)",
|
log.error("all %d attempts failed for %s (last status %s)",
|
||||||
attempts, url, exhausted.response.status_code)
|
attempts, url, exhausted.response.status_code)
|
||||||
return exhausted.response
|
return exhausted.response
|
||||||
except asyncio.TimeoutError:
|
|
||||||
# request() wraps a total timeout as ServerTimeoutError (a ClientError AND a
|
|
||||||
# TimeoutError); catch the timeout case first so it's labeled a timeout rather
|
|
||||||
# than falling into the generic client-error branch below
|
|
||||||
log.error("all %d attempts timed out for %s", attempts, url)
|
|
||||||
return FailureResponse(reason="timeout", url=url)
|
|
||||||
except aiohttp.ClientError as error:
|
except aiohttp.ClientError as error:
|
||||||
log.error("all %d attempts failed for %s (client error: %s)", attempts, url, error)
|
log.error("all %d attempts failed for %s (client error: %s)", attempts, url, error)
|
||||||
return FailureResponse(reason=f"client error: {error}", url=url)
|
return FailureResponse(reason=f"client error: {error}", url=url)
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user