From 6a3c4347ec6e2ea140e67779a1bdb8c0d132f464 Mon Sep 17 00:00:00 2001 From: disqualifier Date: Wed, 24 Jun 2026 18:01:39 -0400 Subject: [PATCH] init: async HTTP session wrapper over aiohttp Signed-off-by: disqualifier --- .gitignore | 15 ++++++ README.md | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ae590bc --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +# claude +CLAUDE.md + +# python +__pycache__/ +*.py[cod] +*.egg-info/ +build/ +dist/ +.eggs/ + +# env +.venv/ +venv/ +.env diff --git a/README.md b/README.md new file mode 100644 index 0000000..f250fa8 --- /dev/null +++ b/README.md @@ -0,0 +1,133 @@ +# aioweb + +Async HTTP session wrapper over `aiohttp`. Adds session-level proxies, header +overwrites, ephemeral (per-request generated) headers, domain rewriting, request +previews / cURL export, and retry-with-backoff. The byte-sending is isolated behind +one overridable method (`_raw_request`), so a TLS-fingerprinting backend can subclass +and swap the HTTP client while inheriting everything else. + +## Install + +`requirements.txt`: + +``` +aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.0 +``` + +Direct: + +```bash +pip install "aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.0" +``` + +Requires `aiohttp` and `yarl` (pulled transitively). + +## Usage + +```python +from aioweb import ExtendedSession + +async with ExtendedSession(proxies={"https": "http://user:pass@host:port"}, timeout=15) as s: + resp = await s.request_with_retries("GET", "https://example.com") + if resp: # FailureResponse is falsy + data = resp.json() # or resp.text(), resp.content +``` + +Sessions must be closed explicitly — use `async with` or `await s.close()`. There is +no `__del__` auto-close (unsafe for async resources); leaking a session emits a +`ResourceWarning`. + +## Responses + +`request`/`request_with_retries` return a `Response` (success or non-retryable status) +or a falsy `FailureResponse` (all retries failed). Both expose the same surface as +**properties**, so callers branch uniformly: + +- `status_code`, `headers`, `url`, `reason`, `cookies`, `history`, `redirect_chain` +- `is_success` (2xx), `is_redirect` +- `content` (bytes), `text(encoding=None)`, `json()` (None if not JSON) +- `raise_for_status()` raises `AiowebError` on non-2xx +- `bool(resp)` / `if resp:` is `is_success` + +## Retries + +`request_with_retries` retries on exceptions **and** retryable statuses (429, 500, +502, 503, 504 by default), with exponential backoff (`backoff_base ** attempt`). + +```python +resp = await s.request_with_retries( + "GET", url, attempts=5, backoff_base=2.0, + retry_statuses={429, 503}, # override which statuses retry +) +``` + +Returns a `FailureResponse` (falsy) if every attempt fails. + +## Header overwrites & ephemeral headers + +```python +s.overwrite_header("User-Agent", "custom") # replace per request +s.overwrite_inject(True) # also add when absent +s.set_ephemeral("X-Time", lambda: str(time.time())) # generated fresh each request +``` + +With `inject=False` (default) overwrites only replace headers already present in a +request; with `inject=True` they're added regardless. + +## Domain rewriting + +```python +s.overwrite_domain("internal.local", "127.0.0.1") # host-substring rewrite +``` + +## Preview / debug + +```python +print(s.preview("POST", url, json={"a": 1}).as_curl()) # equivalent cURL command +``` + +Pass `debug=True` to `request_with_retries` to log the cURL preview and request flow. + +## Custom backends + +The `_raw_request(method, url, **kwargs) -> Response` method is the only place that +touches the HTTP client. To use a different backend (e.g. a TLS-fingerprinting client +like curl_cffi), subclass `ExtendedSession` and override just `_raw_request`, building +a `Response` from that backend's primitives: + +```python +class MySession(ExtendedSession): + async def _raw_request(self, method, url, **kwargs): + r = await my_client.request(method, url, **kwargs) + return Response( + status_code=r.status, headers=r.headers, content=await r.read(), + url=str(r.url), reason=r.reason, + ) +``` + +Everything else — header overwrites, ephemeral headers, domain rewriting, proxy +resolution, retries, previews — is inherited. `Response` is built from primitives +(status, headers, content, url, history) precisely so any backend can produce one. + +## Migrating from the original + +Back-compat shims are in place for the common path: + +- `aiowebResponse` is aliased to `Response` (the class was renamed) — old imports work. +- `request_retries(session, ...)` and `test_proxies(session)` remain as module functions. +- `raise_for_status` now raises `AiowebError` (a subclass of `Exception`), so + `except Exception` still catches it. + +Two changes can't be shimmed without re-introducing the bugs they fix: + +- **`is_success` is now a property on `FailureResponse`, not a method.** Code that + *called* `failure.is_success()` must drop the parens to `failure.is_success`. (Code + that wrote `if failure.is_success` was previously always-truthy — a bug — and now + behaves correctly.) +- **No `__del__` auto-close.** Sessions must be closed via `async with` or + `await s.close()`; a leaked session emits a `ResourceWarning`. The old finalizer-based + auto-close was unsafe and was removed. + +## Versioning + +Tagged `vX.Y.Z`. Pin the tag in `requirements.txt`.