Compare commits
7
Commits
v0.1.2
...
ce240b0757
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ce240b0757 | ||
|
|
eb7745ae9b | ||
|
|
ce0762a37c | ||
|
|
ef9eb3010c | ||
|
|
da8b0bf6f8 | ||
|
|
612861b76c | ||
|
|
ccea880df0 |
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
# claude
|
# claude
|
||||||
CLAUDE.md
|
.claude/
|
||||||
|
|
||||||
# python
|
# python
|
||||||
__pycache__/
|
__pycache__/
|
||||||
|
|||||||
@@ -22,17 +22,17 @@ you want; importing the package never fails because an extra is missing.
|
|||||||
`requirements.txt` (pick the extra you need):
|
`requirements.txt` (pick the extra you need):
|
||||||
|
|
||||||
```
|
```
|
||||||
aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.2
|
aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5
|
||||||
aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.2
|
aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5
|
||||||
aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.2
|
aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5
|
||||||
```
|
```
|
||||||
|
|
||||||
Direct:
|
Direct:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install "aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.2"
|
pip install "aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5"
|
||||||
pip install "aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.2"
|
pip install "aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5"
|
||||||
pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.2"
|
pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5"
|
||||||
```
|
```
|
||||||
|
|
||||||
- `[curl]` → curl_cffi backend · `[noble]` → noble_tls backend · `[all]` → both.
|
- `[curl]` → curl_cffi backend · `[noble]` → noble_tls backend · `[all]` → both.
|
||||||
@@ -44,6 +44,8 @@ pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-publi
|
|||||||
Constructing a backend whose client isn't installed raises that `RuntimeError` at
|
Constructing a backend whose client isn't installed raises that `RuntimeError` at
|
||||||
construction, never at import.
|
construction, never at import.
|
||||||
|
|
||||||
|
Drop the `@v0.1.5` suffix from the line above to install the latest unpinned.
|
||||||
|
|
||||||
## curl_cffi backend
|
## curl_cffi backend
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@@ -61,6 +63,10 @@ async with TLSSession(backend=CurlCffi(impersonate="chrome"), proxies={"https":
|
|||||||
backend kwargs — passing `impersonate=` there raises `TypeError`; set the profile on
|
backend kwargs — passing `impersonate=` there raises `TypeError`; set the profile on
|
||||||
the `CurlCffi` instance for the retrying path.
|
the `CurlCffi` instance for the retrying path.
|
||||||
- curl_cffi forges JA3/JA4 + HTTP/2 fingerprints via the bundled curl-impersonate binary.
|
- curl_cffi forges JA3/JA4 + HTTP/2 fingerprints via the bundled curl-impersonate binary.
|
||||||
|
- A wire-level timeout raises `aiohttp.ServerTimeoutError` (matching aioweb's own
|
||||||
|
contract), not a generic `aiohttp.ClientError` — both backends detect their
|
||||||
|
native timeout (curl_cffi's `Timeout` type, noble_tls's Go-side timeout text) and
|
||||||
|
re-wrap it before the fallback client-error path (v0.1.5).
|
||||||
|
|
||||||
## noble backend
|
## noble backend
|
||||||
|
|
||||||
@@ -74,10 +80,20 @@ async with TLSSession(backend=Noble(client="chrome_133")) as s:
|
|||||||
print(resp.json()["tls"]["ja3"])
|
print(resp.json()["tls"]["ja3"])
|
||||||
```
|
```
|
||||||
|
|
||||||
- `Noble(client="chrome_133")` — accepts a `noble_tls.Client` enum or a string name.
|
- `Noble(client="chrome_133")` — accepts a `noble_tls.Client` enum or a string name;
|
||||||
|
an unknown string raises `ValueError` listing the valid profile names (v0.1.5).
|
||||||
- noble_tls downloads a Go shared library on first use. `await s.setup()` fetches it
|
- noble_tls downloads a Go shared library on first use. `await s.setup()` fetches it
|
||||||
once at startup; if you skip it, the first request fetches it lazily. The fetch is
|
once at startup; if you skip it, the first request fetches it lazily. The fetch is
|
||||||
guarded by a lock, so even concurrent first requests download it exactly once.
|
guarded by a lock, so even concurrent first requests download it exactly once.
|
||||||
|
- Binary bodies (images, zips, PDFs, protobuf) round-trip as true bytes: noble_tls
|
||||||
|
returns response bodies as a UTF-8 JSON string by default, which mangles non-UTF-8
|
||||||
|
bytes (`U+FFFD` replacement, wrong length) even on a 200 response — the Noble
|
||||||
|
backend always requests `is_byte_response=True` and decodes the resulting
|
||||||
|
base64 data-URI back into raw bytes, so `resp.content` is never lossy.
|
||||||
|
- Multi-valued response headers (e.g. two `Set-Cookie` lines) arrive from noble_tls's
|
||||||
|
Go side as a Python list — the Noble backend flattens them to a single
|
||||||
|
comma-joined string per RFC 7230, so `resp.headers[...]` is always a plain string
|
||||||
|
(v0.1.5).
|
||||||
|
|
||||||
## Writing your own backend (the `TLSBackend` protocol)
|
## Writing your own backend (the `TLSBackend` protocol)
|
||||||
|
|
||||||
@@ -97,6 +113,9 @@ for the authoritative contract):
|
|||||||
| `raw_request` | **required** | `async (session, method, url, **kwargs) -> aioweb.Response` | send one request; adapt the client's response into an `aioweb.Response` |
|
| `raw_request` | **required** | `async (session, method, url, **kwargs) -> aioweb.Response` | send one request; adapt the client's response into an `aioweb.Response` |
|
||||||
| `is_closed` | **required** | `(session) -> bool` | whether the session is closed |
|
| `is_closed` | **required** | `(session) -> bool` | whether the session is closed |
|
||||||
| `cookies_for_url` | optional | `(session, url) -> dict` | cookies for `preview()`; defaults to `{}` |
|
| `cookies_for_url` | optional | `(session, url) -> dict` | cookies for `preview()`; defaults to `{}` |
|
||||||
|
| `set_cookie` | optional | `(session, name, value, domain=None, path="/") -> None` | backs `TLSSession.set_cookie()`; raises `NotImplementedError` if absent |
|
||||||
|
| `get_cookies` | optional | `(session) -> dict` | backs `TLSSession.get_cookies()`; raises `NotImplementedError` if absent |
|
||||||
|
| `clear_cookies` | optional | `(session) -> None` | backs `TLSSession.clear_cookies()`; raises `NotImplementedError` if absent |
|
||||||
| `setup` | optional | `async () -> None` | one-time prep (e.g. fetch a native lib); idempotent |
|
| `setup` | optional | `async () -> None` | one-time prep (e.g. fetch a native lib); idempotent |
|
||||||
| `close` | optional | `async (session) -> None` | close the session; defaults to `await session.close()` |
|
| `close` | optional | `async (session) -> None` | close the session; defaults to `await session.close()` |
|
||||||
|
|
||||||
@@ -154,8 +173,9 @@ async with TLSSession(backend=GoTLSBackend("http://localhost:8080")) as s:
|
|||||||
## Inherited features work unchanged
|
## Inherited features work unchanged
|
||||||
|
|
||||||
aioweb's overwrite/domain/ephemeral/proxy/retry/preview logic operates on plain dicts
|
aioweb's overwrite/domain/ephemeral/proxy/retry/preview logic operates on plain dicts
|
||||||
and never touches the HTTP backend — only the seams do. Every aioweb feature behaves
|
and never touches the HTTP backend — only the seams do. Header overwrites, domain
|
||||||
identically on any backend:
|
rewriting, ephemeral headers, proxies, retries, and previews behave identically on
|
||||||
|
any backend:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
async with TLSSession(backend=CurlCffi(impersonate="chrome")) as s:
|
async with TLSSession(backend=CurlCffi(impersonate="chrome")) as s:
|
||||||
@@ -165,6 +185,19 @@ async with TLSSession(backend=CurlCffi(impersonate="chrome")) as s:
|
|||||||
print(s.preview("GET", "https://internal.local/x").as_curl()) # reflects all of the above
|
print(s.preview("GET", "https://internal.local/x").as_curl()) # reflects all of the above
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Session-default headers are never baked into the underlying client (neither
|
||||||
|
`CurlCffi` nor `Noble` passes `headers=` to their client's constructor) — they flow
|
||||||
|
through aioweb's own per-request `_default_headers` merge instead. That keeps
|
||||||
|
`update_headers()` / `clear_headers()` accurate for both backends: what
|
||||||
|
`get_headers()` and `preview()` report is what actually goes out on the wire, with
|
||||||
|
no stale, already-baked value resurfacing after a clear.
|
||||||
|
|
||||||
|
The mutable cookie API — `set_cookie()` / `get_cookies()` / `clear_cookies()` — is
|
||||||
|
also backend-aware: `CurlCffi` and `Noble` each route it through their own client's
|
||||||
|
cookie store (both expose a `requests`-style `session.cookies` with `set()` /
|
||||||
|
`items()` / `clear()`), so these calls work the same way they do on the base
|
||||||
|
`aioweb.ExtendedSession`, not just `_cookies_for_url()` (used by `preview()`).
|
||||||
|
|
||||||
## Honesty note
|
## Honesty note
|
||||||
|
|
||||||
TLS fingerprinting changes one layer — the TLS/HTTP fingerprint. It does **not** by
|
TLS fingerprinting changes one layer — the TLS/HTTP fingerprint. It does **not** by
|
||||||
@@ -173,4 +206,4 @@ are separate signals. Use this as one component, not a complete anti-bot solutio
|
|||||||
|
|
||||||
## Versioning
|
## Versioning
|
||||||
|
|
||||||
Tagged `vX.Y.Z`. Pin the tag in `requirements.txt`.
|
Releases are tagged `vX.Y.Z`. The install line above pins a release; drop the `@vX.Y.Z` suffix to install the latest unpinned. Pin deliberately for reproducible installs.
|
||||||
|
|||||||
+3
-3
@@ -4,11 +4,11 @@ build-backend = "hatchling.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "aioweb_tls"
|
name = "aioweb_tls"
|
||||||
version = "0.1.2"
|
version = "0.1.5"
|
||||||
description = "TLS-fingerprinting backends for aioweb — curl_cffi / noble_tls ExtendedSession subclasses, config-free, installable."
|
description = "TLS-fingerprinting backends (curl_cffi / noble_tls) for aioweb via one injectable TLSSession, config-free, installable."
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.0",
|
"aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.5",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
|
|||||||
+200
-48
@@ -12,13 +12,53 @@ this module never fails because an extra is missing.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import base64
|
||||||
import logging
|
import logging
|
||||||
import math
|
import math
|
||||||
|
|
||||||
|
import aiohttp
|
||||||
from aioweb import Response
|
from aioweb import Response
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _as_client_error(error: Exception, backend: str) -> aiohttp.ClientError:
|
||||||
|
"""wrap a backend-native network exception as an aiohttp.ClientError
|
||||||
|
|
||||||
|
curl_cffi raises RequestException(OSError), noble_tls raises
|
||||||
|
TLSClientException(IOError) — neither is an aiohttp.ClientError, which is all
|
||||||
|
aioweb's request() re-wraps; translating here gives both the same typed contract.
|
||||||
|
"""
|
||||||
|
return aiohttp.ClientError(f"{backend} request failed: {error}")
|
||||||
|
|
||||||
|
|
||||||
|
_TIMEOUT_TEXT = ("timeout", "timed out", "deadline exceeded")
|
||||||
|
|
||||||
|
|
||||||
|
def _is_timeout_error(error: Exception) -> bool:
|
||||||
|
"""whether a backend-native network error is a timeout
|
||||||
|
|
||||||
|
neither backend raises asyncio.TimeoutError for a wire timeout: curl_cffi's own
|
||||||
|
Timeout and noble_tls's Go-side TLSClientException are both plain OSError
|
||||||
|
subclasses with no dedicated timeout type — check curl_cffi's Timeout class by
|
||||||
|
name first (precise), then fall back to matching the error text (covers
|
||||||
|
noble_tls's Go messages, e.g. "context deadline exceeded").
|
||||||
|
"""
|
||||||
|
timeout_type = getattr(error, "__class__", None)
|
||||||
|
if timeout_type is not None and any(base.__name__ == "Timeout" for base in timeout_type.__mro__):
|
||||||
|
return True
|
||||||
|
return any(needle in str(error).lower() for needle in _TIMEOUT_TEXT)
|
||||||
|
|
||||||
|
|
||||||
|
def _as_timeout_error(error: Exception, backend: str) -> aiohttp.ServerTimeoutError:
|
||||||
|
"""wrap a backend-native timeout as aiohttp.ServerTimeoutError
|
||||||
|
|
||||||
|
matches aioweb's own contract (a total timeout raises ServerTimeoutError, both a
|
||||||
|
ClientError and a TimeoutError) instead of surfacing as a generic client error.
|
||||||
|
"""
|
||||||
|
return aiohttp.ServerTimeoutError(f"{backend} request timed out: {error}")
|
||||||
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from curl_cffi import AsyncSession as _CurlAsyncSession
|
from curl_cffi import AsyncSession as _CurlAsyncSession
|
||||||
_CURL_ERROR = None
|
_CURL_ERROR = None
|
||||||
@@ -37,17 +77,65 @@ except ImportError as error:
|
|||||||
|
|
||||||
|
|
||||||
def _coerce_timeout(value):
|
def _coerce_timeout(value):
|
||||||
"""turn aioweb's aiohttp ClientTimeout (or a number) into a plain number
|
"""unwrap aioweb's aiohttp ClientTimeout (or a plain number) to a number
|
||||||
|
|
||||||
aioweb.request() wraps a numeric timeout in an aiohttp.ClientTimeout before the
|
aioweb.request() wraps a numeric timeout in aiohttp.ClientTimeout before the seam
|
||||||
seam sees it; the tls clients want a number, so unwrap .total when present.
|
sees it; the tls clients want a bare number.
|
||||||
"""
|
"""
|
||||||
total = getattr(value, "total", value)
|
total = getattr(value, "total", value)
|
||||||
return total if isinstance(total, (int, float)) else None
|
return total if isinstance(total, (int, float)) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _noble_timeout_seconds(value):
|
||||||
|
"""coerce a raw or wrapped timeout into whole seconds for noble's Go int field
|
||||||
|
|
||||||
|
single source of truth for both Noble timeout paths (session-default and
|
||||||
|
per-call): the Go field timeoutSeconds is an int, so a sub-second float truncates
|
||||||
|
to 0 (= no timeout) and a non-integer fails Go-side JSON unmarshal outright.
|
||||||
|
rounds up so a sub-second timeout still waits at least 1s. None if uncoercible.
|
||||||
|
"""
|
||||||
|
timeout = _coerce_timeout(value)
|
||||||
|
if timeout is None:
|
||||||
|
return None
|
||||||
|
return max(1, math.ceil(timeout))
|
||||||
|
|
||||||
|
|
||||||
|
def _noble_content(response) -> bytes:
|
||||||
|
"""extract true response bytes from a noble_tls response fetched with is_byte_response
|
||||||
|
|
||||||
|
with is_byte_response=True, noble_tls's Go side returns the body as a
|
||||||
|
`data:<mime>;base64,<payload>` URI string in response.text (the whole Go response
|
||||||
|
is UTF-8-decoded before JSON parsing, so raw bytes travel as base64) —
|
||||||
|
response.content just re-encodes that string, so decode the data-URI here
|
||||||
|
instead. falls back to a plain utf-8 encode if the body isn't a data-URI.
|
||||||
|
"""
|
||||||
|
text = getattr(response, "text", "") or ""
|
||||||
|
if text.startswith("data:") and ";base64," in text:
|
||||||
|
_, _, payload = text.partition(";base64,")
|
||||||
|
return base64.b64decode(payload)
|
||||||
|
return text.encode()
|
||||||
|
|
||||||
|
|
||||||
|
def _flatten_headers(headers) -> dict:
|
||||||
|
"""flatten a Go-style map[str][]str header dict into plain str values
|
||||||
|
|
||||||
|
noble_tls keeps a multi-valued header (e.g. two Set-Cookie lines) as a Python
|
||||||
|
list, breaking any downstream .split()/.lower() call; join with ", " per RFC 7230
|
||||||
|
field-value combination, leaving single values untouched.
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
key: ", ".join(value) if isinstance(value, list) else value
|
||||||
|
for key, value in headers.items()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _jar_to_dict(session):
|
def _jar_to_dict(session):
|
||||||
"""best-effort map of a requests-style cookie jar on session to a plain dict"""
|
"""best-effort map of a requests-style cookie jar on session to a plain dict
|
||||||
|
|
||||||
|
feeds preview() only; the two backends expose differently-shaped jars and a
|
||||||
|
cookie read must never crash a request, so a jar that fails to iterate degrades
|
||||||
|
to {} rather than raising.
|
||||||
|
"""
|
||||||
jar = getattr(session, "cookies", None)
|
jar = getattr(session, "cookies", None)
|
||||||
if not jar:
|
if not jar:
|
||||||
return {}
|
return {}
|
||||||
@@ -62,10 +150,10 @@ class CurlCffi:
|
|||||||
|
|
||||||
config:
|
config:
|
||||||
impersonate: browser profile to forge (default "chrome"); override per call
|
impersonate: browser profile to forge (default "chrome"); override per call
|
||||||
by passing impersonate= to the low-level request()/_raw_request path,
|
by passing impersonate= to the low-level request()/_raw_request path
|
||||||
which forwards **kwargs to the backend. NOT request_with_retries — its
|
(forwards **kwargs to the backend) — NOT request_with_retries, whose
|
||||||
signature is fixed (no **kwargs) and would raise TypeError. for a
|
fixed signature has no **kwargs and raises TypeError. for the retrying
|
||||||
per-call profile under retries, set it on the CurlCffi instance instead.
|
path, set the profile on the CurlCffi instance instead.
|
||||||
|
|
||||||
requires the [curl] extra (pip install "aioweb_tls[curl]").
|
requires the [curl] extra (pip install "aioweb_tls[curl]").
|
||||||
"""
|
"""
|
||||||
@@ -79,8 +167,15 @@ class CurlCffi:
|
|||||||
self.impersonate = impersonate
|
self.impersonate = impersonate
|
||||||
|
|
||||||
def create_session(self, headers, timeout, **kwargs):
|
def create_session(self, headers, timeout, **kwargs):
|
||||||
"""build the curl_cffi AsyncSession"""
|
"""build the curl_cffi AsyncSession
|
||||||
return _CurlAsyncSession(headers=headers, timeout=timeout, **kwargs)
|
|
||||||
|
deliberately does NOT pass `headers`: curl_cffi bakes a constructor
|
||||||
|
`headers=` into the client and re-merges it under per-request headers,
|
||||||
|
which would desync update_headers()/clear_headers() from what's actually on
|
||||||
|
the wire — aioweb's session-default headers already apply per request via
|
||||||
|
the base's _default_headers merge (see ExtendedSession._create_session).
|
||||||
|
"""
|
||||||
|
return _CurlAsyncSession(timeout=timeout, **kwargs)
|
||||||
|
|
||||||
async def raw_request(self, session, method, url, **kwargs) -> Response:
|
async def raw_request(self, session, method, url, **kwargs) -> Response:
|
||||||
"""send via curl_cffi and adapt the result into an aioweb.Response"""
|
"""send via curl_cffi and adapt the result into an aioweb.Response"""
|
||||||
@@ -94,10 +189,20 @@ class CurlCffi:
|
|||||||
if proxy:
|
if proxy:
|
||||||
kwargs["proxy"] = proxy
|
kwargs["proxy"] = proxy
|
||||||
|
|
||||||
response = await session.request(method, url, impersonate=impersonate, **kwargs)
|
try:
|
||||||
content = response.content
|
response = await session.request(method, url, impersonate=impersonate, **kwargs)
|
||||||
if content is None:
|
except aiohttp.ClientError:
|
||||||
content = response.text.encode() if response.text else b""
|
raise
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
raise
|
||||||
|
except OSError as error:
|
||||||
|
# RequestException (incl. curl_cffi's own Timeout) subclasses OSError, never
|
||||||
|
# asyncio.TimeoutError — check for a timeout first so it surfaces as
|
||||||
|
# ServerTimeoutError; narrowed from bare Exception so a real bug surfaces
|
||||||
|
if _is_timeout_error(error):
|
||||||
|
raise _as_timeout_error(error, "curl_cffi") from error
|
||||||
|
raise _as_client_error(error, "curl_cffi") from error
|
||||||
|
content = response.content if response.content is not None else b""
|
||||||
return Response(
|
return Response(
|
||||||
status_code=response.status_code,
|
status_code=response.status_code,
|
||||||
headers=dict(response.headers),
|
headers=dict(response.headers),
|
||||||
@@ -110,10 +215,9 @@ class CurlCffi:
|
|||||||
def is_closed(self, session) -> bool:
|
def is_closed(self, session) -> bool:
|
||||||
"""whether the curl_cffi session is closed
|
"""whether the curl_cffi session is closed
|
||||||
|
|
||||||
curl_cffi tracks closed state in the private `_closed` (no public `closed`
|
curl_cffi tracks closed state in the private `_closed` (no public `closed`);
|
||||||
property), so read that; fall back to a public `closed` if a future version
|
fall back to a public `closed` if a future version adds one. TLSSession's own
|
||||||
adds one. TLSSession's own `_closed` flag is the primary signal — this is a
|
`_closed` flag is the primary signal — this is a best-effort out-of-band check.
|
||||||
best-effort backend check for out-of-band closes.
|
|
||||||
"""
|
"""
|
||||||
closed = getattr(session, "_closed", None)
|
closed = getattr(session, "_closed", None)
|
||||||
if closed is None:
|
if closed is None:
|
||||||
@@ -124,6 +228,18 @@ class CurlCffi:
|
|||||||
"""cookies curl_cffi would send for url (best-effort)"""
|
"""cookies curl_cffi would send for url (best-effort)"""
|
||||||
return _jar_to_dict(session)
|
return _jar_to_dict(session)
|
||||||
|
|
||||||
|
def set_cookie(self, session, name, value, domain=None, path="/") -> None:
|
||||||
|
"""set a cookie in curl_cffi's own cookie store"""
|
||||||
|
session.cookies.set(name, value, domain=domain or "", path=path)
|
||||||
|
|
||||||
|
def get_cookies(self, session) -> dict:
|
||||||
|
"""all cookies stored in curl_cffi's cookie store"""
|
||||||
|
return dict(session.cookies.items())
|
||||||
|
|
||||||
|
def clear_cookies(self, session) -> None:
|
||||||
|
"""clear curl_cffi's cookie store"""
|
||||||
|
session.cookies.clear()
|
||||||
|
|
||||||
async def close(self, session) -> None:
|
async def close(self, session) -> None:
|
||||||
"""close the curl_cffi session"""
|
"""close the curl_cffi session"""
|
||||||
await session.close()
|
await session.close()
|
||||||
@@ -153,21 +269,29 @@ class Noble:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _resolve_client(client):
|
def _resolve_client(client):
|
||||||
"""turn a string or Client enum into a noble_tls Client value"""
|
"""turn a string or Client enum into a noble_tls Client value
|
||||||
if isinstance(client, str):
|
|
||||||
return getattr(_NobleClient, client.upper())
|
raises ValueError naming the valid profiles for an unknown string, instead of
|
||||||
return client
|
letting getattr's raw AttributeError leak the enum's internal lookup mechanics.
|
||||||
|
"""
|
||||||
|
if not isinstance(client, str):
|
||||||
|
return client
|
||||||
|
name = client.upper()
|
||||||
|
resolved = getattr(_NobleClient, name, None)
|
||||||
|
if resolved is None:
|
||||||
|
valid = ", ".join(member.name for member in _NobleClient)
|
||||||
|
raise ValueError(f"unknown noble_tls client profile {client!r}; valid profiles: {valid}")
|
||||||
|
return resolved
|
||||||
|
|
||||||
async def setup(self) -> None:
|
async def setup(self) -> None:
|
||||||
"""fetch the noble_tls Go shared library once; idempotent and concurrency-safe
|
"""fetch the noble_tls Go shared library once; idempotent and concurrency-safe
|
||||||
|
|
||||||
download_if_necessary handles the first-time fetch (no lib present);
|
uses noble_tls.download_if_necessary (fetches on first use, no-ops if
|
||||||
update_if_necessary refreshes an existing one. try download first so a
|
present); falls back to update_if_necessary on older noble_tls.
|
||||||
clean environment works, falling back to update.
|
|
||||||
|
|
||||||
guarded by an asyncio.Lock with a check-lock-recheck so concurrent first
|
guarded by an asyncio.Lock with a check-lock-recheck so concurrent first
|
||||||
requests don't both run the fetch: the fast path returns once _updated is
|
requests don't both run the fetch — only the first caller through the lock
|
||||||
set, and only the first caller through the lock does the work.
|
does the work, the rest see _updated already set.
|
||||||
"""
|
"""
|
||||||
if self._updated:
|
if self._updated:
|
||||||
return
|
return
|
||||||
@@ -177,47 +301,63 @@ class Noble:
|
|||||||
download = getattr(noble_tls, "download_if_necessary", None)
|
download = getattr(noble_tls, "download_if_necessary", None)
|
||||||
if download is not None:
|
if download is not None:
|
||||||
await download()
|
await download()
|
||||||
else:
|
elif hasattr(noble_tls, "update_if_necessary"):
|
||||||
await noble_tls.update_if_necessary()
|
await noble_tls.update_if_necessary()
|
||||||
self._updated = True
|
self._updated = True
|
||||||
|
|
||||||
def create_session(self, headers, timeout, **kwargs):
|
def create_session(self, headers, timeout, **kwargs):
|
||||||
"""build the noble_tls Session, honoring session-default headers + timeout
|
"""build the noble_tls Session, honoring the session-default timeout
|
||||||
|
|
||||||
noble_tls.Session takes neither headers nor timeout in its constructor, so
|
noble_tls.Session takes neither headers nor timeout in its constructor.
|
||||||
aioweb's session-default headers (and the coerced timeout) are applied after
|
deliberately does NOT bake `headers` into session.headers, same rationale as
|
||||||
construction — matching CurlCffi, which passes both through. without this a
|
CurlCffi.create_session — aioweb's per-request _default_headers merge already
|
||||||
TLSSession(backend=Noble(...), headers=...) would silently drop the headers.
|
applies them, so baking here would desync update_headers()/clear_headers().
|
||||||
|
|
||||||
|
the coerced timeout IS applied here (matching raw_request's per-call path):
|
||||||
|
noble's Go field timeoutSeconds is an int, so an uncoerced sub-second/float
|
||||||
|
session-default would fail Go-side JSON unmarshal on any request that doesn't
|
||||||
|
override it per call; max(1, ceil()) mirrors raw_request's own guard.
|
||||||
"""
|
"""
|
||||||
session = noble_tls.Session(client=self.client, **kwargs)
|
session = noble_tls.Session(client=self.client, **kwargs)
|
||||||
if headers:
|
timeout_seconds = _noble_timeout_seconds(timeout)
|
||||||
session.headers.update(headers)
|
if timeout_seconds is not None:
|
||||||
if timeout is not None:
|
session.timeout_seconds = timeout_seconds
|
||||||
session.timeout_seconds = timeout
|
|
||||||
return session
|
return session
|
||||||
|
|
||||||
async def raw_request(self, session, method, url, **kwargs) -> Response:
|
async def raw_request(self, session, method, url, **kwargs) -> Response:
|
||||||
"""send via noble_tls and adapt the result into an aioweb.Response"""
|
"""send via noble_tls and adapt the result into an aioweb.Response"""
|
||||||
await self.setup()
|
await self.setup()
|
||||||
|
|
||||||
timeout = _coerce_timeout(kwargs.pop("timeout", None))
|
timeout_seconds = _noble_timeout_seconds(kwargs.pop("timeout", None))
|
||||||
if timeout is not None:
|
if timeout_seconds is not None:
|
||||||
# noble takes whole seconds; round UP so a sub-second timeout (e.g. 0.5)
|
kwargs["timeout_seconds"] = timeout_seconds
|
||||||
# doesn't truncate to 0 (which would mean no/instant timeout)
|
|
||||||
kwargs["timeout_seconds"] = max(1, math.ceil(timeout))
|
|
||||||
|
|
||||||
proxy = kwargs.pop("proxy", None)
|
proxy = kwargs.pop("proxy", None)
|
||||||
if proxy:
|
if proxy:
|
||||||
kwargs["proxy"] = proxy
|
kwargs["proxy"] = proxy
|
||||||
|
|
||||||
response = await session.execute_request(method=method.upper(), url=url, **kwargs)
|
# byte-safe transport: without this, noble_tls's Go side returns the body as a
|
||||||
content = getattr(response, "content", None)
|
# plain UTF-8 JSON string (U+FFFD-mangling any binary payload on a 200 response);
|
||||||
if content is None:
|
# see _noble_content() for the base64 data-URI decode this pairs with.
|
||||||
text = getattr(response, "text", "") or ""
|
kwargs.setdefault("is_byte_response", True)
|
||||||
content = text.encode()
|
|
||||||
|
try:
|
||||||
|
response = await session.execute_request(method=method.upper(), url=url, **kwargs)
|
||||||
|
except aiohttp.ClientError:
|
||||||
|
raise
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
raise
|
||||||
|
except OSError as error:
|
||||||
|
# TLSClientException subclasses IOError (== OSError); the Go side has no
|
||||||
|
# distinct timeout type either, just a text body ("context deadline
|
||||||
|
# exceeded") — check for one before the generic client-error wrap
|
||||||
|
if _is_timeout_error(error):
|
||||||
|
raise _as_timeout_error(error, "noble_tls") from error
|
||||||
|
raise _as_client_error(error, "noble_tls") from error
|
||||||
|
content = _noble_content(response)
|
||||||
return Response(
|
return Response(
|
||||||
status_code=response.status_code,
|
status_code=response.status_code,
|
||||||
headers=dict(getattr(response, "headers", {}) or {}),
|
headers=_flatten_headers(getattr(response, "headers", {}) or {}),
|
||||||
content=content,
|
content=content,
|
||||||
url=str(getattr(response, "url", url)),
|
url=str(getattr(response, "url", url)),
|
||||||
reason=getattr(response, "reason", None),
|
reason=getattr(response, "reason", None),
|
||||||
@@ -232,6 +372,18 @@ class Noble:
|
|||||||
"""cookies noble_tls would send for url (best-effort)"""
|
"""cookies noble_tls would send for url (best-effort)"""
|
||||||
return _jar_to_dict(session)
|
return _jar_to_dict(session)
|
||||||
|
|
||||||
|
def set_cookie(self, session, name, value, domain=None, path="/") -> None:
|
||||||
|
"""set a cookie in noble_tls's own cookie jar"""
|
||||||
|
session.cookies.set(name, value, domain=domain or "", path=path)
|
||||||
|
|
||||||
|
def get_cookies(self, session) -> dict:
|
||||||
|
"""all cookies stored in noble_tls's cookie jar"""
|
||||||
|
return dict(session.cookies.items())
|
||||||
|
|
||||||
|
def clear_cookies(self, session) -> None:
|
||||||
|
"""clear noble_tls's cookie jar"""
|
||||||
|
session.cookies.clear()
|
||||||
|
|
||||||
async def close(self, session) -> None:
|
async def close(self, session) -> None:
|
||||||
"""close the noble_tls session if it exposes a close"""
|
"""close the noble_tls session if it exposes a close"""
|
||||||
close = getattr(session, "close", None)
|
close = getattr(session, "close", None)
|
||||||
|
|||||||
+20
-14
@@ -6,33 +6,39 @@ to one HTTP client. TLSSession owns the live session object (built by create_ses
|
|||||||
and passes it into every backend call, so backends hold no per-request state.
|
and passes it into every backend call, so backends hold no per-request state.
|
||||||
|
|
||||||
implement this protocol to add a custom backend (e.g. a local Go TLS server); inject
|
implement this protocol to add a custom backend (e.g. a local Go TLS server); inject
|
||||||
it via TLSSession(backend=MyBackend(...)) and it inherits all of aioweb's domain /
|
it via TLSSession(backend=MyBackend(...)) and it inherits aioweb's domain / header /
|
||||||
header / ephemeral / proxy / retry / preview logic unchanged — those operate on plain
|
ephemeral / proxy / retry / preview logic unchanged — those operate on plain dicts
|
||||||
dicts and never touch the backend.
|
and never touch the backend.
|
||||||
|
|
||||||
required:
|
required:
|
||||||
create_session(headers: dict, timeout, **kwargs) -> session
|
create_session(headers: dict, timeout, **kwargs) -> session
|
||||||
build and return the live client session object. TLSSession stores it as
|
build and return the live client session object, stored as self.session and
|
||||||
self.session and hands it back to every other method below.
|
passed to every method below.
|
||||||
|
|
||||||
async raw_request(session, method, url, **kwargs) -> aioweb.Response
|
async raw_request(session, method, url, **kwargs) -> aioweb.Response
|
||||||
send one request with `session` and adapt the client's response into an
|
send one request; adapt the client's response into an aioweb.Response built
|
||||||
aioweb.Response built from primitives (status_code, headers, content bytes,
|
from primitives (status_code, headers, content bytes, url, reason). kwargs
|
||||||
url, reason). kwargs arrive aioweb-shaped — the base has already resolved the
|
arrive aioweb-shaped: proxy resolved into kwargs["proxy"], headers merged
|
||||||
proxy into kwargs["proxy"] and merged headers into kwargs["headers"], and a
|
into kwargs["headers"], numeric timeout wrapped in aiohttp.ClientTimeout
|
||||||
numeric timeout is wrapped in an aiohttp.ClientTimeout (unwrap .total).
|
(unwrap .total).
|
||||||
|
|
||||||
is_closed(session) -> bool
|
is_closed(session) -> bool
|
||||||
whether `session` is closed.
|
whether `session` is closed.
|
||||||
|
|
||||||
optional:
|
optional:
|
||||||
cookies_for_url(session, url) -> dict
|
cookies_for_url(session, url) -> dict
|
||||||
cookies the client would send for url, for preview(). default {} (used when
|
cookies the client would send for url, for preview(). default {}.
|
||||||
the backend has no introspectable jar).
|
|
||||||
|
set_cookie(session, name, value, domain=None, path="/") -> None
|
||||||
|
get_cookies(session) -> dict
|
||||||
|
clear_cookies(session) -> None
|
||||||
|
back TLSSession's mutable cookie api. the aiohttp-only base reaches into
|
||||||
|
session.cookie_jar, which TLS backends lack, so an implementation without
|
||||||
|
these raises NotImplementedError instead of a private-attribute AttributeError.
|
||||||
|
|
||||||
async setup() -> None
|
async setup() -> None
|
||||||
one-time async preparation (e.g. fetch a native lib). called once via
|
one-time async preparation (e.g. fetch a native lib); idempotent. called via
|
||||||
TLSSession.setup() and lazily before the first request; make it idempotent.
|
TLSSession.setup() and lazily before the first request.
|
||||||
|
|
||||||
async close(session) -> None
|
async close(session) -> None
|
||||||
close `session`. default awaits session.close() if present.
|
close `session`. default awaits session.close() if present.
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ TLSSession — one aioweb session, any tls backend
|
|||||||
TLSSession subclasses aioweb.ExtendedSession and delegates only the four backend
|
TLSSession subclasses aioweb.ExtendedSession and delegates only the four backend
|
||||||
seams to an injected backend object (see protocol.py). everything else — header
|
seams to an injected backend object (see protocol.py). everything else — header
|
||||||
overwrites, domain rewriting, ephemeral headers, proxies, retries, previews — is
|
overwrites, domain rewriting, ephemeral headers, proxies, retries, previews — is
|
||||||
inherited from aioweb unchanged, because that logic operates on plain dicts and
|
inherited unchanged, since that logic operates on plain dicts and never touches
|
||||||
never touches the backend.
|
the backend.
|
||||||
|
|
||||||
from aioweb_tls import TLSSession, CurlCffi, Noble
|
from aioweb_tls import TLSSession, CurlCffi, Noble
|
||||||
|
|
||||||
@@ -81,6 +81,38 @@ class TLSSession(ExtendedSession):
|
|||||||
return True
|
return True
|
||||||
return self.backend.is_closed(self.session)
|
return self.backend.is_closed(self.session)
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------------
|
||||||
|
# mutable cookie api — the base's set_cookie/get_cookies/clear_cookies reach
|
||||||
|
# into self.session.cookie_jar (aiohttp-only), so TLS backends override them
|
||||||
|
# to route through the backend instead of crashing with AttributeError
|
||||||
|
|
||||||
|
def set_cookie(self, name, value, domain=None, path="/"):
|
||||||
|
"""set a cookie via the backend's own cookie store"""
|
||||||
|
set_cookie = getattr(self.backend, "set_cookie", None)
|
||||||
|
if set_cookie is None:
|
||||||
|
raise NotImplementedError(
|
||||||
|
f"{type(self.backend).__name__} does not support the mutable cookie api"
|
||||||
|
)
|
||||||
|
set_cookie(self.session, name, value, domain=domain, path=path)
|
||||||
|
|
||||||
|
def get_cookies(self) -> dict:
|
||||||
|
"""all cookies stored in the backend's cookie store"""
|
||||||
|
get_cookies = getattr(self.backend, "get_cookies", None)
|
||||||
|
if get_cookies is None:
|
||||||
|
raise NotImplementedError(
|
||||||
|
f"{type(self.backend).__name__} does not support the mutable cookie api"
|
||||||
|
)
|
||||||
|
return get_cookies(self.session)
|
||||||
|
|
||||||
|
def clear_cookies(self) -> None:
|
||||||
|
"""clear the backend's cookie store"""
|
||||||
|
clear_cookies = getattr(self.backend, "clear_cookies", None)
|
||||||
|
if clear_cookies is None:
|
||||||
|
raise NotImplementedError(
|
||||||
|
f"{type(self.backend).__name__} does not support the mutable cookie api"
|
||||||
|
)
|
||||||
|
clear_cookies(self.session)
|
||||||
|
|
||||||
# -------------------------------------------------------------------------
|
# -------------------------------------------------------------------------
|
||||||
# lifecycle — backends close differently, so route through the backend
|
# lifecycle — backends close differently, so route through the backend
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user