Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7ee6cc9b88 | ||
|
|
b5cc6b9374 | ||
|
|
cc5e4a9414 | ||
|
|
ec1a20a3f6 | ||
|
|
8ed97a185f | ||
|
|
df48d1cea3 | ||
|
|
b23e1d399e | ||
|
|
d40be6928a | ||
|
|
76c3024ccc | ||
|
|
92ddd5dc39 | ||
|
|
226f273695 | ||
|
|
ce240b0757 | ||
|
|
eb7745ae9b | ||
|
|
ce0762a37c | ||
|
|
ef9eb3010c | ||
|
|
da8b0bf6f8 | ||
|
|
612861b76c | ||
|
|
ccea880df0 | ||
|
|
5eb689ba73 |
+1
-1
@@ -1,5 +1,5 @@
|
||||
# claude
|
||||
CLAUDE.md
|
||||
.claude/
|
||||
|
||||
# python
|
||||
__pycache__/
|
||||
|
||||
@@ -22,17 +22,17 @@ you want; importing the package never fails because an extra is missing.
|
||||
`requirements.txt` (pick the extra you need):
|
||||
|
||||
```
|
||||
aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.1
|
||||
aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.1
|
||||
aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.1
|
||||
aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v1.0.0
|
||||
aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v1.0.0
|
||||
aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v1.0.0
|
||||
```
|
||||
|
||||
Direct:
|
||||
|
||||
```bash
|
||||
pip install "aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.1"
|
||||
pip install "aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.1"
|
||||
pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.1"
|
||||
pip install "aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v1.0.0"
|
||||
pip install "aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v1.0.0"
|
||||
pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v1.0.0"
|
||||
```
|
||||
|
||||
- `[curl]` → curl_cffi backend · `[noble]` → noble_tls backend · `[all]` → both.
|
||||
@@ -44,6 +44,8 @@ pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-publi
|
||||
Constructing a backend whose client isn't installed raises that `RuntimeError` at
|
||||
construction, never at import.
|
||||
|
||||
Drop the `@v1.0.0` suffix from the line above to install the latest unpinned.
|
||||
|
||||
## curl_cffi backend
|
||||
|
||||
```python
|
||||
@@ -55,9 +57,16 @@ async with TLSSession(backend=CurlCffi(impersonate="chrome"), proxies={"https":
|
||||
print(resp.json()["tls"]["ja3"])
|
||||
```
|
||||
|
||||
- `CurlCffi(impersonate="chrome")` sets the forged profile; override per call by
|
||||
passing `impersonate=` to any request method.
|
||||
- `CurlCffi(impersonate="chrome")` sets the forged profile; override it per call by
|
||||
passing `impersonate=` to the low-level `request()` (which forwards `**kwargs` to the
|
||||
backend). `request_with_retries` has a fixed signature and does **not** accept extra
|
||||
backend kwargs — passing `impersonate=` there raises `TypeError`; set the profile on
|
||||
the `CurlCffi` instance for the retrying path.
|
||||
- curl_cffi forges JA3/JA4 + HTTP/2 fingerprints via the bundled curl-impersonate binary.
|
||||
- A wire-level timeout raises `aiohttp.ServerTimeoutError` (matching aioweb's own
|
||||
contract), not a generic `aiohttp.ClientError` — both backends detect their
|
||||
native timeout (curl_cffi's `Timeout` type, noble_tls's Go-side timeout text) and
|
||||
re-wrap it before the fallback client-error path (v0.1.5).
|
||||
|
||||
## noble backend
|
||||
|
||||
@@ -71,10 +80,20 @@ async with TLSSession(backend=Noble(client="chrome_133")) as s:
|
||||
print(resp.json()["tls"]["ja3"])
|
||||
```
|
||||
|
||||
- `Noble(client="chrome_133")` — accepts a `noble_tls.Client` enum or a string name.
|
||||
- `Noble(client="chrome_133")` — accepts a `noble_tls.Client` enum or a string name;
|
||||
an unknown string raises `ValueError` listing the valid profile names (v0.1.5).
|
||||
- noble_tls downloads a Go shared library on first use. `await s.setup()` fetches it
|
||||
once at startup; if you skip it, the first request fetches it lazily. The fetch is
|
||||
guarded by a lock, so even concurrent first requests download it exactly once.
|
||||
- Binary bodies (images, zips, PDFs, protobuf) round-trip as true bytes: noble_tls
|
||||
returns response bodies as a UTF-8 JSON string by default, which mangles non-UTF-8
|
||||
bytes (`U+FFFD` replacement, wrong length) even on a 200 response — the Noble
|
||||
backend always requests `is_byte_response=True` and decodes the resulting
|
||||
base64 data-URI back into raw bytes, so `resp.content` is never lossy.
|
||||
- Multi-valued response headers (e.g. two `Set-Cookie` lines) arrive from noble_tls's
|
||||
Go side as a Python list — the Noble backend flattens them to a single
|
||||
comma-joined string per RFC 7230, so `resp.headers[...]` is always a plain string
|
||||
(v0.1.5).
|
||||
|
||||
## Writing your own backend (the `TLSBackend` protocol)
|
||||
|
||||
@@ -94,7 +113,10 @@ for the authoritative contract):
|
||||
| `raw_request` | **required** | `async (session, method, url, **kwargs) -> aioweb.Response` | send one request; adapt the client's response into an `aioweb.Response` |
|
||||
| `is_closed` | **required** | `(session) -> bool` | whether the session is closed |
|
||||
| `cookies_for_url` | optional | `(session, url) -> dict` | cookies for `preview()`; defaults to `{}` |
|
||||
| `setup` | optional | `async () -> None` | one-time prep (e.g. fetch a native lib); idempotent |
|
||||
| `set_cookie` | optional | `(session, name, value, domain=None, path="/") -> None` | backs `TLSSession.set_cookie()`; raises `NotImplementedError` if absent |
|
||||
| `get_cookies` | optional | `(session) -> dict` | backs `TLSSession.get_cookies()`; raises `NotImplementedError` if absent |
|
||||
| `clear_cookies` | optional | `(session) -> None` | backs `TLSSession.clear_cookies()`; raises `NotImplementedError` if absent |
|
||||
| `setup` | optional | `async () -> None` | one-time prep (e.g. fetch a native lib); idempotent. `TLSSession` never calls this automatically - a backend needing lazy setup must self-invoke it from its own `raw_request`, as `Noble` does, or the caller must run `await session.setup()` explicitly |
|
||||
| `close` | optional | `async (session) -> None` | close the session; defaults to `await session.close()` |
|
||||
|
||||
`raw_request` receives aioweb-shaped kwargs: the proxy is already resolved into
|
||||
@@ -151,8 +173,9 @@ async with TLSSession(backend=GoTLSBackend("http://localhost:8080")) as s:
|
||||
## Inherited features work unchanged
|
||||
|
||||
aioweb's overwrite/domain/ephemeral/proxy/retry/preview logic operates on plain dicts
|
||||
and never touches the HTTP backend — only the seams do. Every aioweb feature behaves
|
||||
identically on any backend:
|
||||
and never touches the HTTP backend — only the seams do. Header overwrites, domain
|
||||
rewriting, ephemeral headers, proxies, retries, and previews behave identically on
|
||||
any backend:
|
||||
|
||||
```python
|
||||
async with TLSSession(backend=CurlCffi(impersonate="chrome")) as s:
|
||||
@@ -162,12 +185,56 @@ async with TLSSession(backend=CurlCffi(impersonate="chrome")) as s:
|
||||
print(s.preview("GET", "https://internal.local/x").as_curl()) # reflects all of the above
|
||||
```
|
||||
|
||||
Session-default headers are never baked into the underlying client (neither
|
||||
`CurlCffi` nor `Noble` passes `headers=` to their client's constructor) — they flow
|
||||
through aioweb's own per-request `_default_headers` merge instead. That keeps
|
||||
`update_headers()` / `clear_headers()` accurate for both backends: what
|
||||
`get_headers()` and `preview()` report is what actually goes out on the wire, with
|
||||
no stale, already-baked value resurfacing after a clear.
|
||||
|
||||
The mutable cookie API — `set_cookie()` / `get_cookies()` / `clear_cookies()` — is
|
||||
also backend-aware: `CurlCffi` and `Noble` each route it through their own client's
|
||||
cookie store (both expose a `requests`-style `session.cookies` with `set()` /
|
||||
`items()` / `clear()`), so these calls work the same way they do on the base
|
||||
`aioweb.ExtendedSession`, not just `_cookies_for_url()` (used by `preview()`).
|
||||
|
||||
`resp.history` / `resp.redirect_chain` (`[(status, url), ...]`) and `resp.is_redirect`
|
||||
are threaded through from whatever redirect history the backend exposes. **Caveat — the
|
||||
CurlCffi backend has no per-hop history:** `curl_cffi` follows redirects internally in the
|
||||
native curl layer and surfaces only the final URL/status, leaving `Response.history` an
|
||||
empty list (it never populates it). So on `CurlCffi`, `resp.history`/`resp.redirect_chain`
|
||||
are `[]` and `resp.is_redirect` reflects only the final response, even after a redirect —
|
||||
this is a `curl_cffi` limitation, not a bug here, and it differs from the base aiohttp
|
||||
`ExtendedSession` (which does record the hops). The `Noble` backend threads whatever
|
||||
`noble_tls` exposes as its per-response history. If you need the redirect chain, use the
|
||||
base backend or read the final URL.
|
||||
|
||||
## Honesty note
|
||||
|
||||
TLS fingerprinting changes one layer — the TLS/HTTP fingerprint. It does **not** by
|
||||
itself defeat modern bot protection: behavioral analysis, captchas, and JS challenges
|
||||
are separate signals. Use this as one component, not a complete anti-bot solution.
|
||||
|
||||
## Changelog
|
||||
|
||||
### v0.1.8
|
||||
|
||||
- Compressed 4 residual internal-helper docstrings (`_is_timeout_error`,
|
||||
`_noble_timeout_seconds`, `_noble_content`, `_jar_to_dict`) to one-liners.
|
||||
Cosmetic, zero behavior change.
|
||||
|
||||
### v0.1.7
|
||||
|
||||
- **Backends thread whatever redirect history the client exposes.** `CurlCffi.raw_request`
|
||||
and `Noble.raw_request` built their `Response` without `history=`, so
|
||||
`resp.history`/`resp.redirect_chain`/`resp.is_redirect`-after-follow were always empty
|
||||
and aioweb's own `debug=True` "redirect chain:" log line was dead on both TLS backends.
|
||||
Each backend now maps its client's history into aioweb's `(status, url)` tuple shape.
|
||||
**Caveat:** `curl_cffi` never populates `Response.history` (it follows redirects in the
|
||||
native curl layer and surfaces only the final URL/status), so on the `CurlCffi` backend
|
||||
`resp.history` is `[]` even after a redirect — a `curl_cffi` limitation, not addressable
|
||||
here. `Noble` passes through whatever `noble_tls` records.
|
||||
|
||||
## Versioning
|
||||
|
||||
Tagged `vX.Y.Z`. Pin the tag in `requirements.txt`.
|
||||
Releases are tagged `vX.Y.Z`. The install line above pins a release; drop the `@vX.Y.Z` suffix to install the latest unpinned. Pin deliberately for reproducible installs.
|
||||
|
||||
+3
-3
@@ -4,11 +4,11 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "aioweb_tls"
|
||||
version = "0.1.1"
|
||||
description = "TLS-fingerprinting backends for aioweb — curl_cffi / noble_tls ExtendedSession subclasses, config-free, installable."
|
||||
version = "1.0.0"
|
||||
description = "TLS-fingerprinting backends (curl_cffi / noble_tls) for aioweb via one injectable TLSSession, config-free, installable."
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.0",
|
||||
"aioweb @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb.git@v0.1.10",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
||||
@@ -1,28 +1,21 @@
|
||||
"""
|
||||
tls-fingerprinting backends for aioweb
|
||||
|
||||
one session class, TLSSession, takes an injected backend that swaps the HTTP client
|
||||
(and thus the TLS/HTTP fingerprint) while inheriting every aioweb feature — header
|
||||
overwrites, domain rewriting, ephemeral headers, proxies, retries, previews —
|
||||
unchanged.
|
||||
tls-fingerprinting backends for aioweb - see README for usage and the extras contract
|
||||
|
||||
from aioweb_tls import TLSSession, CurlCffi, Noble
|
||||
|
||||
async with TLSSession(backend=CurlCffi(impersonate="chrome")) as s: # [curl] extra
|
||||
resp = await s.request_with_retries("GET", url)
|
||||
|
||||
async with TLSSession(backend=Noble(client="chrome_133")) as s: # [noble] extra
|
||||
await s.setup() # fetch Go lib once
|
||||
resp = await s.request_with_retries("GET", url)
|
||||
|
||||
the tls clients are optional extras, not base deps. importing this package never
|
||||
fails because an extra is missing; the matching RuntimeError is raised only when you
|
||||
construct a backend whose client isn't installed. custom backends implement the
|
||||
TLSBackend protocol and inject the same way.
|
||||
"""
|
||||
|
||||
from importlib.metadata import version, PackageNotFoundError
|
||||
|
||||
from .session import TLSSession
|
||||
from .backends import CurlCffi, Noble
|
||||
from .protocol import TLSBackend
|
||||
|
||||
try:
|
||||
__version__ = version("aioweb_tls")
|
||||
except PackageNotFoundError:
|
||||
__version__ = "0.0.0+unknown"
|
||||
|
||||
__all__ = ["TLSSession", "CurlCffi", "Noble", "TLSBackend"]
|
||||
|
||||
+192
-59
@@ -1,24 +1,41 @@
|
||||
"""
|
||||
tls backends for TLSSession
|
||||
|
||||
each backend is a stateless config+behavior object implementing the TLSBackend
|
||||
protocol (see protocol.py). it owns its own config vocabulary — CurlCffi takes
|
||||
impersonate=, Noble takes client= — so there is no shared kwarg-soup. TLSSession
|
||||
owns the live session object and passes it into every method here.
|
||||
|
||||
the underlying tls clients are optional extras: constructing a backend whose client
|
||||
is not installed raises a clear RuntimeError naming the extra to install. importing
|
||||
this module never fails because an extra is missing.
|
||||
tls backends for TLSSession - stateless config+behavior objects implementing the
|
||||
TLSBackend protocol (see protocol.py); see README for the extras contract
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import logging
|
||||
import math
|
||||
|
||||
import aiohttp
|
||||
from multidict import CIMultiDict
|
||||
from aioweb import Response
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _as_client_error(error: Exception, backend: str) -> aiohttp.ClientError:
|
||||
"""wrap a backend-native network exception (neither is an aiohttp.ClientError) as one"""
|
||||
return aiohttp.ClientError(f"{backend} request failed: {error}")
|
||||
|
||||
|
||||
_TIMEOUT_TEXT = ("timeout", "timed out", "deadline exceeded")
|
||||
|
||||
|
||||
def _is_timeout_error(error: Exception) -> bool:
|
||||
"""whether a backend-native OSError is a timeout, by type name (curl_cffi) or Go error text (noble_tls)"""
|
||||
timeout_type = getattr(error, "__class__", None)
|
||||
if timeout_type is not None and any(base.__name__ == "Timeout" for base in timeout_type.__mro__):
|
||||
return True
|
||||
return any(needle in str(error).lower() for needle in _TIMEOUT_TEXT)
|
||||
|
||||
|
||||
def _as_timeout_error(error: Exception, backend: str) -> aiohttp.ServerTimeoutError:
|
||||
"""wrap a backend-native timeout as aiohttp.ServerTimeoutError, matching aioweb's own contract"""
|
||||
return aiohttp.ServerTimeoutError(f"{backend} request timed out: {error}")
|
||||
|
||||
|
||||
try:
|
||||
from curl_cffi import AsyncSession as _CurlAsyncSession
|
||||
_CURL_ERROR = None
|
||||
@@ -37,21 +54,74 @@ except ImportError as error:
|
||||
|
||||
|
||||
def _coerce_timeout(value):
|
||||
"""turn aioweb's aiohttp ClientTimeout (or a number) into a plain number
|
||||
|
||||
aioweb.request() wraps a numeric timeout in an aiohttp.ClientTimeout before the
|
||||
seam sees it; the tls clients want a number, so unwrap .total when present.
|
||||
"""
|
||||
"""unwrap aioweb's aiohttp.ClientTimeout (or a plain number) to a bare number for the tls clients"""
|
||||
total = getattr(value, "total", value)
|
||||
return total if isinstance(total, (int, float)) else None
|
||||
|
||||
|
||||
def _noble_timeout_seconds(value):
|
||||
"""coerce a raw or wrapped timeout into whole seconds (min 1), rounding up so noble's Go int
|
||||
field never truncates to 0/fails unmarshal; None if uncoercible; shared by both Noble timeout paths"""
|
||||
timeout = _coerce_timeout(value)
|
||||
if timeout is None:
|
||||
return None
|
||||
return max(1, math.ceil(timeout))
|
||||
|
||||
|
||||
def _noble_content(response) -> bytes:
|
||||
"""extract true response bytes from a noble_tls response fetched with is_byte_response, decoding
|
||||
the base64 data-URI in response.text (falls back to a plain utf-8 encode if not a data-URI)"""
|
||||
text = getattr(response, "text", "") or ""
|
||||
if text.startswith("data:") and ";base64," in text:
|
||||
_, _, payload = text.partition(";base64,")
|
||||
return base64.b64decode(payload)
|
||||
return text.encode()
|
||||
|
||||
|
||||
def _flatten_headers(headers) -> CIMultiDict:
|
||||
"""flatten a Go-style map[str][]str header dict (multi-valued headers as a list)
|
||||
into plain str values, joined with ", " per RFC 7230, into a case-insensitive
|
||||
mapping matching aioweb's aiohttp-backed Response.headers"""
|
||||
return CIMultiDict(
|
||||
(key, ", ".join(value) if isinstance(value, list) else value)
|
||||
for key, value in headers.items()
|
||||
)
|
||||
|
||||
|
||||
def _history_entries(history) -> list:
|
||||
"""map a backend-native redirect history into aioweb's `Response(history=...)` shape
|
||||
|
||||
aioweb's own `_raw_request` threads `[(status, url), ...]`; curl_cffi's
|
||||
`Response.history` is `list[dict]` (dict-shaped hop records) and noble_tls's is
|
||||
`list[Response]` (object-shaped, same accessors as the top-level response) - this
|
||||
reads a hop's status/url either way. an unparseable hop is skipped rather than
|
||||
raising, so a redirect-history quirk never breaks the response it's attached to.
|
||||
"""
|
||||
entries = []
|
||||
for hop in history or []:
|
||||
if isinstance(hop, dict):
|
||||
status = hop.get("status_code", hop.get("status"))
|
||||
url = hop.get("url")
|
||||
else:
|
||||
status = getattr(hop, "status_code", getattr(hop, "status", None))
|
||||
url = getattr(hop, "url", None)
|
||||
if status is None or url is None:
|
||||
continue
|
||||
entries.append((status, str(url)))
|
||||
return entries
|
||||
|
||||
|
||||
def _jar_to_dict(session):
|
||||
"""best-effort map of a requests-style cookie jar on session to a plain dict"""
|
||||
"""best-effort map of a requests-style cookie jar on session to a plain dict, feeding preview()
|
||||
only; a jar that fails to iterate degrades to {} rather than raising"""
|
||||
jar = getattr(session, "cookies", None)
|
||||
if not jar:
|
||||
return {}
|
||||
try:
|
||||
# prefer get_dict() where the jar exposes it: items() raises curl_cffi CookieConflict
|
||||
# when the same name lives on two domains, get_dict() flattens instead (mirrors get_cookies)
|
||||
if hasattr(jar, "get_dict"):
|
||||
return jar.get_dict()
|
||||
return {k: v for k, v in jar.items()}
|
||||
except Exception:
|
||||
return {}
|
||||
@@ -62,7 +132,9 @@ class CurlCffi:
|
||||
|
||||
config:
|
||||
impersonate: browser profile to forge (default "chrome"); override per call
|
||||
by passing impersonate= to any request method.
|
||||
via request()/_raw_request (forwards **kwargs) - NOT request_with_retries,
|
||||
whose fixed signature raises TypeError on it. for the retrying path, set
|
||||
the profile on the CurlCffi instance instead.
|
||||
|
||||
requires the [curl] extra (pip install "aioweb_tls[curl]").
|
||||
"""
|
||||
@@ -76,8 +148,15 @@ class CurlCffi:
|
||||
self.impersonate = impersonate
|
||||
|
||||
def create_session(self, headers, timeout, **kwargs):
|
||||
"""build the curl_cffi AsyncSession"""
|
||||
return _CurlAsyncSession(headers=headers, timeout=timeout, **kwargs)
|
||||
"""build the curl_cffi AsyncSession
|
||||
|
||||
deliberately does NOT pass `headers`: curl_cffi bakes a constructor
|
||||
`headers=` into the client and re-merges it under per-request headers,
|
||||
which would desync update_headers()/clear_headers() from what's actually on
|
||||
the wire - aioweb's session-default headers already apply per request via
|
||||
the base's _default_headers merge (see ExtendedSession._create_session).
|
||||
"""
|
||||
return _CurlAsyncSession(timeout=timeout, **kwargs)
|
||||
|
||||
async def raw_request(self, session, method, url, **kwargs) -> Response:
|
||||
"""send via curl_cffi and adapt the result into an aioweb.Response"""
|
||||
@@ -91,26 +170,33 @@ class CurlCffi:
|
||||
if proxy:
|
||||
kwargs["proxy"] = proxy
|
||||
|
||||
try:
|
||||
response = await session.request(method, url, impersonate=impersonate, **kwargs)
|
||||
content = response.content
|
||||
if content is None:
|
||||
content = response.text.encode() if response.text else b""
|
||||
except aiohttp.ClientError:
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
raise
|
||||
except OSError as error:
|
||||
if _is_timeout_error(error):
|
||||
raise _as_timeout_error(error, "curl_cffi") from error
|
||||
raise _as_client_error(error, "curl_cffi") from error
|
||||
content = response.content if response.content is not None else b""
|
||||
return Response(
|
||||
status_code=response.status_code,
|
||||
headers=dict(response.headers),
|
||||
headers=CIMultiDict(response.headers.multi_items()),
|
||||
content=content,
|
||||
url=str(response.url),
|
||||
reason=getattr(response, "reason", None),
|
||||
history=_history_entries(getattr(response, "history", None)),
|
||||
cookies=getattr(response, "cookies", None),
|
||||
)
|
||||
|
||||
def is_closed(self, session) -> bool:
|
||||
"""whether the curl_cffi session is closed
|
||||
|
||||
curl_cffi tracks closed state in the private `_closed` (no public `closed`
|
||||
property), so read that; fall back to a public `closed` if a future version
|
||||
adds one. TLSSession's own `_closed` flag is the primary signal — this is a
|
||||
best-effort backend check for out-of-band closes.
|
||||
curl_cffi tracks this in the private `_closed` (no public `closed` today);
|
||||
falls back to a public `closed` if a future version adds one. best-effort -
|
||||
TLSSession's own `_closed` flag is the primary signal.
|
||||
"""
|
||||
closed = getattr(session, "_closed", None)
|
||||
if closed is None:
|
||||
@@ -121,6 +207,23 @@ class CurlCffi:
|
||||
"""cookies curl_cffi would send for url (best-effort)"""
|
||||
return _jar_to_dict(session)
|
||||
|
||||
def set_cookie(self, session, name, value, domain=None, path="/") -> None:
|
||||
"""set a cookie in curl_cffi's own cookie store"""
|
||||
session.cookies.set(name, value, domain=domain or "", path=path)
|
||||
|
||||
def get_cookies(self, session) -> dict:
|
||||
"""all cookies stored in curl_cffi's cookie store
|
||||
|
||||
uses get_dict() rather than dict(cookies.items()): items() raises curl_cffi
|
||||
CookieConflict when the same name exists on two domains, get_dict() flattens
|
||||
(last value wins) without raising.
|
||||
"""
|
||||
return session.cookies.get_dict()
|
||||
|
||||
def clear_cookies(self, session) -> None:
|
||||
"""clear curl_cffi's cookie store"""
|
||||
session.cookies.clear()
|
||||
|
||||
async def close(self, session) -> None:
|
||||
"""close the curl_cffi session"""
|
||||
await session.close()
|
||||
@@ -132,8 +235,8 @@ class Noble:
|
||||
config:
|
||||
client: noble_tls Client profile (enum or string, default "chrome_133").
|
||||
|
||||
noble_tls downloads a Go shared library on first use; setup() fetches it once
|
||||
(run via TLSSession.setup() or lazily before the first request).
|
||||
downloads a Go shared library on first use; setup() fetches it once (via
|
||||
TLSSession.setup() or lazily before the first request).
|
||||
|
||||
requires the [noble] extra (pip install "aioweb_tls[noble]").
|
||||
"""
|
||||
@@ -150,21 +253,23 @@ class Noble:
|
||||
|
||||
@staticmethod
|
||||
def _resolve_client(client):
|
||||
"""turn a string or Client enum into a noble_tls Client value"""
|
||||
if isinstance(client, str):
|
||||
return getattr(_NobleClient, client.upper())
|
||||
"""turn a string or Client enum into a noble_tls Client value, raising ValueError
|
||||
naming the valid profiles for an unknown string"""
|
||||
if not isinstance(client, str):
|
||||
return client
|
||||
name = client.upper()
|
||||
resolved = getattr(_NobleClient, name, None)
|
||||
if resolved is None:
|
||||
valid = ", ".join(member.name for member in _NobleClient)
|
||||
raise ValueError(f"unknown noble_tls client profile {client!r}; valid profiles: {valid}")
|
||||
return resolved
|
||||
|
||||
async def setup(self) -> None:
|
||||
"""fetch the noble_tls Go shared library once; idempotent and concurrency-safe
|
||||
|
||||
download_if_necessary handles the first-time fetch (no lib present);
|
||||
update_if_necessary refreshes an existing one. try download first so a
|
||||
clean environment works, falling back to update.
|
||||
|
||||
guarded by an asyncio.Lock with a check-lock-recheck so concurrent first
|
||||
requests don't both run the fetch: the fast path returns once _updated is
|
||||
set, and only the first caller through the lock does the work.
|
||||
uses download_if_necessary (falls back to update_if_necessary on older
|
||||
noble_tls). guarded by an asyncio.Lock with a check-lock-recheck so concurrent
|
||||
first requests don't both run the fetch.
|
||||
"""
|
||||
if self._updated:
|
||||
return
|
||||
@@ -174,50 +279,59 @@ class Noble:
|
||||
download = getattr(noble_tls, "download_if_necessary", None)
|
||||
if download is not None:
|
||||
await download()
|
||||
else:
|
||||
elif hasattr(noble_tls, "update_if_necessary"):
|
||||
await noble_tls.update_if_necessary()
|
||||
self._updated = True
|
||||
|
||||
def create_session(self, headers, timeout, **kwargs):
|
||||
"""build the noble_tls Session, honoring session-default headers + timeout
|
||||
"""build the noble_tls Session, honoring the session-default timeout
|
||||
|
||||
noble_tls.Session takes neither headers nor timeout in its constructor, so
|
||||
aioweb's session-default headers (and the coerced timeout) are applied after
|
||||
construction — matching CurlCffi, which passes both through. without this a
|
||||
TLSSession(backend=Noble(...), headers=...) would silently drop the headers.
|
||||
deliberately does NOT bake `headers` into session.headers - same header-baking
|
||||
rationale as CurlCffi.create_session (see its docstring).
|
||||
|
||||
the coerced timeout IS applied here (matching raw_request's per-call path):
|
||||
noble's Go field timeoutSeconds is an int, so an uncoerced sub-second/float
|
||||
session-default would fail Go-side JSON unmarshal on any request that doesn't
|
||||
override it per call; max(1, ceil()) mirrors raw_request's own guard.
|
||||
"""
|
||||
session = noble_tls.Session(client=self.client, **kwargs)
|
||||
if headers:
|
||||
session.headers.update(headers)
|
||||
if timeout is not None:
|
||||
session.timeout_seconds = timeout
|
||||
timeout_seconds = _noble_timeout_seconds(timeout)
|
||||
if timeout_seconds is not None:
|
||||
session.timeout_seconds = timeout_seconds
|
||||
return session
|
||||
|
||||
async def raw_request(self, session, method, url, **kwargs) -> Response:
|
||||
"""send via noble_tls and adapt the result into an aioweb.Response"""
|
||||
await self.setup()
|
||||
|
||||
timeout = _coerce_timeout(kwargs.pop("timeout", None))
|
||||
if timeout is not None:
|
||||
# noble takes whole seconds; round UP so a sub-second timeout (e.g. 0.5)
|
||||
# doesn't truncate to 0 (which would mean no/instant timeout)
|
||||
kwargs["timeout_seconds"] = max(1, math.ceil(timeout))
|
||||
timeout_seconds = _noble_timeout_seconds(kwargs.pop("timeout", None))
|
||||
if timeout_seconds is not None:
|
||||
kwargs["timeout_seconds"] = timeout_seconds
|
||||
|
||||
proxy = kwargs.pop("proxy", None)
|
||||
if proxy:
|
||||
kwargs["proxy"] = proxy
|
||||
|
||||
kwargs.setdefault("is_byte_response", True)
|
||||
|
||||
try:
|
||||
response = await session.execute_request(method=method.upper(), url=url, **kwargs)
|
||||
content = getattr(response, "content", None)
|
||||
if content is None:
|
||||
text = getattr(response, "text", "") or ""
|
||||
content = text.encode()
|
||||
except aiohttp.ClientError:
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
raise
|
||||
except OSError as error:
|
||||
if _is_timeout_error(error):
|
||||
raise _as_timeout_error(error, "noble_tls") from error
|
||||
raise _as_client_error(error, "noble_tls") from error
|
||||
content = _noble_content(response)
|
||||
return Response(
|
||||
status_code=response.status_code,
|
||||
headers=dict(getattr(response, "headers", {}) or {}),
|
||||
headers=_flatten_headers(getattr(response, "headers", {}) or {}),
|
||||
content=content,
|
||||
url=str(getattr(response, "url", url)),
|
||||
reason=getattr(response, "reason", None),
|
||||
history=_history_entries(getattr(response, "history", None)),
|
||||
cookies=getattr(response, "cookies", None),
|
||||
)
|
||||
|
||||
@@ -229,6 +343,25 @@ class Noble:
|
||||
"""cookies noble_tls would send for url (best-effort)"""
|
||||
return _jar_to_dict(session)
|
||||
|
||||
def set_cookie(self, session, name, value, domain=None, path="/") -> None:
|
||||
"""set a cookie in noble_tls's own cookie jar"""
|
||||
session.cookies.set(name, value, domain=domain or "", path=path)
|
||||
|
||||
def get_cookies(self, session) -> dict:
|
||||
"""all cookies stored in noble_tls's cookie jar
|
||||
|
||||
prefers get_dict() when the jar exposes it (flattens cross-domain duplicate
|
||||
names without raising, like curl_cffi); falls back to items() otherwise
|
||||
"""
|
||||
jar = session.cookies
|
||||
if hasattr(jar, "get_dict"):
|
||||
return jar.get_dict()
|
||||
return dict(jar.items())
|
||||
|
||||
def clear_cookies(self, session) -> None:
|
||||
"""clear noble_tls's cookie jar"""
|
||||
session.cookies.clear()
|
||||
|
||||
async def close(self, session) -> None:
|
||||
"""close the noble_tls session if it exposes a close"""
|
||||
close = getattr(session, "close", None)
|
||||
|
||||
@@ -1,42 +1,4 @@
|
||||
"""
|
||||
the tls backend protocol
|
||||
|
||||
a backend is a stateless config+behavior object that teaches TLSSession how to talk
|
||||
to one HTTP client. TLSSession owns the live session object (built by create_session)
|
||||
and passes it into every backend call, so backends hold no per-request state.
|
||||
|
||||
implement this protocol to add a custom backend (e.g. a local Go TLS server); inject
|
||||
it via TLSSession(backend=MyBackend(...)) and it inherits all of aioweb's domain /
|
||||
header / ephemeral / proxy / retry / preview logic unchanged — those operate on plain
|
||||
dicts and never touch the backend.
|
||||
|
||||
required:
|
||||
create_session(headers: dict, timeout, **kwargs) -> session
|
||||
build and return the live client session object. TLSSession stores it as
|
||||
self.session and hands it back to every other method below.
|
||||
|
||||
async raw_request(session, method, url, **kwargs) -> aioweb.Response
|
||||
send one request with `session` and adapt the client's response into an
|
||||
aioweb.Response built from primitives (status_code, headers, content bytes,
|
||||
url, reason). kwargs arrive aioweb-shaped — the base has already resolved the
|
||||
proxy into kwargs["proxy"] and merged headers into kwargs["headers"], and a
|
||||
numeric timeout is wrapped in an aiohttp.ClientTimeout (unwrap .total).
|
||||
|
||||
is_closed(session) -> bool
|
||||
whether `session` is closed.
|
||||
|
||||
optional:
|
||||
cookies_for_url(session, url) -> dict
|
||||
cookies the client would send for url, for preview(). default {} (used when
|
||||
the backend has no introspectable jar).
|
||||
|
||||
async setup() -> None
|
||||
one-time async preparation (e.g. fetch a native lib). called once via
|
||||
TLSSession.setup() and lazily before the first request; make it idempotent.
|
||||
|
||||
async close(session) -> None
|
||||
close `session`. default awaits session.close() if present.
|
||||
"""
|
||||
"""the tls backend protocol - see TLSBackend below and README"""
|
||||
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
|
||||
+53
-25
@@ -1,26 +1,16 @@
|
||||
"""
|
||||
TLSSession — one aioweb session, any tls backend
|
||||
TLSSession - one aioweb session, any tls backend
|
||||
|
||||
TLSSession subclasses aioweb.ExtendedSession and delegates only the four backend
|
||||
seams to an injected backend object (see protocol.py). everything else — header
|
||||
overwrites, domain rewriting, ephemeral headers, proxies, retries, previews — is
|
||||
inherited from aioweb unchanged, because that logic operates on plain dicts and
|
||||
never touches the backend.
|
||||
subclasses aioweb.ExtendedSession, delegating only the four backend seams to an
|
||||
injected backend object (see protocol.py); everything else is inherited unchanged.
|
||||
see README for usage.
|
||||
|
||||
from aioweb_tls import TLSSession, CurlCffi, Noble
|
||||
from aioweb_tls import TLSSession, CurlCffi
|
||||
|
||||
async with TLSSession(backend=CurlCffi(impersonate="chrome")) as s:
|
||||
resp = await s.request_with_retries("GET", "https://tls.peet.ws/api/all")
|
||||
if resp:
|
||||
print(resp.json()["tls"]["ja3"])
|
||||
|
||||
s = TLSSession(backend=Noble(client="chrome_133"))
|
||||
await s.setup() # fetch noble's Go lib once
|
||||
...
|
||||
|
||||
a custom backend (e.g. a local Go TLS server) injects the same way — implement the
|
||||
TLSBackend protocol and pass it as backend=. one TLSSession is the only session
|
||||
class; the backend swaps the wire, not the session.
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -46,12 +36,12 @@ class TLSSession(ExtendedSession):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
async def setup(self) -> None:
|
||||
"""run the backend's one-time setup if it has one (idempotent)
|
||||
"""run the backend's one-time setup if it has one (idempotent, e.g. noble's Go lib fetch)
|
||||
|
||||
delegates to backend.setup() when defined (e.g. noble fetching its Go lib);
|
||||
a no-op for backends without it. also invoked lazily before the first request
|
||||
by backends that guard their own setup, so calling this is optional but lets
|
||||
callers pre-warm at startup.
|
||||
optional to call upfront to pre-warm at startup. TLSSession itself never calls
|
||||
this automatically - a backend needing lazy setup must self-invoke it from its
|
||||
own raw_request, as Noble does; a custom backend that skips this will never run
|
||||
setup() unless the caller calls TLSSession.setup() explicitly.
|
||||
"""
|
||||
setup = getattr(self.backend, "setup", None)
|
||||
if setup is not None:
|
||||
@@ -76,19 +66,57 @@ class TLSSession(ExtendedSession):
|
||||
return cookies_for_url(self.session, url)
|
||||
|
||||
def _is_closed(self) -> bool:
|
||||
"""closed if explicitly closed here or the backend reports it"""
|
||||
"""closed if explicitly closed here, never built, or the backend reports it
|
||||
|
||||
an unbuilt session counts as closed: nothing was opened, nothing to leak.
|
||||
"""
|
||||
if self._closed:
|
||||
return True
|
||||
if self._session is None:
|
||||
return True
|
||||
return self.backend.is_closed(self.session)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# lifecycle — backends close differently, so route through the backend
|
||||
# mutable cookie api - base reaches into aiohttp-only session.cookie_jar, so
|
||||
# TLS backends override these to route through the backend instead
|
||||
|
||||
def set_cookie(self, name, value, domain=None, path="/"):
|
||||
"""set a cookie via the backend's own cookie store"""
|
||||
set_cookie = getattr(self.backend, "set_cookie", None)
|
||||
if set_cookie is None:
|
||||
raise NotImplementedError(
|
||||
f"{type(self.backend).__name__} does not support the mutable cookie api"
|
||||
)
|
||||
set_cookie(self.session, name, value, domain=domain, path=path)
|
||||
|
||||
def get_cookies(self) -> dict:
|
||||
"""all cookies stored in the backend's cookie store"""
|
||||
get_cookies = getattr(self.backend, "get_cookies", None)
|
||||
if get_cookies is None:
|
||||
raise NotImplementedError(
|
||||
f"{type(self.backend).__name__} does not support the mutable cookie api"
|
||||
)
|
||||
return get_cookies(self.session)
|
||||
|
||||
def clear_cookies(self) -> None:
|
||||
"""clear the backend's cookie store"""
|
||||
clear_cookies = getattr(self.backend, "clear_cookies", None)
|
||||
if clear_cookies is None:
|
||||
raise NotImplementedError(
|
||||
f"{type(self.backend).__name__} does not support the mutable cookie api"
|
||||
)
|
||||
clear_cookies(self.session)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# lifecycle - backends close differently, so route through the backend
|
||||
|
||||
async def close(self) -> None:
|
||||
"""close via the backend's close (falls back to session.close)"""
|
||||
"""close via the backend's close (falls back to session.close); no-op if never built"""
|
||||
self._closed = True
|
||||
if self._session is None:
|
||||
return
|
||||
close = getattr(self.backend, "close", None)
|
||||
if close is not None:
|
||||
await close(self.session)
|
||||
await close(self._session)
|
||||
else:
|
||||
await self.session.close()
|
||||
await self._session.close()
|
||||
|
||||
Reference in New Issue
Block a user