Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5eb689ba73 | ||
|
|
87debe8465 | ||
|
|
b2876d005e | ||
|
|
ae4c653ecc | ||
|
|
7ea8ecf888 |
@@ -22,17 +22,17 @@ you want; importing the package never fails because an extra is missing.
|
||||
`requirements.txt` (pick the extra you need):
|
||||
|
||||
```
|
||||
aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.0
|
||||
aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.0
|
||||
aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.0
|
||||
aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.2
|
||||
aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.2
|
||||
aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.2
|
||||
```
|
||||
|
||||
Direct:
|
||||
|
||||
```bash
|
||||
pip install "aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.0"
|
||||
pip install "aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.0"
|
||||
pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.0"
|
||||
pip install "aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.2"
|
||||
pip install "aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.2"
|
||||
pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.2"
|
||||
```
|
||||
|
||||
- `[curl]` → curl_cffi backend · `[noble]` → noble_tls backend · `[all]` → both.
|
||||
@@ -55,8 +55,11 @@ async with TLSSession(backend=CurlCffi(impersonate="chrome"), proxies={"https":
|
||||
print(resp.json()["tls"]["ja3"])
|
||||
```
|
||||
|
||||
- `CurlCffi(impersonate="chrome")` sets the forged profile; override per call by
|
||||
passing `impersonate=` to any request method.
|
||||
- `CurlCffi(impersonate="chrome")` sets the forged profile; override it per call by
|
||||
passing `impersonate=` to the low-level `request()` (which forwards `**kwargs` to the
|
||||
backend). `request_with_retries` has a fixed signature and does **not** accept extra
|
||||
backend kwargs — passing `impersonate=` there raises `TypeError`; set the profile on
|
||||
the `CurlCffi` instance for the retrying path.
|
||||
- curl_cffi forges JA3/JA4 + HTTP/2 fingerprints via the bundled curl-impersonate binary.
|
||||
|
||||
## noble backend
|
||||
@@ -73,8 +76,8 @@ async with TLSSession(backend=Noble(client="chrome_133")) as s:
|
||||
|
||||
- `Noble(client="chrome_133")` — accepts a `noble_tls.Client` enum or a string name.
|
||||
- noble_tls downloads a Go shared library on first use. `await s.setup()` fetches it
|
||||
once at startup; if you skip it, the first request fetches it lazily (guarded to run
|
||||
once).
|
||||
once at startup; if you skip it, the first request fetches it lazily. The fetch is
|
||||
guarded by a lock, so even concurrent first requests download it exactly once.
|
||||
|
||||
## Writing your own backend (the `TLSBackend` protocol)
|
||||
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "aioweb_tls"
|
||||
version = "0.1.0"
|
||||
version = "0.1.2"
|
||||
description = "TLS-fingerprinting backends for aioweb — curl_cffi / noble_tls ExtendedSession subclasses, config-free, installable."
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
|
||||
@@ -11,7 +11,9 @@ is not installed raises a clear RuntimeError naming the extra to install. import
|
||||
this module never fails because an extra is missing.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
|
||||
from aioweb import Response
|
||||
|
||||
@@ -60,7 +62,10 @@ class CurlCffi:
|
||||
|
||||
config:
|
||||
impersonate: browser profile to forge (default "chrome"); override per call
|
||||
by passing impersonate= to any request method.
|
||||
by passing impersonate= to the low-level request()/_raw_request path,
|
||||
which forwards **kwargs to the backend. NOT request_with_retries — its
|
||||
signature is fixed (no **kwargs) and would raise TypeError. for a
|
||||
per-call profile under retries, set it on the CurlCffi instance instead.
|
||||
|
||||
requires the [curl] extra (pip install "aioweb_tls[curl]").
|
||||
"""
|
||||
@@ -103,8 +108,17 @@ class CurlCffi:
|
||||
)
|
||||
|
||||
def is_closed(self, session) -> bool:
|
||||
"""whether the curl_cffi session is closed"""
|
||||
return bool(getattr(session, "closed", False))
|
||||
"""whether the curl_cffi session is closed
|
||||
|
||||
curl_cffi tracks closed state in the private `_closed` (no public `closed`
|
||||
property), so read that; fall back to a public `closed` if a future version
|
||||
adds one. TLSSession's own `_closed` flag is the primary signal — this is a
|
||||
best-effort backend check for out-of-band closes.
|
||||
"""
|
||||
closed = getattr(session, "_closed", None)
|
||||
if closed is None:
|
||||
closed = getattr(session, "closed", False)
|
||||
return bool(closed)
|
||||
|
||||
def cookies_for_url(self, session, url) -> dict:
|
||||
"""cookies curl_cffi would send for url (best-effort)"""
|
||||
@@ -135,6 +149,7 @@ class Noble:
|
||||
) from _NOBLE_ERROR
|
||||
self.client = self._resolve_client(client)
|
||||
self._updated = False
|
||||
self._setup_lock = asyncio.Lock()
|
||||
|
||||
@staticmethod
|
||||
def _resolve_client(client):
|
||||
@@ -144,12 +159,19 @@ class Noble:
|
||||
return client
|
||||
|
||||
async def setup(self) -> None:
|
||||
"""fetch the noble_tls Go shared library once; idempotent
|
||||
"""fetch the noble_tls Go shared library once; idempotent and concurrency-safe
|
||||
|
||||
download_if_necessary handles the first-time fetch (no lib present);
|
||||
update_if_necessary refreshes an existing one. try download first so a
|
||||
clean environment works, falling back to update.
|
||||
|
||||
guarded by an asyncio.Lock with a check-lock-recheck so concurrent first
|
||||
requests don't both run the fetch: the fast path returns once _updated is
|
||||
set, and only the first caller through the lock does the work.
|
||||
"""
|
||||
if self._updated:
|
||||
return
|
||||
async with self._setup_lock:
|
||||
if self._updated:
|
||||
return
|
||||
download = getattr(noble_tls, "download_if_necessary", None)
|
||||
@@ -160,8 +182,19 @@ class Noble:
|
||||
self._updated = True
|
||||
|
||||
def create_session(self, headers, timeout, **kwargs):
|
||||
"""build the noble_tls Session"""
|
||||
return noble_tls.Session(client=self.client, **kwargs)
|
||||
"""build the noble_tls Session, honoring session-default headers + timeout
|
||||
|
||||
noble_tls.Session takes neither headers nor timeout in its constructor, so
|
||||
aioweb's session-default headers (and the coerced timeout) are applied after
|
||||
construction — matching CurlCffi, which passes both through. without this a
|
||||
TLSSession(backend=Noble(...), headers=...) would silently drop the headers.
|
||||
"""
|
||||
session = noble_tls.Session(client=self.client, **kwargs)
|
||||
if headers:
|
||||
session.headers.update(headers)
|
||||
if timeout is not None:
|
||||
session.timeout_seconds = timeout
|
||||
return session
|
||||
|
||||
async def raw_request(self, session, method, url, **kwargs) -> Response:
|
||||
"""send via noble_tls and adapt the result into an aioweb.Response"""
|
||||
@@ -169,7 +202,9 @@ class Noble:
|
||||
|
||||
timeout = _coerce_timeout(kwargs.pop("timeout", None))
|
||||
if timeout is not None:
|
||||
kwargs["timeout_seconds"] = int(timeout)
|
||||
# noble takes whole seconds; round UP so a sub-second timeout (e.g. 0.5)
|
||||
# doesn't truncate to 0 (which would mean no/instant timeout)
|
||||
kwargs["timeout_seconds"] = max(1, math.ceil(timeout))
|
||||
|
||||
proxy = kwargs.pop("proxy", None)
|
||||
if proxy:
|
||||
|
||||
Reference in New Issue
Block a user