From 1a438640fc64063dda2405d98a4eafb81897ca6a Mon Sep 17 00:00:00 2001 From: disqualifier Date: Wed, 24 Jun 2026 18:43:52 -0400 Subject: [PATCH] init: tls-fingerprinting backends for aioweb Signed-off-by: disqualifier --- .gitignore | 15 +++++ README.md | 164 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ae590bc --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +# claude +CLAUDE.md + +# python +__pycache__/ +*.py[cod] +*.egg-info/ +build/ +dist/ +.eggs/ + +# env +.venv/ +venv/ +.env diff --git a/README.md b/README.md new file mode 100644 index 0000000..793b223 --- /dev/null +++ b/README.md @@ -0,0 +1,164 @@ +# aioweb_tls + +TLS-fingerprinting backends for [aioweb](https://git.rethinkstudios.io/rethink-public/aioweb). +One session class — `TLSSession` — takes an **injected backend** that swaps the HTTP +client (and thus the TLS/HTTP fingerprint), while inheriting every aioweb feature — +header overwrites, domain rewriting, ephemeral headers, proxies, retries, previews — +unchanged. The backend swaps the wire, not the session. + +```python +from aioweb_tls import TLSSession, CurlCffi, Noble # the two bundled backends + +TLSSession(backend=CurlCffi(impersonate="chrome")) +TLSSession(backend=Noble(client="chrome_133")) +TLSSession(backend=MyBackend(...)) # or one you write — see "Writing your own backend" +``` + +## Install + +Depends on `aioweb`. The TLS clients are **optional extras** — install the backend you +want; importing the package never fails because an extra is missing. + +`requirements.txt`: + +``` +aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.0 +``` + +Direct: + +```bash +pip install "aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.0" +pip install "aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.0" +``` + +Constructing a backend whose client isn't installed raises a clear `RuntimeError` +naming the extra — the error fires at construction, never at import. + +## curl_cffi backend + +```python +from aioweb_tls import TLSSession, CurlCffi + +async with TLSSession(backend=CurlCffi(impersonate="chrome"), proxies={"https": "http://..."}) as s: + resp = await s.request_with_retries("GET", "https://tls.peet.ws/api/all") + if resp: # FailureResponse is falsy + print(resp.json()["tls"]["ja3"]) +``` + +- `CurlCffi(impersonate="chrome")` sets the forged profile; override per call by + passing `impersonate=` to any request method. +- curl_cffi forges JA3/JA4 + HTTP/2 fingerprints via the bundled curl-impersonate binary. + +## noble backend + +```python +from aioweb_tls import TLSSession, Noble + +async with TLSSession(backend=Noble(client="chrome_133")) as s: + await s.setup() # fetch noble's Go shared lib once (network) + resp = await s.request_with_retries("GET", "https://tls.peet.ws/api/all") + if resp: + print(resp.json()["tls"]["ja3"]) +``` + +- `Noble(client="chrome_133")` — accepts a `noble_tls.Client` enum or a string name. +- noble_tls downloads a Go shared library on first use. `await s.setup()` fetches it + once at startup; if you skip it, the first request fetches it lazily (guarded to run + once). + +## Writing your own backend (the `TLSBackend` protocol) + +**aioweb_tls ships exactly two backends — `CurlCffi` and `Noble` — plus the +`TLSBackend` protocol for writing your own.** There is no Go, remote, or other +bundled backend; anything beyond curl/noble is something you implement against the +protocol. `TLSSession` itself is backend-agnostic, so a backend you write injects +exactly like the built-ins (`TLSSession(backend=YourBackend(...))`) and inherits +every aioweb feature for free. + +To add a backend, write a small object satisfying `TLSBackend` (see `protocol.py` +for the authoritative contract): + +| member | required? | signature | purpose | +|---|---|---|---| +| `create_session` | **required** | `(headers, timeout, **kwargs) -> session` | build the client object `TLSSession` stores as `self.session` | +| `raw_request` | **required** | `async (session, method, url, **kwargs) -> aioweb.Response` | send one request; adapt the client's response into an `aioweb.Response` | +| `is_closed` | **required** | `(session) -> bool` | whether the session is closed | +| `cookies_for_url` | optional | `(session, url) -> dict` | cookies for `preview()`; defaults to `{}` | +| `setup` | optional | `async () -> None` | one-time prep (e.g. fetch a native lib); idempotent | +| `close` | optional | `async (session) -> None` | close the session; defaults to `await session.close()` | + +`raw_request` receives aioweb-shaped kwargs: the proxy is already resolved into +`kwargs["proxy"]`, headers are merged into `kwargs["headers"]`, and a numeric timeout +is wrapped in an `aiohttp.ClientTimeout` (unwrap `.total`). + +### Example — a backend you would write (not included) + +The class below is **illustrative template code you implement yourself**, shown to +make the protocol concrete. It is **not** part of aioweb_tls — there is no +`from aioweb_tls import GoTLSBackend`. It sketches one case (talking to a local Go +TLS service over HTTP) so the shape of a conformant backend is clear: + +```python +# example: a backend YOU write to drive a local Go TLS service. +# NOT shipped by aioweb_tls — this is a TLSBackend-conformance template. +from aioweb import Response +from aioweb_tls import TLSSession + +class GoTLSBackend: + """example user-written backend: drives a local Go tls-server over HTTP""" + + def __init__(self, endpoint: str): + self.endpoint = endpoint + + def create_session(self, headers, timeout, **kwargs): + # build/return whatever client object raw_request will use + import aiohttp + return aiohttp.ClientSession(headers=headers) + + async def raw_request(self, session, method, url, **kwargs) -> Response: + # kwargs arrive aioweb-shaped: proxy in kwargs["proxy"], headers merged, + # numeric timeout wrapped in an aiohttp.ClientTimeout (unwrap .total). + payload = {"method": method, "url": url, "headers": kwargs.get("headers", {})} + async with session.post(self.endpoint, json=payload) as r: + body = await r.json() + return Response( + status_code=body["status"], headers=body["headers"], + content=body["body"].encode(), url=url, reason=body.get("reason"), + ) + + def is_closed(self, session) -> bool: + return session.closed + + async def close(self, session) -> None: + await session.close() + +# inject your backend — same shape as CurlCffi / Noble, all aioweb features inherited +async with TLSSession(backend=GoTLSBackend("http://localhost:8080")) as s: + s.overwrite_domain("internal.local", "127.0.0.1") + resp = await s.request_with_retries("GET", "https://internal.local/x") +``` + +## Inherited features work unchanged + +aioweb's overwrite/domain/ephemeral/proxy/retry/preview logic operates on plain dicts +and never touches the HTTP backend — only the seams do. Every aioweb feature behaves +identically on any backend: + +```python +async with TLSSession(backend=CurlCffi(impersonate="chrome")) as s: + s.overwrite_domain("internal.local", "127.0.0.1") + s.set_ephemeral("X-Time", lambda: str(time.time())) + s.overwrite_inject(True) + print(s.preview("GET", "https://internal.local/x").as_curl()) # reflects all of the above +``` + +## Honesty note + +TLS fingerprinting changes one layer — the TLS/HTTP fingerprint. It does **not** by +itself defeat modern bot protection: behavioral analysis, captchas, and JS challenges +are separate signals. Use this as one component, not a complete anti-bot solution. + +## Versioning + +Tagged `vX.Y.Z`. Pin the tag in `requirements.txt`.