From eb7745ae9b53501d00fdf007f12c69b138763b38 Mon Sep 17 00:00:00 2001 From: disqualifier Date: Thu, 2 Jul 2026 17:09:11 -0400 Subject: [PATCH] fix: seam mutable cookie api, byte-safe noble bodies, drop header baking, coerce noble session timeout (v0.1.4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TLSSession.set_cookie/get_cookies/clear_cookies crashed with AttributeError on both backends (they reached into self.session.cookie_jar, which curl_cffi and noble_tls sessions don't have); both backends now route the mutable cookie api through their own requests-style session.cookies store. Noble.raw_request now requests is_byte_response=True and decodes the resulting base64 data-URI body, since noble_tls's default text response silently corrupts any binary payload (image/zip/pdf) via lossy UTF-8 decoding on the Go side. CurlCffi/Noble.create_session no longer bake session-default headers into the underlying client; baking caused clear_headers()/get_headers() to lie about what's actually still on the wire (a credential-leak divergence from the aiohttp base, which never bakes). Headers flow through aioweb's per-request merge only, matching the base's documented contract. Noble.create_session now applies the same max(1, ceil()) timeout coercion raw_request already had (extracted into a shared _noble_timeout_seconds helper) — without it, a sub-second/float session-default timeout made every request fail Go-side JSON unmarshal. README corrected: dropped the 'every aioweb feature behaves identically' overclaim re: cookies, documented the binary-body handling and the no-header-baking rationale, bumped install pins to v0.1.4. Signed-off-by: disqualifier --- README.md | 40 ++++++++++--- pyproject.toml | 2 +- src/aioweb_tls/backends.py | 116 ++++++++++++++++++++++++++++++------- src/aioweb_tls/protocol.py | 9 +++ src/aioweb_tls/session.py | 32 ++++++++++ 5 files changed, 169 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 69c804e..169eed0 100644 --- a/README.md +++ b/README.md @@ -22,17 +22,17 @@ you want; importing the package never fails because an extra is missing. `requirements.txt` (pick the extra you need): ``` -aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.3 -aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.3 -aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.3 +aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.4 +aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.4 +aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.4 ``` Direct: ```bash -pip install "aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.3" -pip install "aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.3" -pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.3" +pip install "aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.4" +pip install "aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.4" +pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.4" ``` - `[curl]` → curl_cffi backend · `[noble]` → noble_tls backend · `[all]` → both. @@ -44,7 +44,7 @@ pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-publi Constructing a backend whose client isn't installed raises that `RuntimeError` at construction, never at import. -Drop the `@v0.1.3` suffix from the line above to install the latest unpinned. +Drop the `@v0.1.4` suffix from the line above to install the latest unpinned. ## curl_cffi backend @@ -80,6 +80,11 @@ async with TLSSession(backend=Noble(client="chrome_133")) as s: - noble_tls downloads a Go shared library on first use. `await s.setup()` fetches it once at startup; if you skip it, the first request fetches it lazily. The fetch is guarded by a lock, so even concurrent first requests download it exactly once. +- Binary bodies (images, zips, PDFs, protobuf) round-trip as true bytes: noble_tls + returns response bodies as a UTF-8 JSON string by default, which mangles non-UTF-8 + bytes (`U+FFFD` replacement, wrong length) even on a 200 response — the Noble + backend always requests `is_byte_response=True` and decodes the resulting + base64 data-URI back into raw bytes, so `resp.content` is never lossy. ## Writing your own backend (the `TLSBackend` protocol) @@ -99,6 +104,9 @@ for the authoritative contract): | `raw_request` | **required** | `async (session, method, url, **kwargs) -> aioweb.Response` | send one request; adapt the client's response into an `aioweb.Response` | | `is_closed` | **required** | `(session) -> bool` | whether the session is closed | | `cookies_for_url` | optional | `(session, url) -> dict` | cookies for `preview()`; defaults to `{}` | +| `set_cookie` | optional | `(session, name, value, domain=None, path="/") -> None` | backs `TLSSession.set_cookie()`; raises `NotImplementedError` if absent | +| `get_cookies` | optional | `(session) -> dict` | backs `TLSSession.get_cookies()`; raises `NotImplementedError` if absent | +| `clear_cookies` | optional | `(session) -> None` | backs `TLSSession.clear_cookies()`; raises `NotImplementedError` if absent | | `setup` | optional | `async () -> None` | one-time prep (e.g. fetch a native lib); idempotent | | `close` | optional | `async (session) -> None` | close the session; defaults to `await session.close()` | @@ -156,8 +164,9 @@ async with TLSSession(backend=GoTLSBackend("http://localhost:8080")) as s: ## Inherited features work unchanged aioweb's overwrite/domain/ephemeral/proxy/retry/preview logic operates on plain dicts -and never touches the HTTP backend — only the seams do. Every aioweb feature behaves -identically on any backend: +and never touches the HTTP backend — only the seams do. Header overwrites, domain +rewriting, ephemeral headers, proxies, retries, and previews behave identically on +any backend: ```python async with TLSSession(backend=CurlCffi(impersonate="chrome")) as s: @@ -167,6 +176,19 @@ async with TLSSession(backend=CurlCffi(impersonate="chrome")) as s: print(s.preview("GET", "https://internal.local/x").as_curl()) # reflects all of the above ``` +Session-default headers are never baked into the underlying client (neither +`CurlCffi` nor `Noble` passes `headers=` to their client's constructor) — they flow +through aioweb's own per-request `_default_headers` merge instead. That keeps +`update_headers()` / `clear_headers()` accurate for both backends: what +`get_headers()` and `preview()` report is what actually goes out on the wire, with +no stale, already-baked value resurfacing after a clear. + +The mutable cookie API — `set_cookie()` / `get_cookies()` / `clear_cookies()` — is +also backend-aware: `CurlCffi` and `Noble` each route it through their own client's +cookie store (both expose a `requests`-style `session.cookies` with `set()` / +`items()` / `clear()`), so these calls work the same way they do on the base +`aioweb.ExtendedSession`, not just `_cookies_for_url()` (used by `preview()`). + ## Honesty note TLS fingerprinting changes one layer — the TLS/HTTP fingerprint. It does **not** by diff --git a/pyproject.toml b/pyproject.toml index 6ded982..1a4038a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "aioweb_tls" -version = "0.1.3" +version = "0.1.4" description = "TLS-fingerprinting backends (curl_cffi / noble_tls) for aioweb via one injectable TLSSession, config-free, installable." requires-python = ">=3.10" dependencies = [ diff --git a/src/aioweb_tls/backends.py b/src/aioweb_tls/backends.py index 686efdc..b9ae602 100644 --- a/src/aioweb_tls/backends.py +++ b/src/aioweb_tls/backends.py @@ -12,6 +12,7 @@ this module never fails because an extra is missing. """ import asyncio +import base64 import logging import math @@ -59,6 +60,39 @@ def _coerce_timeout(value): return total if isinstance(total, (int, float)) else None +def _noble_timeout_seconds(value): + """coerce a raw or wrapped timeout into whole seconds for noble's Go int field + + single source of truth for both of Noble's timeout paths (create_session's + session-default and raw_request's per-call override): noble's Go field + timeoutSeconds is an int, so a sub-second float (e.g. 0.5) truncates to 0 — which + Go reads as no/instant timeout — and any non-integer value fails Go-side JSON + unmarshal outright. rounds UP so a sub-second timeout still waits at least 1s + instead of truncating away. returns None when there is no timeout to coerce. + """ + timeout = _coerce_timeout(value) + if timeout is None: + return None + return max(1, math.ceil(timeout)) + + +def _noble_content(response) -> bytes: + """extract true response bytes from a noble_tls response fetched with is_byte_response + + with is_byte_response=True, noble_tls's Go side returns the body as a + `data:;base64,` URI string in response.text (the whole Go response is + UTF-8-decoded before JSON parsing, so raw bytes have to travel as base64 to survive + that round trip) — response.content is useless here since it is just + `self.text.encode()`, re-encoding the data-URI string itself rather than decoding it. + falls back to a plain utf-8 encode if the body isn't a data-URI (e.g. an error body). + """ + text = getattr(response, "text", "") or "" + if text.startswith("data:") and ";base64," in text: + _, _, payload = text.partition(";base64,") + return base64.b64decode(payload) + return text.encode() + + def _jar_to_dict(session): """best-effort map of a requests-style cookie jar on session to a plain dict @@ -97,8 +131,17 @@ class CurlCffi: self.impersonate = impersonate def create_session(self, headers, timeout, **kwargs): - """build the curl_cffi AsyncSession""" - return _CurlAsyncSession(headers=headers, timeout=timeout, **kwargs) + """build the curl_cffi AsyncSession + + deliberately does NOT pass `headers` to AsyncSession: curl_cffi bakes a + constructor `headers=` into the client and re-merges it under whatever + per-request headers() sends, so update_headers()/clear_headers() would stop + matching what's actually on the wire (aioweb's session-default headers are + already applied per request by the base's _default_headers merge — see + aioweb.ExtendedSession._create_session's docstring for why baking breaks the + mutable header api). + """ + return _CurlAsyncSession(timeout=timeout, **kwargs) async def raw_request(self, session, method, url, **kwargs) -> Response: """send via curl_cffi and adapt the result into an aioweb.Response""" @@ -150,6 +193,18 @@ class CurlCffi: """cookies curl_cffi would send for url (best-effort)""" return _jar_to_dict(session) + def set_cookie(self, session, name, value, domain=None, path="/") -> None: + """set a cookie in curl_cffi's own cookie store""" + session.cookies.set(name, value, domain=domain or "", path=path) + + def get_cookies(self, session) -> dict: + """all cookies stored in curl_cffi's cookie store""" + return dict(session.cookies.items()) + + def clear_cookies(self, session) -> None: + """clear curl_cffi's cookie store""" + session.cookies.clear() + async def close(self, session) -> None: """close the curl_cffi session""" await session.close() @@ -208,34 +263,46 @@ class Noble: self._updated = True def create_session(self, headers, timeout, **kwargs): - """build the noble_tls Session, honoring session-default headers + timeout + """build the noble_tls Session, honoring the session-default timeout - noble_tls.Session takes neither headers nor timeout in its constructor, so - aioweb's session-default headers (and the coerced timeout) are applied after - construction — matching CurlCffi, which passes both through. without this a - TLSSession(backend=Noble(...), headers=...) would silently drop the headers. + noble_tls.Session takes neither headers nor timeout in its constructor. + deliberately does NOT bake `headers` into session.headers: aioweb's + session-default headers are already applied per request by the base's + _default_headers merge, and baking them here would make + update_headers()/clear_headers() stop matching what's actually on the wire + (see aioweb.ExtendedSession._create_session's docstring, and CurlCffi.create_session + above for the same rationale) — the per-request merge means headers are never + silently dropped, contrary to what this docstring used to claim. + + the coerced timeout IS applied here (matching raw_request's per-call path): noble's + Go field timeoutSeconds is an int, so a raw sub-second/float session-default (e.g. + timeout=7.5) would fail Go-side JSON unmarshal on every request that doesn't + override it per call. max(1, ceil()) mirrors the guard raw_request already has. """ session = noble_tls.Session(client=self.client, **kwargs) - if headers: - session.headers.update(headers) - if timeout is not None: - session.timeout_seconds = timeout + timeout_seconds = _noble_timeout_seconds(timeout) + if timeout_seconds is not None: + session.timeout_seconds = timeout_seconds return session async def raw_request(self, session, method, url, **kwargs) -> Response: """send via noble_tls and adapt the result into an aioweb.Response""" await self.setup() - timeout = _coerce_timeout(kwargs.pop("timeout", None)) - if timeout is not None: - # noble takes whole seconds; round UP so a sub-second timeout (e.g. 0.5) - # doesn't truncate to 0 (which would mean no/instant timeout) - kwargs["timeout_seconds"] = max(1, math.ceil(timeout)) + timeout_seconds = _noble_timeout_seconds(kwargs.pop("timeout", None)) + if timeout_seconds is not None: + kwargs["timeout_seconds"] = timeout_seconds proxy = kwargs.pop("proxy", None) if proxy: kwargs["proxy"] = proxy + # force byte-safe transport: without this, noble_tls's Go side returns the body + # as a plain UTF-8 JSON string, replacing invalid bytes with U+FFFD — silently + # corrupting any binary payload (image/zip/pdf) even though the request succeeds + # with status 200. see _noble_content() for how the byte-safe body is decoded back. + kwargs.setdefault("is_byte_response", True) + try: response = await session.execute_request(method=method.upper(), url=url, **kwargs) except aiohttp.ClientError: @@ -246,10 +313,7 @@ class Noble: # noble_tls's TLSClientException subclasses IOError (== OSError); translate # the native network error, narrowed from bare Exception so a real bug surfaces raise _as_client_error(error, "noble_tls") from error - content = getattr(response, "content", None) - if content is None: - text = getattr(response, "text", "") or "" - content = text.encode() + content = _noble_content(response) return Response( status_code=response.status_code, headers=dict(getattr(response, "headers", {}) or {}), @@ -267,6 +331,18 @@ class Noble: """cookies noble_tls would send for url (best-effort)""" return _jar_to_dict(session) + def set_cookie(self, session, name, value, domain=None, path="/") -> None: + """set a cookie in noble_tls's own cookie jar""" + session.cookies.set(name, value, domain=domain or "", path=path) + + def get_cookies(self, session) -> dict: + """all cookies stored in noble_tls's cookie jar""" + return dict(session.cookies.items()) + + def clear_cookies(self, session) -> None: + """clear noble_tls's cookie jar""" + session.cookies.clear() + async def close(self, session) -> None: """close the noble_tls session if it exposes a close""" close = getattr(session, "close", None) diff --git a/src/aioweb_tls/protocol.py b/src/aioweb_tls/protocol.py index a5e05e0..cfb96e6 100644 --- a/src/aioweb_tls/protocol.py +++ b/src/aioweb_tls/protocol.py @@ -30,6 +30,15 @@ optional: cookies the client would send for url, for preview(). default {} (used when the backend has no introspectable jar). + set_cookie(session, name, value, domain=None, path="/") -> None + get_cookies(session) -> dict + clear_cookies(session) -> None + the mutable cookie api TLSSession.set_cookie/get_cookies/clear_cookies + delegate to. the aiohttp-only base implementations reach into + session.cookie_jar, which TLS backends don't have, so a backend without + these raises NotImplementedError from TLSSession rather than crashing with + an AttributeError on a private aiohttp attribute. + async setup() -> None one-time async preparation (e.g. fetch a native lib). called once via TLSSession.setup() and lazily before the first request; make it idempotent. diff --git a/src/aioweb_tls/session.py b/src/aioweb_tls/session.py index 218a486..28ffe63 100644 --- a/src/aioweb_tls/session.py +++ b/src/aioweb_tls/session.py @@ -81,6 +81,38 @@ class TLSSession(ExtendedSession): return True return self.backend.is_closed(self.session) + # ------------------------------------------------------------------------- + # mutable cookie api — the base's set_cookie/get_cookies/clear_cookies reach + # into self.session.cookie_jar (aiohttp-only), so TLS backends override them + # to route through the backend instead of crashing with AttributeError + + def set_cookie(self, name, value, domain=None, path="/"): + """set a cookie via the backend's own cookie store""" + set_cookie = getattr(self.backend, "set_cookie", None) + if set_cookie is None: + raise NotImplementedError( + f"{type(self.backend).__name__} does not support the mutable cookie api" + ) + set_cookie(self.session, name, value, domain=domain, path=path) + + def get_cookies(self) -> dict: + """all cookies stored in the backend's cookie store""" + get_cookies = getattr(self.backend, "get_cookies", None) + if get_cookies is None: + raise NotImplementedError( + f"{type(self.backend).__name__} does not support the mutable cookie api" + ) + return get_cookies(self.session) + + def clear_cookies(self) -> None: + """clear the backend's cookie store""" + clear_cookies = getattr(self.backend, "clear_cookies", None) + if clear_cookies is None: + raise NotImplementedError( + f"{type(self.backend).__name__} does not support the mutable cookie api" + ) + clear_cookies(self.session) + # ------------------------------------------------------------------------- # lifecycle — backends close differently, so route through the backend