diff --git a/README.md b/README.md index 169eed0..692fed0 100644 --- a/README.md +++ b/README.md @@ -22,17 +22,17 @@ you want; importing the package never fails because an extra is missing. `requirements.txt` (pick the extra you need): ``` -aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.4 -aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.4 -aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.4 +aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5 +aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5 +aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5 ``` Direct: ```bash -pip install "aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.4" -pip install "aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.4" -pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.4" +pip install "aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5" +pip install "aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5" +pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5" ``` - `[curl]` → curl_cffi backend · `[noble]` → noble_tls backend · `[all]` → both. @@ -44,7 +44,7 @@ pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-publi Constructing a backend whose client isn't installed raises that `RuntimeError` at construction, never at import. -Drop the `@v0.1.4` suffix from the line above to install the latest unpinned. +Drop the `@v0.1.5` suffix from the line above to install the latest unpinned. ## curl_cffi backend @@ -63,6 +63,10 @@ async with TLSSession(backend=CurlCffi(impersonate="chrome"), proxies={"https": backend kwargs — passing `impersonate=` there raises `TypeError`; set the profile on the `CurlCffi` instance for the retrying path. - curl_cffi forges JA3/JA4 + HTTP/2 fingerprints via the bundled curl-impersonate binary. +- A wire-level timeout raises `aiohttp.ServerTimeoutError` (matching aioweb's own + contract), not a generic `aiohttp.ClientError` — both backends detect their + native timeout (curl_cffi's `Timeout` type, noble_tls's Go-side timeout text) and + re-wrap it before the fallback client-error path (v0.1.5). ## noble backend @@ -76,7 +80,8 @@ async with TLSSession(backend=Noble(client="chrome_133")) as s: print(resp.json()["tls"]["ja3"]) ``` -- `Noble(client="chrome_133")` — accepts a `noble_tls.Client` enum or a string name. +- `Noble(client="chrome_133")` — accepts a `noble_tls.Client` enum or a string name; + an unknown string raises `ValueError` listing the valid profile names (v0.1.5). - noble_tls downloads a Go shared library on first use. `await s.setup()` fetches it once at startup; if you skip it, the first request fetches it lazily. The fetch is guarded by a lock, so even concurrent first requests download it exactly once. @@ -85,6 +90,10 @@ async with TLSSession(backend=Noble(client="chrome_133")) as s: bytes (`U+FFFD` replacement, wrong length) even on a 200 response — the Noble backend always requests `is_byte_response=True` and decodes the resulting base64 data-URI back into raw bytes, so `resp.content` is never lossy. +- Multi-valued response headers (e.g. two `Set-Cookie` lines) arrive from noble_tls's + Go side as a Python list — the Noble backend flattens them to a single + comma-joined string per RFC 7230, so `resp.headers[...]` is always a plain string + (v0.1.5). ## Writing your own backend (the `TLSBackend` protocol) diff --git a/pyproject.toml b/pyproject.toml index 1a4038a..4b4d767 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "aioweb_tls" -version = "0.1.4" +version = "0.1.5" description = "TLS-fingerprinting backends (curl_cffi / noble_tls) for aioweb via one injectable TLSSession, config-free, installable." requires-python = ">=3.10" dependencies = [ diff --git a/src/aioweb_tls/backends.py b/src/aioweb_tls/backends.py index b9ae602..6ff37b5 100644 --- a/src/aioweb_tls/backends.py +++ b/src/aioweb_tls/backends.py @@ -25,14 +25,40 @@ log = logging.getLogger(__name__) def _as_client_error(error: Exception, backend: str) -> aiohttp.ClientError: """wrap a backend-native network exception as an aiohttp.ClientError - aioweb's request() only re-wraps aiohttp.ClientError; curl_cffi raises - RequestException(OSError) and noble_tls raises TLSClientException(IOError), neither - of which is an aiohttp.ClientError. translating here gives TLS backends the same - typed failure contract as the aiohttp path on the bare request() route. + curl_cffi raises RequestException(OSError), noble_tls raises + TLSClientException(IOError) — neither is an aiohttp.ClientError, which is all + aioweb's request() re-wraps; translating here gives both the same typed contract. """ return aiohttp.ClientError(f"{backend} request failed: {error}") +_TIMEOUT_TEXT = ("timeout", "timed out", "deadline exceeded") + + +def _is_timeout_error(error: Exception) -> bool: + """whether a backend-native network error is a timeout + + neither backend raises asyncio.TimeoutError for a wire timeout: curl_cffi's own + Timeout and noble_tls's Go-side TLSClientException are both plain OSError + subclasses with no dedicated timeout type — check curl_cffi's Timeout class by + name first (precise), then fall back to matching the error text (covers + noble_tls's Go messages, e.g. "context deadline exceeded"). + """ + timeout_type = getattr(error, "__class__", None) + if timeout_type is not None and any(base.__name__ == "Timeout" for base in timeout_type.__mro__): + return True + return any(needle in str(error).lower() for needle in _TIMEOUT_TEXT) + + +def _as_timeout_error(error: Exception, backend: str) -> aiohttp.ServerTimeoutError: + """wrap a backend-native timeout as aiohttp.ServerTimeoutError + + matches aioweb's own contract (a total timeout raises ServerTimeoutError, both a + ClientError and a TimeoutError) instead of surfacing as a generic client error. + """ + return aiohttp.ServerTimeoutError(f"{backend} request timed out: {error}") + + try: from curl_cffi import AsyncSession as _CurlAsyncSession _CURL_ERROR = None @@ -51,10 +77,10 @@ except ImportError as error: def _coerce_timeout(value): - """turn aioweb's aiohttp ClientTimeout (or a number) into a plain number + """unwrap aioweb's aiohttp ClientTimeout (or a plain number) to a number - aioweb.request() wraps a numeric timeout in an aiohttp.ClientTimeout before the - seam sees it; the tls clients want a number, so unwrap .total when present. + aioweb.request() wraps a numeric timeout in aiohttp.ClientTimeout before the seam + sees it; the tls clients want a bare number. """ total = getattr(value, "total", value) return total if isinstance(total, (int, float)) else None @@ -63,12 +89,10 @@ def _coerce_timeout(value): def _noble_timeout_seconds(value): """coerce a raw or wrapped timeout into whole seconds for noble's Go int field - single source of truth for both of Noble's timeout paths (create_session's - session-default and raw_request's per-call override): noble's Go field - timeoutSeconds is an int, so a sub-second float (e.g. 0.5) truncates to 0 — which - Go reads as no/instant timeout — and any non-integer value fails Go-side JSON - unmarshal outright. rounds UP so a sub-second timeout still waits at least 1s - instead of truncating away. returns None when there is no timeout to coerce. + single source of truth for both Noble timeout paths (session-default and + per-call): the Go field timeoutSeconds is an int, so a sub-second float truncates + to 0 (= no timeout) and a non-integer fails Go-side JSON unmarshal outright. + rounds up so a sub-second timeout still waits at least 1s. None if uncoercible. """ timeout = _coerce_timeout(value) if timeout is None: @@ -80,11 +104,10 @@ def _noble_content(response) -> bytes: """extract true response bytes from a noble_tls response fetched with is_byte_response with is_byte_response=True, noble_tls's Go side returns the body as a - `data:;base64,` URI string in response.text (the whole Go response is - UTF-8-decoded before JSON parsing, so raw bytes have to travel as base64 to survive - that round trip) — response.content is useless here since it is just - `self.text.encode()`, re-encoding the data-URI string itself rather than decoding it. - falls back to a plain utf-8 encode if the body isn't a data-URI (e.g. an error body). + `data:;base64,` URI string in response.text (the whole Go response + is UTF-8-decoded before JSON parsing, so raw bytes travel as base64) — + response.content just re-encodes that string, so decode the data-URI here + instead. falls back to a plain utf-8 encode if the body isn't a data-URI. """ text = getattr(response, "text", "") or "" if text.startswith("data:") and ";base64," in text: @@ -93,12 +116,25 @@ def _noble_content(response) -> bytes: return text.encode() +def _flatten_headers(headers) -> dict: + """flatten a Go-style map[str][]str header dict into plain str values + + noble_tls keeps a multi-valued header (e.g. two Set-Cookie lines) as a Python + list, breaking any downstream .split()/.lower() call; join with ", " per RFC 7230 + field-value combination, leaving single values untouched. + """ + return { + key: ", ".join(value) if isinstance(value, list) else value + for key, value in headers.items() + } + + def _jar_to_dict(session): """best-effort map of a requests-style cookie jar on session to a plain dict - intentionally broad: this feeds preview() only, the two backends expose differently- - shaped jars, and a cookie read must never crash a request — so any jar that doesn't - iterate cleanly degrades to {} rather than raising. + feeds preview() only; the two backends expose differently-shaped jars and a + cookie read must never crash a request, so a jar that fails to iterate degrades + to {} rather than raising. """ jar = getattr(session, "cookies", None) if not jar: @@ -114,10 +150,10 @@ class CurlCffi: config: impersonate: browser profile to forge (default "chrome"); override per call - by passing impersonate= to the low-level request()/_raw_request path, - which forwards **kwargs to the backend. NOT request_with_retries — its - signature is fixed (no **kwargs) and would raise TypeError. for a - per-call profile under retries, set it on the CurlCffi instance instead. + by passing impersonate= to the low-level request()/_raw_request path + (forwards **kwargs to the backend) — NOT request_with_retries, whose + fixed signature has no **kwargs and raises TypeError. for the retrying + path, set the profile on the CurlCffi instance instead. requires the [curl] extra (pip install "aioweb_tls[curl]"). """ @@ -133,13 +169,11 @@ class CurlCffi: def create_session(self, headers, timeout, **kwargs): """build the curl_cffi AsyncSession - deliberately does NOT pass `headers` to AsyncSession: curl_cffi bakes a - constructor `headers=` into the client and re-merges it under whatever - per-request headers() sends, so update_headers()/clear_headers() would stop - matching what's actually on the wire (aioweb's session-default headers are - already applied per request by the base's _default_headers merge — see - aioweb.ExtendedSession._create_session's docstring for why baking breaks the - mutable header api). + deliberately does NOT pass `headers`: curl_cffi bakes a constructor + `headers=` into the client and re-merges it under per-request headers, + which would desync update_headers()/clear_headers() from what's actually on + the wire — aioweb's session-default headers already apply per request via + the base's _default_headers merge (see ExtendedSession._create_session). """ return _CurlAsyncSession(timeout=timeout, **kwargs) @@ -162,9 +196,11 @@ class CurlCffi: except asyncio.TimeoutError: raise except OSError as error: - # curl_cffi's RequestException subclasses OSError; translate the native - # network error into aiohttp.ClientError. narrowed from a bare Exception so a - # real bug (AttributeError/TypeError) isn't laundered into 'client error' + # RequestException (incl. curl_cffi's own Timeout) subclasses OSError, never + # asyncio.TimeoutError — check for a timeout first so it surfaces as + # ServerTimeoutError; narrowed from bare Exception so a real bug surfaces + if _is_timeout_error(error): + raise _as_timeout_error(error, "curl_cffi") from error raise _as_client_error(error, "curl_cffi") from error content = response.content if response.content is not None else b"" return Response( @@ -179,10 +215,9 @@ class CurlCffi: def is_closed(self, session) -> bool: """whether the curl_cffi session is closed - curl_cffi tracks closed state in the private `_closed` (no public `closed` - property), so read that; fall back to a public `closed` if a future version - adds one. TLSSession's own `_closed` flag is the primary signal — this is a - best-effort backend check for out-of-band closes. + curl_cffi tracks closed state in the private `_closed` (no public `closed`); + fall back to a public `closed` if a future version adds one. TLSSession's own + `_closed` flag is the primary signal — this is a best-effort out-of-band check. """ closed = getattr(session, "_closed", None) if closed is None: @@ -234,21 +269,29 @@ class Noble: @staticmethod def _resolve_client(client): - """turn a string or Client enum into a noble_tls Client value""" - if isinstance(client, str): - return getattr(_NobleClient, client.upper()) - return client + """turn a string or Client enum into a noble_tls Client value + + raises ValueError naming the valid profiles for an unknown string, instead of + letting getattr's raw AttributeError leak the enum's internal lookup mechanics. + """ + if not isinstance(client, str): + return client + name = client.upper() + resolved = getattr(_NobleClient, name, None) + if resolved is None: + valid = ", ".join(member.name for member in _NobleClient) + raise ValueError(f"unknown noble_tls client profile {client!r}; valid profiles: {valid}") + return resolved async def setup(self) -> None: """fetch the noble_tls Go shared library once; idempotent and concurrency-safe - uses noble_tls.download_if_necessary (the current API: it fetches the asset on - first use and no-ops when it already exists). older noble_tls without that name - is handled via update_if_necessary as a fallback. + uses noble_tls.download_if_necessary (fetches on first use, no-ops if + present); falls back to update_if_necessary on older noble_tls. guarded by an asyncio.Lock with a check-lock-recheck so concurrent first - requests don't both run the fetch: the fast path returns once _updated is - set, and only the first caller through the lock does the work. + requests don't both run the fetch — only the first caller through the lock + does the work, the rest see _updated already set. """ if self._updated: return @@ -266,18 +309,14 @@ class Noble: """build the noble_tls Session, honoring the session-default timeout noble_tls.Session takes neither headers nor timeout in its constructor. - deliberately does NOT bake `headers` into session.headers: aioweb's - session-default headers are already applied per request by the base's - _default_headers merge, and baking them here would make - update_headers()/clear_headers() stop matching what's actually on the wire - (see aioweb.ExtendedSession._create_session's docstring, and CurlCffi.create_session - above for the same rationale) — the per-request merge means headers are never - silently dropped, contrary to what this docstring used to claim. + deliberately does NOT bake `headers` into session.headers, same rationale as + CurlCffi.create_session — aioweb's per-request _default_headers merge already + applies them, so baking here would desync update_headers()/clear_headers(). - the coerced timeout IS applied here (matching raw_request's per-call path): noble's - Go field timeoutSeconds is an int, so a raw sub-second/float session-default (e.g. - timeout=7.5) would fail Go-side JSON unmarshal on every request that doesn't - override it per call. max(1, ceil()) mirrors the guard raw_request already has. + the coerced timeout IS applied here (matching raw_request's per-call path): + noble's Go field timeoutSeconds is an int, so an uncoerced sub-second/float + session-default would fail Go-side JSON unmarshal on any request that doesn't + override it per call; max(1, ceil()) mirrors raw_request's own guard. """ session = noble_tls.Session(client=self.client, **kwargs) timeout_seconds = _noble_timeout_seconds(timeout) @@ -297,10 +336,9 @@ class Noble: if proxy: kwargs["proxy"] = proxy - # force byte-safe transport: without this, noble_tls's Go side returns the body - # as a plain UTF-8 JSON string, replacing invalid bytes with U+FFFD — silently - # corrupting any binary payload (image/zip/pdf) even though the request succeeds - # with status 200. see _noble_content() for how the byte-safe body is decoded back. + # byte-safe transport: without this, noble_tls's Go side returns the body as a + # plain UTF-8 JSON string (U+FFFD-mangling any binary payload on a 200 response); + # see _noble_content() for the base64 data-URI decode this pairs with. kwargs.setdefault("is_byte_response", True) try: @@ -310,13 +348,16 @@ class Noble: except asyncio.TimeoutError: raise except OSError as error: - # noble_tls's TLSClientException subclasses IOError (== OSError); translate - # the native network error, narrowed from bare Exception so a real bug surfaces + # TLSClientException subclasses IOError (== OSError); the Go side has no + # distinct timeout type either, just a text body ("context deadline + # exceeded") — check for one before the generic client-error wrap + if _is_timeout_error(error): + raise _as_timeout_error(error, "noble_tls") from error raise _as_client_error(error, "noble_tls") from error content = _noble_content(response) return Response( status_code=response.status_code, - headers=dict(getattr(response, "headers", {}) or {}), + headers=_flatten_headers(getattr(response, "headers", {}) or {}), content=content, url=str(getattr(response, "url", url)), reason=getattr(response, "reason", None), diff --git a/src/aioweb_tls/protocol.py b/src/aioweb_tls/protocol.py index cfb96e6..a674efd 100644 --- a/src/aioweb_tls/protocol.py +++ b/src/aioweb_tls/protocol.py @@ -6,42 +6,39 @@ to one HTTP client. TLSSession owns the live session object (built by create_ses and passes it into every backend call, so backends hold no per-request state. implement this protocol to add a custom backend (e.g. a local Go TLS server); inject -it via TLSSession(backend=MyBackend(...)) and it inherits all of aioweb's domain / -header / ephemeral / proxy / retry / preview logic unchanged — those operate on plain -dicts and never touch the backend. +it via TLSSession(backend=MyBackend(...)) and it inherits aioweb's domain / header / +ephemeral / proxy / retry / preview logic unchanged — those operate on plain dicts +and never touch the backend. required: create_session(headers: dict, timeout, **kwargs) -> session - build and return the live client session object. TLSSession stores it as - self.session and hands it back to every other method below. + build and return the live client session object, stored as self.session and + passed to every method below. async raw_request(session, method, url, **kwargs) -> aioweb.Response - send one request with `session` and adapt the client's response into an - aioweb.Response built from primitives (status_code, headers, content bytes, - url, reason). kwargs arrive aioweb-shaped — the base has already resolved the - proxy into kwargs["proxy"] and merged headers into kwargs["headers"], and a - numeric timeout is wrapped in an aiohttp.ClientTimeout (unwrap .total). + send one request; adapt the client's response into an aioweb.Response built + from primitives (status_code, headers, content bytes, url, reason). kwargs + arrive aioweb-shaped: proxy resolved into kwargs["proxy"], headers merged + into kwargs["headers"], numeric timeout wrapped in aiohttp.ClientTimeout + (unwrap .total). is_closed(session) -> bool whether `session` is closed. optional: cookies_for_url(session, url) -> dict - cookies the client would send for url, for preview(). default {} (used when - the backend has no introspectable jar). + cookies the client would send for url, for preview(). default {}. set_cookie(session, name, value, domain=None, path="/") -> None get_cookies(session) -> dict clear_cookies(session) -> None - the mutable cookie api TLSSession.set_cookie/get_cookies/clear_cookies - delegate to. the aiohttp-only base implementations reach into - session.cookie_jar, which TLS backends don't have, so a backend without - these raises NotImplementedError from TLSSession rather than crashing with - an AttributeError on a private aiohttp attribute. + back TLSSession's mutable cookie api. the aiohttp-only base reaches into + session.cookie_jar, which TLS backends lack, so an implementation without + these raises NotImplementedError instead of a private-attribute AttributeError. async setup() -> None - one-time async preparation (e.g. fetch a native lib). called once via - TLSSession.setup() and lazily before the first request; make it idempotent. + one-time async preparation (e.g. fetch a native lib); idempotent. called via + TLSSession.setup() and lazily before the first request. async close(session) -> None close `session`. default awaits session.close() if present. diff --git a/src/aioweb_tls/session.py b/src/aioweb_tls/session.py index 28ffe63..0534735 100644 --- a/src/aioweb_tls/session.py +++ b/src/aioweb_tls/session.py @@ -4,8 +4,8 @@ TLSSession — one aioweb session, any tls backend TLSSession subclasses aioweb.ExtendedSession and delegates only the four backend seams to an injected backend object (see protocol.py). everything else — header overwrites, domain rewriting, ephemeral headers, proxies, retries, previews — is -inherited from aioweb unchanged, because that logic operates on plain dicts and -never touches the backend. +inherited unchanged, since that logic operates on plain dicts and never touches +the backend. from aioweb_tls import TLSSession, CurlCffi, Noble