diff --git a/README.md b/README.md index 692fed0..76d2201 100644 --- a/README.md +++ b/README.md @@ -22,17 +22,17 @@ you want; importing the package never fails because an extra is missing. `requirements.txt` (pick the extra you need): ``` -aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5 -aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5 -aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5 +aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.6 +aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.6 +aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.6 ``` Direct: ```bash -pip install "aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5" -pip install "aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5" -pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.5" +pip install "aioweb_tls[curl] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.6" +pip install "aioweb_tls[noble] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.6" +pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-public/aioweb_tls.git@v0.1.6" ``` - `[curl]` → curl_cffi backend · `[noble]` → noble_tls backend · `[all]` → both. @@ -44,7 +44,7 @@ pip install "aioweb_tls[all] @ git+ssh://git@git.rethinkstudios.io/rethink-publi Constructing a backend whose client isn't installed raises that `RuntimeError` at construction, never at import. -Drop the `@v0.1.5` suffix from the line above to install the latest unpinned. +Drop the `@v0.1.6` suffix from the line above to install the latest unpinned. ## curl_cffi backend diff --git a/pyproject.toml b/pyproject.toml index 4b4d767..004c504 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "aioweb_tls" -version = "0.1.5" +version = "0.1.6" description = "TLS-fingerprinting backends (curl_cffi / noble_tls) for aioweb via one injectable TLSSession, config-free, installable." requires-python = ">=3.10" dependencies = [ diff --git a/src/aioweb_tls/__init__.py b/src/aioweb_tls/__init__.py index 4afdc03..16cc038 100644 --- a/src/aioweb_tls/__init__.py +++ b/src/aioweb_tls/__init__.py @@ -1,24 +1,10 @@ """ -tls-fingerprinting backends for aioweb - -one session class, TLSSession, takes an injected backend that swaps the HTTP client -(and thus the TLS/HTTP fingerprint) while inheriting every aioweb feature — header -overwrites, domain rewriting, ephemeral headers, proxies, retries, previews — -unchanged. +tls-fingerprinting backends for aioweb - see README for usage and the extras contract from aioweb_tls import TLSSession, CurlCffi, Noble async with TLSSession(backend=CurlCffi(impersonate="chrome")) as s: # [curl] extra resp = await s.request_with_retries("GET", url) - - async with TLSSession(backend=Noble(client="chrome_133")) as s: # [noble] extra - await s.setup() # fetch Go lib once - resp = await s.request_with_retries("GET", url) - -the tls clients are optional extras, not base deps. importing this package never -fails because an extra is missing; the matching RuntimeError is raised only when you -construct a backend whose client isn't installed. custom backends implement the -TLSBackend protocol and inject the same way. """ from .session import TLSSession diff --git a/src/aioweb_tls/backends.py b/src/aioweb_tls/backends.py index 6ff37b5..cfef36d 100644 --- a/src/aioweb_tls/backends.py +++ b/src/aioweb_tls/backends.py @@ -1,14 +1,6 @@ """ -tls backends for TLSSession - -each backend is a stateless config+behavior object implementing the TLSBackend -protocol (see protocol.py). it owns its own config vocabulary — CurlCffi takes -impersonate=, Noble takes client= — so there is no shared kwarg-soup. TLSSession -owns the live session object and passes it into every method here. - -the underlying tls clients are optional extras: constructing a backend whose client -is not installed raises a clear RuntimeError naming the extra to install. importing -this module never fails because an extra is missing. +tls backends for TLSSession - stateless config+behavior objects implementing the +TLSBackend protocol (see protocol.py); see README for the extras contract """ import asyncio @@ -23,12 +15,7 @@ log = logging.getLogger(__name__) def _as_client_error(error: Exception, backend: str) -> aiohttp.ClientError: - """wrap a backend-native network exception as an aiohttp.ClientError - - curl_cffi raises RequestException(OSError), noble_tls raises - TLSClientException(IOError) — neither is an aiohttp.ClientError, which is all - aioweb's request() re-wraps; translating here gives both the same typed contract. - """ + """wrap a backend-native network exception (neither is an aiohttp.ClientError) as one""" return aiohttp.ClientError(f"{backend} request failed: {error}") @@ -36,14 +23,8 @@ _TIMEOUT_TEXT = ("timeout", "timed out", "deadline exceeded") def _is_timeout_error(error: Exception) -> bool: - """whether a backend-native network error is a timeout - - neither backend raises asyncio.TimeoutError for a wire timeout: curl_cffi's own - Timeout and noble_tls's Go-side TLSClientException are both plain OSError - subclasses with no dedicated timeout type — check curl_cffi's Timeout class by - name first (precise), then fall back to matching the error text (covers - noble_tls's Go messages, e.g. "context deadline exceeded"). - """ + """whether a backend-native OSError is a timeout: curl_cffi's Timeout type by name, + else fall back to matching noble_tls's Go error text (e.g. "context deadline exceeded")""" timeout_type = getattr(error, "__class__", None) if timeout_type is not None and any(base.__name__ == "Timeout" for base in timeout_type.__mro__): return True @@ -51,11 +32,7 @@ def _is_timeout_error(error: Exception) -> bool: def _as_timeout_error(error: Exception, backend: str) -> aiohttp.ServerTimeoutError: - """wrap a backend-native timeout as aiohttp.ServerTimeoutError - - matches aioweb's own contract (a total timeout raises ServerTimeoutError, both a - ClientError and a TimeoutError) instead of surfacing as a generic client error. - """ + """wrap a backend-native timeout as aiohttp.ServerTimeoutError, matching aioweb's own contract""" return aiohttp.ServerTimeoutError(f"{backend} request timed out: {error}") @@ -77,11 +54,7 @@ except ImportError as error: def _coerce_timeout(value): - """unwrap aioweb's aiohttp ClientTimeout (or a plain number) to a number - - aioweb.request() wraps a numeric timeout in aiohttp.ClientTimeout before the seam - sees it; the tls clients want a bare number. - """ + """unwrap aioweb's aiohttp.ClientTimeout (or a plain number) to a bare number for the tls clients""" total = getattr(value, "total", value) return total if isinstance(total, (int, float)) else None @@ -89,10 +62,10 @@ def _coerce_timeout(value): def _noble_timeout_seconds(value): """coerce a raw or wrapped timeout into whole seconds for noble's Go int field - single source of truth for both Noble timeout paths (session-default and - per-call): the Go field timeoutSeconds is an int, so a sub-second float truncates - to 0 (= no timeout) and a non-integer fails Go-side JSON unmarshal outright. - rounds up so a sub-second timeout still waits at least 1s. None if uncoercible. + noble's Go timeoutSeconds field is an int: a sub-second float truncates to 0 + (= no timeout) and a non-integer fails Go-side JSON unmarshal outright, so this + rounds up (min 1s). shared by both Noble timeout paths (session-default, per-call). + None if uncoercible. """ timeout = _coerce_timeout(value) if timeout is None: @@ -103,11 +76,11 @@ def _noble_timeout_seconds(value): def _noble_content(response) -> bytes: """extract true response bytes from a noble_tls response fetched with is_byte_response - with is_byte_response=True, noble_tls's Go side returns the body as a + with is_byte_response=True, the Go side returns the body as a `data:;base64,` URI string in response.text (the whole Go response - is UTF-8-decoded before JSON parsing, so raw bytes travel as base64) — - response.content just re-encodes that string, so decode the data-URI here - instead. falls back to a plain utf-8 encode if the body isn't a data-URI. + is UTF-8-decoded before JSON parsing, so raw bytes travel as base64); decode that + here instead of response.content, which just re-encodes the string. falls back to + a plain utf-8 encode if the body isn't a data-URI. """ text = getattr(response, "text", "") or "" if text.startswith("data:") and ";base64," in text: @@ -117,12 +90,8 @@ def _noble_content(response) -> bytes: def _flatten_headers(headers) -> dict: - """flatten a Go-style map[str][]str header dict into plain str values - - noble_tls keeps a multi-valued header (e.g. two Set-Cookie lines) as a Python - list, breaking any downstream .split()/.lower() call; join with ", " per RFC 7230 - field-value combination, leaving single values untouched. - """ + """flatten a Go-style map[str][]str header dict (multi-valued headers as a list) + into plain str values, joined with ", " per RFC 7230""" return { key: ", ".join(value) if isinstance(value, list) else value for key, value in headers.items() @@ -132,9 +101,7 @@ def _flatten_headers(headers) -> dict: def _jar_to_dict(session): """best-effort map of a requests-style cookie jar on session to a plain dict - feeds preview() only; the two backends expose differently-shaped jars and a - cookie read must never crash a request, so a jar that fails to iterate degrades - to {} rather than raising. + feeds preview() only; a jar that fails to iterate degrades to {} rather than raising. """ jar = getattr(session, "cookies", None) if not jar: @@ -150,10 +117,9 @@ class CurlCffi: config: impersonate: browser profile to forge (default "chrome"); override per call - by passing impersonate= to the low-level request()/_raw_request path - (forwards **kwargs to the backend) — NOT request_with_retries, whose - fixed signature has no **kwargs and raises TypeError. for the retrying - path, set the profile on the CurlCffi instance instead. + via request()/_raw_request (forwards **kwargs) - NOT request_with_retries, + whose fixed signature raises TypeError on it. for the retrying path, set + the profile on the CurlCffi instance instead. requires the [curl] extra (pip install "aioweb_tls[curl]"). """ @@ -172,7 +138,7 @@ class CurlCffi: deliberately does NOT pass `headers`: curl_cffi bakes a constructor `headers=` into the client and re-merges it under per-request headers, which would desync update_headers()/clear_headers() from what's actually on - the wire — aioweb's session-default headers already apply per request via + the wire - aioweb's session-default headers already apply per request via the base's _default_headers merge (see ExtendedSession._create_session). """ return _CurlAsyncSession(timeout=timeout, **kwargs) @@ -196,9 +162,6 @@ class CurlCffi: except asyncio.TimeoutError: raise except OSError as error: - # RequestException (incl. curl_cffi's own Timeout) subclasses OSError, never - # asyncio.TimeoutError — check for a timeout first so it surfaces as - # ServerTimeoutError; narrowed from bare Exception so a real bug surfaces if _is_timeout_error(error): raise _as_timeout_error(error, "curl_cffi") from error raise _as_client_error(error, "curl_cffi") from error @@ -215,9 +178,9 @@ class CurlCffi: def is_closed(self, session) -> bool: """whether the curl_cffi session is closed - curl_cffi tracks closed state in the private `_closed` (no public `closed`); - fall back to a public `closed` if a future version adds one. TLSSession's own - `_closed` flag is the primary signal — this is a best-effort out-of-band check. + curl_cffi tracks this in the private `_closed` (no public `closed` today); + falls back to a public `closed` if a future version adds one. best-effort - + TLSSession's own `_closed` flag is the primary signal. """ closed = getattr(session, "_closed", None) if closed is None: @@ -251,8 +214,8 @@ class Noble: config: client: noble_tls Client profile (enum or string, default "chrome_133"). - noble_tls downloads a Go shared library on first use; setup() fetches it once - (run via TLSSession.setup() or lazily before the first request). + downloads a Go shared library on first use; setup() fetches it once (via + TLSSession.setup() or lazily before the first request). requires the [noble] extra (pip install "aioweb_tls[noble]"). """ @@ -269,11 +232,8 @@ class Noble: @staticmethod def _resolve_client(client): - """turn a string or Client enum into a noble_tls Client value - - raises ValueError naming the valid profiles for an unknown string, instead of - letting getattr's raw AttributeError leak the enum's internal lookup mechanics. - """ + """turn a string or Client enum into a noble_tls Client value, raising ValueError + naming the valid profiles for an unknown string""" if not isinstance(client, str): return client name = client.upper() @@ -286,12 +246,9 @@ class Noble: async def setup(self) -> None: """fetch the noble_tls Go shared library once; idempotent and concurrency-safe - uses noble_tls.download_if_necessary (fetches on first use, no-ops if - present); falls back to update_if_necessary on older noble_tls. - - guarded by an asyncio.Lock with a check-lock-recheck so concurrent first - requests don't both run the fetch — only the first caller through the lock - does the work, the rest see _updated already set. + uses download_if_necessary (falls back to update_if_necessary on older + noble_tls). guarded by an asyncio.Lock with a check-lock-recheck so concurrent + first requests don't both run the fetch. """ if self._updated: return @@ -308,10 +265,8 @@ class Noble: def create_session(self, headers, timeout, **kwargs): """build the noble_tls Session, honoring the session-default timeout - noble_tls.Session takes neither headers nor timeout in its constructor. - deliberately does NOT bake `headers` into session.headers, same rationale as - CurlCffi.create_session — aioweb's per-request _default_headers merge already - applies them, so baking here would desync update_headers()/clear_headers(). + deliberately does NOT bake `headers` into session.headers - same header-baking + rationale as CurlCffi.create_session (see its docstring). the coerced timeout IS applied here (matching raw_request's per-call path): noble's Go field timeoutSeconds is an int, so an uncoerced sub-second/float @@ -336,9 +291,6 @@ class Noble: if proxy: kwargs["proxy"] = proxy - # byte-safe transport: without this, noble_tls's Go side returns the body as a - # plain UTF-8 JSON string (U+FFFD-mangling any binary payload on a 200 response); - # see _noble_content() for the base64 data-URI decode this pairs with. kwargs.setdefault("is_byte_response", True) try: @@ -348,9 +300,6 @@ class Noble: except asyncio.TimeoutError: raise except OSError as error: - # TLSClientException subclasses IOError (== OSError); the Go side has no - # distinct timeout type either, just a text body ("context deadline - # exceeded") — check for one before the generic client-error wrap if _is_timeout_error(error): raise _as_timeout_error(error, "noble_tls") from error raise _as_client_error(error, "noble_tls") from error diff --git a/src/aioweb_tls/protocol.py b/src/aioweb_tls/protocol.py index a674efd..2b363ae 100644 --- a/src/aioweb_tls/protocol.py +++ b/src/aioweb_tls/protocol.py @@ -1,48 +1,4 @@ -""" -the tls backend protocol - -a backend is a stateless config+behavior object that teaches TLSSession how to talk -to one HTTP client. TLSSession owns the live session object (built by create_session) -and passes it into every backend call, so backends hold no per-request state. - -implement this protocol to add a custom backend (e.g. a local Go TLS server); inject -it via TLSSession(backend=MyBackend(...)) and it inherits aioweb's domain / header / -ephemeral / proxy / retry / preview logic unchanged — those operate on plain dicts -and never touch the backend. - -required: - create_session(headers: dict, timeout, **kwargs) -> session - build and return the live client session object, stored as self.session and - passed to every method below. - - async raw_request(session, method, url, **kwargs) -> aioweb.Response - send one request; adapt the client's response into an aioweb.Response built - from primitives (status_code, headers, content bytes, url, reason). kwargs - arrive aioweb-shaped: proxy resolved into kwargs["proxy"], headers merged - into kwargs["headers"], numeric timeout wrapped in aiohttp.ClientTimeout - (unwrap .total). - - is_closed(session) -> bool - whether `session` is closed. - -optional: - cookies_for_url(session, url) -> dict - cookies the client would send for url, for preview(). default {}. - - set_cookie(session, name, value, domain=None, path="/") -> None - get_cookies(session) -> dict - clear_cookies(session) -> None - back TLSSession's mutable cookie api. the aiohttp-only base reaches into - session.cookie_jar, which TLS backends lack, so an implementation without - these raises NotImplementedError instead of a private-attribute AttributeError. - - async setup() -> None - one-time async preparation (e.g. fetch a native lib); idempotent. called via - TLSSession.setup() and lazily before the first request. - - async close(session) -> None - close `session`. default awaits session.close() if present. -""" +"""the tls backend protocol - see TLSBackend below and README""" from typing import Any, Protocol, runtime_checkable diff --git a/src/aioweb_tls/session.py b/src/aioweb_tls/session.py index 0534735..1a6aefa 100644 --- a/src/aioweb_tls/session.py +++ b/src/aioweb_tls/session.py @@ -1,26 +1,16 @@ """ -TLSSession — one aioweb session, any tls backend +TLSSession - one aioweb session, any tls backend -TLSSession subclasses aioweb.ExtendedSession and delegates only the four backend -seams to an injected backend object (see protocol.py). everything else — header -overwrites, domain rewriting, ephemeral headers, proxies, retries, previews — is -inherited unchanged, since that logic operates on plain dicts and never touches -the backend. +subclasses aioweb.ExtendedSession, delegating only the four backend seams to an +injected backend object (see protocol.py); everything else is inherited unchanged. +see README for usage. - from aioweb_tls import TLSSession, CurlCffi, Noble + from aioweb_tls import TLSSession, CurlCffi async with TLSSession(backend=CurlCffi(impersonate="chrome")) as s: resp = await s.request_with_retries("GET", "https://tls.peet.ws/api/all") if resp: print(resp.json()["tls"]["ja3"]) - - s = TLSSession(backend=Noble(client="chrome_133")) - await s.setup() # fetch noble's Go lib once - ... - -a custom backend (e.g. a local Go TLS server) injects the same way — implement the -TLSBackend protocol and pass it as backend=. one TLSSession is the only session -class; the backend swaps the wire, not the session. """ import logging @@ -46,12 +36,10 @@ class TLSSession(ExtendedSession): super().__init__(*args, **kwargs) async def setup(self) -> None: - """run the backend's one-time setup if it has one (idempotent) + """run the backend's one-time setup if it has one (idempotent, e.g. noble's Go lib fetch) - delegates to backend.setup() when defined (e.g. noble fetching its Go lib); - a no-op for backends without it. also invoked lazily before the first request - by backends that guard their own setup, so calling this is optional but lets - callers pre-warm at startup. + optional to call: also invoked lazily before the first request, but calling + it upfront lets callers pre-warm at startup. """ setup = getattr(self.backend, "setup", None) if setup is not None: @@ -82,9 +70,8 @@ class TLSSession(ExtendedSession): return self.backend.is_closed(self.session) # ------------------------------------------------------------------------- - # mutable cookie api — the base's set_cookie/get_cookies/clear_cookies reach - # into self.session.cookie_jar (aiohttp-only), so TLS backends override them - # to route through the backend instead of crashing with AttributeError + # mutable cookie api - base reaches into aiohttp-only session.cookie_jar, so + # TLS backends override these to route through the backend instead def set_cookie(self, name, value, domain=None, path="/"): """set a cookie via the backend's own cookie store""" @@ -114,7 +101,7 @@ class TLSSession(ExtendedSession): clear_cookies(self.session) # ------------------------------------------------------------------------- - # lifecycle — backends close differently, so route through the backend + # lifecycle - backends close differently, so route through the backend async def close(self) -> None: """close via the backend's close (falls back to session.close)"""