migrate session handling to aioweb_tls/refactoring of helpers
This commit is contained in:
TPV committed
1 parent
3a8a12ac6e
commit
1441e7179c
10 files changed
+244
-295
No files matched your search
+52
-188
@@ -1,225 +1,89 @@
|
||||
"""Shared HTTP session helpers for all monitors.
|
||||
|
||||
Known gap: UA/TLS fingerprint does not always match headers/header order,
|
||||
but targeted antibots do not currently seem to care.
|
||||
A single aioweb_tls TLSSession (curl_cffi backend, random client fingerprint per
|
||||
session) replaces the old aiohttp/curl_cffi split. send_request wraps aioweb's
|
||||
request_with_retries, adding per-request proxy rotation and old-style JSON/text
|
||||
return values so monitor call sites change as little as possible.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import AsyncGenerator
|
||||
|
||||
import aiohttp
|
||||
import requests
|
||||
from aioproxies import AioProxies
|
||||
from curl_cffi.requests import AsyncSession
|
||||
from curl_cffi.requests.errors import RequestsError
|
||||
from curl_cffi.requests.exceptions import Timeout as CurlTimeout
|
||||
from fake_useragent import FakeUserAgent
|
||||
from tenacity import (
|
||||
after_log,
|
||||
before_log,
|
||||
retry,
|
||||
retry_if_exception_type,
|
||||
stop_after_attempt,
|
||||
wait_fixed,
|
||||
)
|
||||
from aioweb import Response
|
||||
from aioweb_tls import CurlCffi, TLSSession
|
||||
|
||||
from monitor_utils.constants import (
|
||||
CLIENT_IDENTIFIERS,
|
||||
CLIENT_TIMEOUT,
|
||||
RETRY_LIMIT,
|
||||
RETRY_WAIT,
|
||||
TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
UA = FakeUserAgent()
|
||||
from monitor_utils.constants import CLIENT_IDENTIFIERS, RETRY_LIMIT
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_base_session(
|
||||
headers: dict | None = None,
|
||||
cookies: dict | None = None,
|
||||
connector: aiohttp.TCPConnector | None = None,
|
||||
timeout: aiohttp.ClientTimeout = CLIENT_TIMEOUT,
|
||||
proxies: AioProxies | None = None,
|
||||
) -> AsyncGenerator[aiohttp.ClientSession, None]:
|
||||
"""Yield an aiohttp session with a random User-Agent and attached proxy pool.
|
||||
def get_default_session() -> TLSSession:
|
||||
"""Return a TLSSession impersonating a random client fingerprint.
|
||||
|
||||
The proxy pool is stored on the session as proxy_pool for send_request to
|
||||
rotate per request. The session is always closed on exit.
|
||||
Fingerprint rotation lives here so every monitor rotates the same way; the
|
||||
caller owns the session lifetime via ``async with``.
|
||||
|
||||
Args:
|
||||
headers: Headers applied to the session; User-Agent is defaulted if
|
||||
absent.
|
||||
cookies: Cookies applied to the session.
|
||||
connector: Optional TCP connector.
|
||||
timeout: Client timeout applied to the session.
|
||||
proxies: Proxy pool attached to the session.
|
||||
|
||||
Yields:
|
||||
The configured aiohttp session.
|
||||
Returns:
|
||||
A TLSSession over a curl_cffi backend with a random impersonation.
|
||||
"""
|
||||
|
||||
headers = headers if headers else {}
|
||||
headers.setdefault("User-Agent", UA.random)
|
||||
session = aiohttp.ClientSession(
|
||||
headers=headers, cookies=cookies, connector=connector, timeout=timeout
|
||||
)
|
||||
session.proxy_pool = proxies
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
await session.close()
|
||||
return TLSSession(backend=CurlCffi(impersonate=random.choice(CLIENT_IDENTIFIERS)))
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_tls_session(
|
||||
headers: dict | None = None,
|
||||
timeout: int = TIMEOUT_SECONDS,
|
||||
proxies: AioProxies | None = None,
|
||||
) -> AsyncGenerator[AsyncSession, None]:
|
||||
"""Yield a curl_cffi session impersonating a random client fingerprint.
|
||||
|
||||
The proxy pool is stored on the session as proxy_pool for send_request to
|
||||
rotate per request.
|
||||
|
||||
Args:
|
||||
headers: Headers applied to the session; a Chrome User-Agent is
|
||||
defaulted if absent.
|
||||
timeout: Request timeout in seconds.
|
||||
proxies: Proxy pool attached to the session.
|
||||
|
||||
Yields:
|
||||
The configured curl_cffi session.
|
||||
"""
|
||||
|
||||
headers = headers if headers else {}
|
||||
headers.setdefault("User-Agent", UA.chrome)
|
||||
async with AsyncSession(
|
||||
impersonate=random.choice(CLIENT_IDENTIFIERS),
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
) as session:
|
||||
session.proxy_pool = proxies
|
||||
yield session
|
||||
|
||||
|
||||
def build_session_details(
|
||||
session: aiohttp.ClientSession | AsyncSession, headers: dict = None
|
||||
):
|
||||
"""Merge headers into an existing session, defaulting a random User-Agent.
|
||||
|
||||
Args:
|
||||
session: Session whose headers are updated in place.
|
||||
headers: Headers to merge; User-Agent is defaulted if absent.
|
||||
"""
|
||||
|
||||
headers = headers or {}
|
||||
headers.setdefault("User-Agent", UA.random)
|
||||
session.headers.update(headers)
|
||||
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(RETRY_LIMIT),
|
||||
wait=wait_fixed(RETRY_WAIT),
|
||||
retry=retry_if_exception_type(
|
||||
(
|
||||
aiohttp.ClientError,
|
||||
asyncio.TimeoutError,
|
||||
RequestsError,
|
||||
requests.Timeout,
|
||||
CurlTimeout,
|
||||
)
|
||||
),
|
||||
before=before_log(LOGGER, logging.DEBUG),
|
||||
after=after_log(LOGGER, logging.DEBUG),
|
||||
reraise=True,
|
||||
)
|
||||
async def send_request(
|
||||
session: aiohttp.ClientSession | AsyncSession,
|
||||
session: TLSSession,
|
||||
url: str,
|
||||
method: str,
|
||||
*,
|
||||
proxies: AioProxies,
|
||||
params: dict | None = None,
|
||||
json_data: dict | None = None,
|
||||
data: dict | None = None,
|
||||
headers: dict | None = None,
|
||||
reraise_json_error: bool = False,
|
||||
development_proxy: str | None = None,
|
||||
) -> dict | str:
|
||||
"""Send a request through either session type with retries and proxy rotation.
|
||||
) -> Response:
|
||||
"""Send a request through a TLSSession with retries and proxy rotation.
|
||||
|
||||
Retries transport errors and timeouts up to RETRY_LIMIT with a fixed wait.
|
||||
When development_proxy is set it overrides the session's proxy pool and
|
||||
disables TLS verification for local traffic inspection; otherwise the next
|
||||
proxy is drawn from the session's proxy_pool when present. Responses are
|
||||
parsed as JSON when possible.
|
||||
Wraps aioweb's request_with_retries, which retries transport errors,
|
||||
timeouts, and retryable statuses (429/5xx) up to RETRY_LIMIT attempts with
|
||||
exponential backoff. The next proxy is drawn from the pool per call and
|
||||
applied to both schemes. A non-2xx response or a fully failed request raises
|
||||
AiowebError, an expected task error, so a failed fetch surfaces the same way
|
||||
the old HTTPError path did. On success the aioweb Response is returned for
|
||||
the caller to read via response.json(), response.text(), etc.
|
||||
|
||||
Args:
|
||||
session: aiohttp or curl_cffi session to send through.
|
||||
session: The TLSSession to send through.
|
||||
url: Request URL.
|
||||
method: HTTP method.
|
||||
proxies: Proxy pool (required); the next proxy is drawn per call and
|
||||
applied to both http and https.
|
||||
params: Query parameters.
|
||||
json_data: JSON body.
|
||||
data: Form body.
|
||||
headers: Per-request headers merged over the session headers,
|
||||
avoiding mutation of shared session state from concurrent tasks.
|
||||
reraise_json_error: Raise JSONDecodeError instead of returning raw
|
||||
text when the response is not valid JSON.
|
||||
development_proxy: Local proxy URL that overrides the pool and
|
||||
disables verification.
|
||||
json_data: JSON body. A dict here is sent as a JSON body.
|
||||
data: Alternative body. Note aioweb routes any dict or list to a JSON
|
||||
body, so true form-encoding requires a pre-encoded string; a form
|
||||
dict passed here is sent as JSON.
|
||||
headers: Per-request headers merged over the session defaults.
|
||||
|
||||
Returns:
|
||||
Parsed JSON dict, or raw response text when parsing fails and
|
||||
reraise_json_error is False.
|
||||
The aioweb Response, guaranteed 2xx (non-2xx raises). Read the body with
|
||||
response.json() (dict/list, or None if not valid JSON) or
|
||||
response.text().
|
||||
|
||||
Raises:
|
||||
json.JSONDecodeError: When parsing fails and reraise_json_error is
|
||||
True.
|
||||
AiowebError: On a non-2xx response or after all retries fail.
|
||||
"""
|
||||
|
||||
verify = True
|
||||
if development_proxy:
|
||||
proxy = development_proxy
|
||||
verify = False
|
||||
else:
|
||||
manager = getattr(session, "proxy_pool", None)
|
||||
proxy = manager.next().url() if manager else None
|
||||
proxy_url = proxies.next().url()
|
||||
proxy = {"http": proxy_url, "https": proxy_url}
|
||||
|
||||
if isinstance(session, aiohttp.ClientSession):
|
||||
async with session.request(
|
||||
method=method,
|
||||
url=url,
|
||||
params=params,
|
||||
json=json_data,
|
||||
data=data,
|
||||
headers=headers,
|
||||
proxy=proxy,
|
||||
ssl=verify,
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
text = await response.text()
|
||||
else:
|
||||
response = await session.request(
|
||||
method=method.upper(),
|
||||
url=url,
|
||||
params=params,
|
||||
json=json_data,
|
||||
data=data,
|
||||
headers=headers,
|
||||
proxy=proxy,
|
||||
verify=verify,
|
||||
)
|
||||
response.raise_for_status()
|
||||
text = response.text
|
||||
body = json_data if json_data is not None else data
|
||||
response = await session.request_with_retries(
|
||||
method.upper(),
|
||||
url,
|
||||
params=params,
|
||||
data=body,
|
||||
headers=headers,
|
||||
proxies=proxy,
|
||||
attempts=RETRY_LIMIT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
if reraise_json_error:
|
||||
raise
|
||||
|
||||
return text
|
||||
return response
|
||||
Reference in new issue
Block a user