fixed a small database issue with price drop tracking and added doc strings
This commit is contained in:
1 parent
a61dbdfaa9
commit
99d660ee52
6 files changed
+172
-4
No files matched your search
@@ -1,4 +1,9 @@
|
||||
# ik ua/tls doesn't match headers/header order but targeted antibots don't seem to care might fix later
|
||||
"""Shared HTTP session helpers for all monitors.
|
||||
|
||||
Known gap: UA/TLS fingerprint does not always match headers/header order,
|
||||
but targeted antibots do not currently seem to care.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
@@ -42,6 +47,22 @@ async def get_base_session(
|
||||
timeout: aiohttp.ClientTimeout = CLIENT_TIMEOUT,
|
||||
proxies: AioProxies | None = None,
|
||||
) -> AsyncGenerator[aiohttp.ClientSession, None]:
|
||||
"""Yield an aiohttp session with a random User-Agent and attached proxy pool.
|
||||
|
||||
The proxy pool is stored on the session as proxy_pool for send_request to
|
||||
rotate per request. The session is always closed on exit.
|
||||
|
||||
Args:
|
||||
headers: Headers applied to the session; User-Agent is defaulted if
|
||||
absent.
|
||||
cookies: Cookies applied to the session.
|
||||
connector: Optional TCP connector.
|
||||
timeout: Client timeout applied to the session.
|
||||
proxies: Proxy pool attached to the session.
|
||||
|
||||
Yields:
|
||||
The configured aiohttp session.
|
||||
"""
|
||||
headers = headers if headers else {}
|
||||
headers.setdefault("User-Agent", UA.random)
|
||||
session = aiohttp.ClientSession(
|
||||
@@ -60,6 +81,20 @@ async def get_tls_session(
|
||||
timeout: int = TIMEOUT_SECONDS,
|
||||
proxies: AioProxies | None = None,
|
||||
) -> AsyncGenerator[AsyncSession, None]:
|
||||
"""Yield a curl_cffi session impersonating a random client fingerprint.
|
||||
|
||||
The proxy pool is stored on the session as proxy_pool for send_request to
|
||||
rotate per request.
|
||||
|
||||
Args:
|
||||
headers: Headers applied to the session; a Chrome User-Agent is
|
||||
defaulted if absent.
|
||||
timeout: Request timeout in seconds.
|
||||
proxies: Proxy pool attached to the session.
|
||||
|
||||
Yields:
|
||||
The configured curl_cffi session.
|
||||
"""
|
||||
headers = headers if headers else {}
|
||||
headers.setdefault("User-Agent", UA.chrome)
|
||||
async with AsyncSession(
|
||||
@@ -74,6 +109,12 @@ async def get_tls_session(
|
||||
def build_session_details(
|
||||
session: aiohttp.ClientSession | AsyncSession, headers: dict = None
|
||||
):
|
||||
"""Merge headers into an existing session, defaulting a random User-Agent.
|
||||
|
||||
Args:
|
||||
session: Session whose headers are updated in place.
|
||||
headers: Headers to merge; User-Agent is defaulted if absent.
|
||||
"""
|
||||
headers = headers or {}
|
||||
headers.setdefault("User-Agent", UA.random)
|
||||
session.headers.update(headers)
|
||||
@@ -105,6 +146,34 @@ async def send_request(
|
||||
reraise_json_error: bool = False,
|
||||
development_proxy: str | None = None,
|
||||
) -> dict | str:
|
||||
"""Send a request through either session type with retries and proxy rotation.
|
||||
|
||||
Retries transport errors and timeouts up to RETRY_LIMIT with a fixed wait.
|
||||
When development_proxy is set it overrides the session's proxy pool and
|
||||
disables TLS verification for local traffic inspection; otherwise the next
|
||||
proxy is drawn from the session's proxy_pool when present. Responses are
|
||||
parsed as JSON when possible.
|
||||
|
||||
Args:
|
||||
session: aiohttp or curl_cffi session to send through.
|
||||
url: Request URL.
|
||||
method: HTTP method.
|
||||
params: Query parameters.
|
||||
json_data: JSON body.
|
||||
data: Form body.
|
||||
reraise_json_error: Raise JSONDecodeError instead of returning raw
|
||||
text when the response is not valid JSON.
|
||||
development_proxy: Local proxy URL that overrides the pool and
|
||||
disables verification.
|
||||
|
||||
Returns:
|
||||
Parsed JSON dict, or raw response text when parsing fails and
|
||||
reraise_json_error is False.
|
||||
|
||||
Raises:
|
||||
json.JSONDecodeError: When parsing fails and reraise_json_error is
|
||||
True.
|
||||
"""
|
||||
verify = True
|
||||
if development_proxy:
|
||||
proxy = development_proxy
|
||||
|
||||
Reference in new issue
Block a user