fixed a small database issue with price drop tracking and added doc strings

This commit is contained in:
tpv committed 2026-07-03 16:39:43 -07:00
1 parent a61dbdfaa9
commit 99d660ee52
6 files changed
+172 -4

No files matched your search

+70 -1
View File
@@ -1,4 +1,9 @@
# ik ua/tls doesn't match headers/header order but targeted antibots don't seem to care might fix later
"""Shared HTTP session helpers for all monitors.
Known gap: UA/TLS fingerprint does not always match headers/header order,
but targeted antibots do not currently seem to care.
"""
import asyncio
import json
import logging
@@ -42,6 +47,22 @@ async def get_base_session(
timeout: aiohttp.ClientTimeout = CLIENT_TIMEOUT,
proxies: AioProxies | None = None,
) -> AsyncGenerator[aiohttp.ClientSession, None]:
"""Yield an aiohttp session with a random User-Agent and attached proxy pool.
The proxy pool is stored on the session as proxy_pool for send_request to
rotate per request. The session is always closed on exit.
Args:
headers: Headers applied to the session; User-Agent is defaulted if
absent.
cookies: Cookies applied to the session.
connector: Optional TCP connector.
timeout: Client timeout applied to the session.
proxies: Proxy pool attached to the session.
Yields:
The configured aiohttp session.
"""
headers = headers if headers else {}
headers.setdefault("User-Agent", UA.random)
session = aiohttp.ClientSession(
@@ -60,6 +81,20 @@ async def get_tls_session(
timeout: int = TIMEOUT_SECONDS,
proxies: AioProxies | None = None,
) -> AsyncGenerator[AsyncSession, None]:
"""Yield a curl_cffi session impersonating a random client fingerprint.
The proxy pool is stored on the session as proxy_pool for send_request to
rotate per request.
Args:
headers: Headers applied to the session; a Chrome User-Agent is
defaulted if absent.
timeout: Request timeout in seconds.
proxies: Proxy pool attached to the session.
Yields:
The configured curl_cffi session.
"""
headers = headers if headers else {}
headers.setdefault("User-Agent", UA.chrome)
async with AsyncSession(
@@ -74,6 +109,12 @@ async def get_tls_session(
def build_session_details(
session: aiohttp.ClientSession | AsyncSession, headers: dict = None
):
"""Merge headers into an existing session, defaulting a random User-Agent.
Args:
session: Session whose headers are updated in place.
headers: Headers to merge; User-Agent is defaulted if absent.
"""
headers = headers or {}
headers.setdefault("User-Agent", UA.random)
session.headers.update(headers)
@@ -105,6 +146,34 @@ async def send_request(
reraise_json_error: bool = False,
development_proxy: str | None = None,
) -> dict | str:
"""Send a request through either session type with retries and proxy rotation.
Retries transport errors and timeouts up to RETRY_LIMIT with a fixed wait.
When development_proxy is set it overrides the session's proxy pool and
disables TLS verification for local traffic inspection; otherwise the next
proxy is drawn from the session's proxy_pool when present. Responses are
parsed as JSON when possible.
Args:
session: aiohttp or curl_cffi session to send through.
url: Request URL.
method: HTTP method.
params: Query parameters.
json_data: JSON body.
data: Form body.
reraise_json_error: Raise JSONDecodeError instead of returning raw
text when the response is not valid JSON.
development_proxy: Local proxy URL that overrides the pool and
disables verification.
Returns:
Parsed JSON dict, or raw response text when parsing fails and
reraise_json_error is False.
Raises:
json.JSONDecodeError: When parsing fails and reraise_json_error is
True.
"""
verify = True
if development_proxy:
proxy = development_proxy