143 lines
3.7 KiB
Python
143 lines
3.7 KiB
Python
# ik ua/tls doesn't match headers/header order but targeted antibots don't seem to care might fix later
|
|
import asyncio
|
|
import random
|
|
import json
|
|
import logging
|
|
from contextlib import asynccontextmanager
|
|
from typing import AsyncGenerator
|
|
|
|
import aiohttp
|
|
import requests
|
|
from curl_cffi.requests import AsyncSession
|
|
from curl_cffi.requests.errors import RequestsError
|
|
from curl_cffi.requests.exceptions import Timeout as CurlTimeout
|
|
from fake_useragent import FakeUserAgent
|
|
from tenacity import (
|
|
after_log,
|
|
before_log,
|
|
retry,
|
|
retry_if_exception_type,
|
|
stop_after_attempt,
|
|
wait_fixed,
|
|
)
|
|
|
|
from monitor_utils.config import get_proxies
|
|
from monitor_utils.constants import (
|
|
CLIENT_TIMEOUT,
|
|
TIMEOUT_SECONDS,
|
|
CLIENT_IDENTIFIERS,
|
|
RETRY_LIMIT,
|
|
RETRY_WAIT,
|
|
)
|
|
|
|
LOGGER = logging.getLogger(__name__)
|
|
UA = FakeUserAgent()
|
|
|
|
|
|
@asynccontextmanager
|
|
async def get_base_session(
|
|
headers: dict | None = None,
|
|
cookies: dict | None = None,
|
|
connector: aiohttp.TCPConnector | None = None,
|
|
timeout: aiohttp.ClientTimeout = CLIENT_TIMEOUT,
|
|
) -> AsyncGenerator[aiohttp.ClientSession, None]:
|
|
headers = headers if headers else {}
|
|
headers.setdefault("User-Agent", UA.random)
|
|
session = aiohttp.ClientSession(
|
|
headers=headers, cookies=cookies, connector=connector, timeout=timeout
|
|
)
|
|
try:
|
|
yield session
|
|
finally:
|
|
await session.close()
|
|
|
|
|
|
@asynccontextmanager
|
|
async def get_tls_session(
|
|
headers: dict | None = None, timeout: int = TIMEOUT_SECONDS
|
|
) -> AsyncGenerator[AsyncSession, None]:
|
|
headers = headers if headers else {}
|
|
headers.setdefault("User-Agent", UA.chrome)
|
|
async with AsyncSession(
|
|
impersonate=random.choice(CLIENT_IDENTIFIERS),
|
|
timeout=timeout,
|
|
headers=headers,
|
|
) as session:
|
|
yield session
|
|
|
|
|
|
def build_session_details(
|
|
session: aiohttp.ClientSession | AsyncSession, headers: dict = None
|
|
):
|
|
# ik headers doesn't match tls i'm lazy and the antibots used don't care
|
|
headers = headers or {}
|
|
headers.setdefault("User-Agent", FakeUserAgent().random)
|
|
session.headers.update(headers)
|
|
|
|
|
|
@retry(
|
|
stop=stop_after_attempt(RETRY_LIMIT),
|
|
wait=wait_fixed(RETRY_WAIT),
|
|
retry=retry_if_exception_type(
|
|
(
|
|
aiohttp.ClientError,
|
|
asyncio.TimeoutError,
|
|
RequestsError,
|
|
requests.Timeout,
|
|
CurlTimeout,
|
|
)
|
|
),
|
|
before=before_log(LOGGER, logging.DEBUG),
|
|
after=after_log(LOGGER, logging.DEBUG),
|
|
reraise=True,
|
|
)
|
|
async def send_request(
|
|
session: aiohttp.ClientSession | AsyncSession,
|
|
url: str,
|
|
method: str,
|
|
params: dict | None = None,
|
|
json_data: dict | None = None,
|
|
data: dict | None = None,
|
|
reraise_json_error: bool = False,
|
|
development_proxy: str | None = None,
|
|
) -> dict | str:
|
|
skip_verify = False
|
|
if development_proxy:
|
|
proxy = development_proxy
|
|
skip_verify = True
|
|
else:
|
|
proxy = random.choice(get_proxies())
|
|
|
|
if isinstance(session, aiohttp.ClientSession):
|
|
async with session.request(
|
|
method=method,
|
|
url=url,
|
|
params=params,
|
|
json=json_data,
|
|
data=data,
|
|
proxy=proxy,
|
|
ssl=skip_verify,
|
|
) as response:
|
|
response.raise_for_status()
|
|
text = await response.text()
|
|
else:
|
|
response = await session.request(
|
|
method=method.upper(),
|
|
url=url,
|
|
params=params,
|
|
json=json_data,
|
|
data=data,
|
|
proxy=proxy,
|
|
verify=skip_verify,
|
|
)
|
|
response.raise_for_status()
|
|
text = response.text
|
|
|
|
try:
|
|
return json.loads(text)
|
|
except json.JSONDecodeError:
|
|
if reraise_json_error:
|
|
raise
|
|
|
|
return text
|