added proxy managament from dsqls module

This commit is contained in:
tpv committed 2026-06-30 12:11:20 -07:00
1 parent 1cb39e2e0d
commit 4f7dd161ef
3 files changed
+23 -13

No files matched your search

+5 -1
View File
@@ -8,6 +8,7 @@ version = "0.1.0"
requires-python = ">=3.10"
dependencies = [
"aiohttp>=3.14.1",
"aioproxies",
"curl-cffi>=0.15.0",
"fake-useragent>=2.2.0",
"motor>=3.7.1",
@@ -19,4 +20,7 @@ dependencies = [
[tool.setuptools.packages.find]
where = ["src"]
include = ["monitor_utils*"]
include = ["monitor_utils*"]
[tool.uv.sources]
aioproxies = { git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git" }
+11 -12
View File
@@ -8,6 +8,7 @@ from typing import AsyncGenerator
import aiohttp
import requests
from aioproxies import AioProxies
from curl_cffi.requests import AsyncSession
from curl_cffi.requests.errors import RequestsError
from curl_cffi.requests.exceptions import Timeout as CurlTimeout
@@ -21,7 +22,6 @@ from tenacity import (
wait_fixed,
)
from monitor_utils.config import get_proxies
from monitor_utils.constants import (
CLIENT_TIMEOUT,
TIMEOUT_SECONDS,
@@ -40,12 +40,14 @@ async def get_base_session(
cookies: dict | None = None,
connector: aiohttp.TCPConnector | None = None,
timeout: aiohttp.ClientTimeout = CLIENT_TIMEOUT,
proxies: AioProxies | None = None,
) -> AsyncGenerator[aiohttp.ClientSession, None]:
headers = headers if headers else {}
headers.setdefault("User-Agent", UA.random)
session = aiohttp.ClientSession(
headers=headers, cookies=cookies, connector=connector, timeout=timeout
)
session.proxy_pool = proxies
try:
yield session
finally:
@@ -54,7 +56,9 @@ async def get_base_session(
@asynccontextmanager
async def get_tls_session(
headers: dict | None = None, timeout: int = TIMEOUT_SECONDS
headers: dict | None = None,
timeout: int = TIMEOUT_SECONDS,
proxies: AioProxies | None = None,
) -> AsyncGenerator[AsyncSession, None]:
headers = headers if headers else {}
headers.setdefault("User-Agent", UA.chrome)
@@ -63,15 +67,15 @@ async def get_tls_session(
timeout=timeout,
headers=headers,
) as session:
session.proxy_pool = proxies
yield session
def build_session_details(
session: aiohttp.ClientSession | AsyncSession, headers: dict = None
):
# ik headers doesn't match tls i'm lazy and the antibots used don't care
headers = headers or {}
headers.setdefault("User-Agent", FakeUserAgent().random)
headers.setdefault("User-Agent", UA.random)
session.headers.update(headers)
@@ -106,7 +110,8 @@ async def send_request(
proxy = development_proxy
verify = False
else:
proxy = random.choice(get_proxies())
manager = getattr(session, "proxy_pool", None)
proxy = manager.next().url() if manager else None
if isinstance(session, aiohttp.ClientSession):
async with session.request(
@@ -121,12 +126,6 @@ async def send_request(
response.raise_for_status()
text = await response.text()
else:
if (
url
== "https://www.hp.com/wcs/resources/store/10151/component/configure/addtocartconfig"
):
pass
response = await session.request(
method=method.upper(),
url=url,
@@ -145,4 +144,4 @@ async def send_request(
if reraise_json_error:
raise
return text
return text
Generated
+7
View File
@@ -148,6 +148,11 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" },
]
[[package]]
name = "aioproxies"
version = "0.2.2"
source = { git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git#eef4b25f07cba8f44c376228d4a1a85f526ce031" }
[[package]]
name = "aiosignal"
version = "1.4.0"
@@ -592,6 +597,7 @@ version = "0.1.0"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },
{ name = "aioproxies" },
{ name = "curl-cffi" },
{ name = "fake-useragent" },
{ name = "motor" },
@@ -604,6 +610,7 @@ dependencies = [
[package.metadata]
requires-dist = [
{ name = "aiohttp", specifier = ">=3.14.1" },
{ name = "aioproxies", git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git" },
{ name = "curl-cffi", specifier = ">=0.15.0" },
{ name = "fake-useragent", specifier = ">=2.2.0" },
{ name = "motor", specifier = ">=3.7.1" },