From 4f7dd161efa25b52fbc61e7a7d00844e48d461bc Mon Sep 17 00:00:00 2001 From: tpv Date: Tue, 30 Jun 2026 12:11:20 -0700 Subject: [PATCH] added proxy managament from dsqls module --- pyproject.toml | 6 +++++- src/monitor_utils/sessions.py | 23 +++++++++++------------ uv.lock | 7 +++++++ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 39316e1..34e8438 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ version = "0.1.0" requires-python = ">=3.10" dependencies = [ "aiohttp>=3.14.1", + "aioproxies", "curl-cffi>=0.15.0", "fake-useragent>=2.2.0", "motor>=3.7.1", @@ -19,4 +20,7 @@ dependencies = [ [tool.setuptools.packages.find] where = ["src"] -include = ["monitor_utils*"] \ No newline at end of file +include = ["monitor_utils*"] + +[tool.uv.sources] +aioproxies = { git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git" } diff --git a/src/monitor_utils/sessions.py b/src/monitor_utils/sessions.py index f8c1e30..30a2019 100644 --- a/src/monitor_utils/sessions.py +++ b/src/monitor_utils/sessions.py @@ -8,6 +8,7 @@ from typing import AsyncGenerator import aiohttp import requests +from aioproxies import AioProxies from curl_cffi.requests import AsyncSession from curl_cffi.requests.errors import RequestsError from curl_cffi.requests.exceptions import Timeout as CurlTimeout @@ -21,7 +22,6 @@ from tenacity import ( wait_fixed, ) -from monitor_utils.config import get_proxies from monitor_utils.constants import ( CLIENT_TIMEOUT, TIMEOUT_SECONDS, @@ -40,12 +40,14 @@ async def get_base_session( cookies: dict | None = None, connector: aiohttp.TCPConnector | None = None, timeout: aiohttp.ClientTimeout = CLIENT_TIMEOUT, + proxies: AioProxies | None = None, ) -> AsyncGenerator[aiohttp.ClientSession, None]: headers = headers if headers else {} headers.setdefault("User-Agent", UA.random) session = aiohttp.ClientSession( headers=headers, cookies=cookies, connector=connector, timeout=timeout ) + session.proxy_pool = proxies try: yield session finally: @@ -54,7 +56,9 @@ async def get_base_session( @asynccontextmanager async def get_tls_session( - headers: dict | None = None, timeout: int = TIMEOUT_SECONDS + headers: dict | None = None, + timeout: int = TIMEOUT_SECONDS, + proxies: AioProxies | None = None, ) -> AsyncGenerator[AsyncSession, None]: headers = headers if headers else {} headers.setdefault("User-Agent", UA.chrome) @@ -63,15 +67,15 @@ async def get_tls_session( timeout=timeout, headers=headers, ) as session: + session.proxy_pool = proxies yield session def build_session_details( session: aiohttp.ClientSession | AsyncSession, headers: dict = None ): - # ik headers doesn't match tls i'm lazy and the antibots used don't care headers = headers or {} - headers.setdefault("User-Agent", FakeUserAgent().random) + headers.setdefault("User-Agent", UA.random) session.headers.update(headers) @@ -106,7 +110,8 @@ async def send_request( proxy = development_proxy verify = False else: - proxy = random.choice(get_proxies()) + manager = getattr(session, "proxy_pool", None) + proxy = manager.next().url() if manager else None if isinstance(session, aiohttp.ClientSession): async with session.request( @@ -121,12 +126,6 @@ async def send_request( response.raise_for_status() text = await response.text() else: - if ( - url - == "https://www.hp.com/wcs/resources/store/10151/component/configure/addtocartconfig" - ): - pass - response = await session.request( method=method.upper(), url=url, @@ -145,4 +144,4 @@ async def send_request( if reraise_json_error: raise - return text + return text \ No newline at end of file diff --git a/uv.lock b/uv.lock index 588ae5a..e4c20ed 100644 --- a/uv.lock +++ b/uv.lock @@ -148,6 +148,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" }, ] +[[package]] +name = "aioproxies" +version = "0.2.2" +source = { git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git#eef4b25f07cba8f44c376228d4a1a85f526ce031" } + [[package]] name = "aiosignal" version = "1.4.0" @@ -592,6 +597,7 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, + { name = "aioproxies" }, { name = "curl-cffi" }, { name = "fake-useragent" }, { name = "motor" }, @@ -604,6 +610,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "aiohttp", specifier = ">=3.14.1" }, + { name = "aioproxies", git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git" }, { name = "curl-cffi", specifier = ">=0.15.0" }, { name = "fake-useragent", specifier = ">=2.2.0" }, { name = "motor", specifier = ">=3.7.1" },