added proxy managament from dsqls module

This commit is contained in:
tpv committed 2026-06-30 12:11:20 -07:00
1 parent 1cb39e2e0d
commit 4f7dd161ef
3 files changed
+21 -11

No files matched your search

+4
View File
@@ -8,6 +8,7 @@ version = "0.1.0"
requires-python = ">=3.10" requires-python = ">=3.10"
dependencies = [ dependencies = [
"aiohttp>=3.14.1", "aiohttp>=3.14.1",
"aioproxies",
"curl-cffi>=0.15.0", "curl-cffi>=0.15.0",
"fake-useragent>=2.2.0", "fake-useragent>=2.2.0",
"motor>=3.7.1", "motor>=3.7.1",
@@ -20,3 +21,6 @@ dependencies = [
[tool.setuptools.packages.find] [tool.setuptools.packages.find]
where = ["src"] where = ["src"]
include = ["monitor_utils*"] include = ["monitor_utils*"]
[tool.uv.sources]
aioproxies = { git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git" }
+10 -11
View File
@@ -8,6 +8,7 @@ from typing import AsyncGenerator
import aiohttp import aiohttp
import requests import requests
from aioproxies import AioProxies
from curl_cffi.requests import AsyncSession from curl_cffi.requests import AsyncSession
from curl_cffi.requests.errors import RequestsError from curl_cffi.requests.errors import RequestsError
from curl_cffi.requests.exceptions import Timeout as CurlTimeout from curl_cffi.requests.exceptions import Timeout as CurlTimeout
@@ -21,7 +22,6 @@ from tenacity import (
wait_fixed, wait_fixed,
) )
from monitor_utils.config import get_proxies
from monitor_utils.constants import ( from monitor_utils.constants import (
CLIENT_TIMEOUT, CLIENT_TIMEOUT,
TIMEOUT_SECONDS, TIMEOUT_SECONDS,
@@ -40,12 +40,14 @@ async def get_base_session(
cookies: dict | None = None, cookies: dict | None = None,
connector: aiohttp.TCPConnector | None = None, connector: aiohttp.TCPConnector | None = None,
timeout: aiohttp.ClientTimeout = CLIENT_TIMEOUT, timeout: aiohttp.ClientTimeout = CLIENT_TIMEOUT,
proxies: AioProxies | None = None,
) -> AsyncGenerator[aiohttp.ClientSession, None]: ) -> AsyncGenerator[aiohttp.ClientSession, None]:
headers = headers if headers else {} headers = headers if headers else {}
headers.setdefault("User-Agent", UA.random) headers.setdefault("User-Agent", UA.random)
session = aiohttp.ClientSession( session = aiohttp.ClientSession(
headers=headers, cookies=cookies, connector=connector, timeout=timeout headers=headers, cookies=cookies, connector=connector, timeout=timeout
) )
session.proxy_pool = proxies
try: try:
yield session yield session
finally: finally:
@@ -54,7 +56,9 @@ async def get_base_session(
@asynccontextmanager @asynccontextmanager
async def get_tls_session( async def get_tls_session(
headers: dict | None = None, timeout: int = TIMEOUT_SECONDS headers: dict | None = None,
timeout: int = TIMEOUT_SECONDS,
proxies: AioProxies | None = None,
) -> AsyncGenerator[AsyncSession, None]: ) -> AsyncGenerator[AsyncSession, None]:
headers = headers if headers else {} headers = headers if headers else {}
headers.setdefault("User-Agent", UA.chrome) headers.setdefault("User-Agent", UA.chrome)
@@ -63,15 +67,15 @@ async def get_tls_session(
timeout=timeout, timeout=timeout,
headers=headers, headers=headers,
) as session: ) as session:
session.proxy_pool = proxies
yield session yield session
def build_session_details( def build_session_details(
session: aiohttp.ClientSession | AsyncSession, headers: dict = None session: aiohttp.ClientSession | AsyncSession, headers: dict = None
): ):
# ik headers doesn't match tls i'm lazy and the antibots used don't care
headers = headers or {} headers = headers or {}
headers.setdefault("User-Agent", FakeUserAgent().random) headers.setdefault("User-Agent", UA.random)
session.headers.update(headers) session.headers.update(headers)
@@ -106,7 +110,8 @@ async def send_request(
proxy = development_proxy proxy = development_proxy
verify = False verify = False
else: else:
proxy = random.choice(get_proxies()) manager = getattr(session, "proxy_pool", None)
proxy = manager.next().url() if manager else None
if isinstance(session, aiohttp.ClientSession): if isinstance(session, aiohttp.ClientSession):
async with session.request( async with session.request(
@@ -121,12 +126,6 @@ async def send_request(
response.raise_for_status() response.raise_for_status()
text = await response.text() text = await response.text()
else: else:
if (
url
== "https://www.hp.com/wcs/resources/store/10151/component/configure/addtocartconfig"
):
pass
response = await session.request( response = await session.request(
method=method.upper(), method=method.upper(),
url=url, url=url,
Generated
+7
View File
@@ -148,6 +148,11 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" }, { url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" },
] ]
[[package]]
name = "aioproxies"
version = "0.2.2"
source = { git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git#eef4b25f07cba8f44c376228d4a1a85f526ce031" }
[[package]] [[package]]
name = "aiosignal" name = "aiosignal"
version = "1.4.0" version = "1.4.0"
@@ -592,6 +597,7 @@ version = "0.1.0"
source = { editable = "." } source = { editable = "." }
dependencies = [ dependencies = [
{ name = "aiohttp" }, { name = "aiohttp" },
{ name = "aioproxies" },
{ name = "curl-cffi" }, { name = "curl-cffi" },
{ name = "fake-useragent" }, { name = "fake-useragent" },
{ name = "motor" }, { name = "motor" },
@@ -604,6 +610,7 @@ dependencies = [
[package.metadata] [package.metadata]
requires-dist = [ requires-dist = [
{ name = "aiohttp", specifier = ">=3.14.1" }, { name = "aiohttp", specifier = ">=3.14.1" },
{ name = "aioproxies", git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git" },
{ name = "curl-cffi", specifier = ">=0.15.0" }, { name = "curl-cffi", specifier = ">=0.15.0" },
{ name = "fake-useragent", specifier = ">=2.2.0" }, { name = "fake-useragent", specifier = ">=2.2.0" },
{ name = "motor", specifier = ">=3.7.1" }, { name = "motor", specifier = ">=3.7.1" },