added proxy managament from dsqls module
This commit is contained in:
1 parent
1cb39e2e0d
commit
4f7dd161ef
3 files changed
+23
-13
No files matched your search
+5
-1
@@ -8,6 +8,7 @@ version = "0.1.0"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"aiohttp>=3.14.1",
|
||||
"aioproxies",
|
||||
"curl-cffi>=0.15.0",
|
||||
"fake-useragent>=2.2.0",
|
||||
"motor>=3.7.1",
|
||||
@@ -19,4 +20,7 @@ dependencies = [
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["src"]
|
||||
include = ["monitor_utils*"]
|
||||
include = ["monitor_utils*"]
|
||||
|
||||
[tool.uv.sources]
|
||||
aioproxies = { git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git" }
|
||||
@@ -8,6 +8,7 @@ from typing import AsyncGenerator
|
||||
|
||||
import aiohttp
|
||||
import requests
|
||||
from aioproxies import AioProxies
|
||||
from curl_cffi.requests import AsyncSession
|
||||
from curl_cffi.requests.errors import RequestsError
|
||||
from curl_cffi.requests.exceptions import Timeout as CurlTimeout
|
||||
@@ -21,7 +22,6 @@ from tenacity import (
|
||||
wait_fixed,
|
||||
)
|
||||
|
||||
from monitor_utils.config import get_proxies
|
||||
from monitor_utils.constants import (
|
||||
CLIENT_TIMEOUT,
|
||||
TIMEOUT_SECONDS,
|
||||
@@ -40,12 +40,14 @@ async def get_base_session(
|
||||
cookies: dict | None = None,
|
||||
connector: aiohttp.TCPConnector | None = None,
|
||||
timeout: aiohttp.ClientTimeout = CLIENT_TIMEOUT,
|
||||
proxies: AioProxies | None = None,
|
||||
) -> AsyncGenerator[aiohttp.ClientSession, None]:
|
||||
headers = headers if headers else {}
|
||||
headers.setdefault("User-Agent", UA.random)
|
||||
session = aiohttp.ClientSession(
|
||||
headers=headers, cookies=cookies, connector=connector, timeout=timeout
|
||||
)
|
||||
session.proxy_pool = proxies
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
@@ -54,7 +56,9 @@ async def get_base_session(
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_tls_session(
|
||||
headers: dict | None = None, timeout: int = TIMEOUT_SECONDS
|
||||
headers: dict | None = None,
|
||||
timeout: int = TIMEOUT_SECONDS,
|
||||
proxies: AioProxies | None = None,
|
||||
) -> AsyncGenerator[AsyncSession, None]:
|
||||
headers = headers if headers else {}
|
||||
headers.setdefault("User-Agent", UA.chrome)
|
||||
@@ -63,15 +67,15 @@ async def get_tls_session(
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
) as session:
|
||||
session.proxy_pool = proxies
|
||||
yield session
|
||||
|
||||
|
||||
def build_session_details(
|
||||
session: aiohttp.ClientSession | AsyncSession, headers: dict = None
|
||||
):
|
||||
# ik headers doesn't match tls i'm lazy and the antibots used don't care
|
||||
headers = headers or {}
|
||||
headers.setdefault("User-Agent", FakeUserAgent().random)
|
||||
headers.setdefault("User-Agent", UA.random)
|
||||
session.headers.update(headers)
|
||||
|
||||
|
||||
@@ -106,7 +110,8 @@ async def send_request(
|
||||
proxy = development_proxy
|
||||
verify = False
|
||||
else:
|
||||
proxy = random.choice(get_proxies())
|
||||
manager = getattr(session, "proxy_pool", None)
|
||||
proxy = manager.next().url() if manager else None
|
||||
|
||||
if isinstance(session, aiohttp.ClientSession):
|
||||
async with session.request(
|
||||
@@ -121,12 +126,6 @@ async def send_request(
|
||||
response.raise_for_status()
|
||||
text = await response.text()
|
||||
else:
|
||||
if (
|
||||
url
|
||||
== "https://www.hp.com/wcs/resources/store/10151/component/configure/addtocartconfig"
|
||||
):
|
||||
pass
|
||||
|
||||
response = await session.request(
|
||||
method=method.upper(),
|
||||
url=url,
|
||||
@@ -145,4 +144,4 @@ async def send_request(
|
||||
if reraise_json_error:
|
||||
raise
|
||||
|
||||
return text
|
||||
return text
|
||||
@@ -148,6 +148,11 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aioproxies"
|
||||
version = "0.2.2"
|
||||
source = { git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git#eef4b25f07cba8f44c376228d4a1a85f526ce031" }
|
||||
|
||||
[[package]]
|
||||
name = "aiosignal"
|
||||
version = "1.4.0"
|
||||
@@ -592,6 +597,7 @@ version = "0.1.0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "aiohttp" },
|
||||
{ name = "aioproxies" },
|
||||
{ name = "curl-cffi" },
|
||||
{ name = "fake-useragent" },
|
||||
{ name = "motor" },
|
||||
@@ -604,6 +610,7 @@ dependencies = [
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "aiohttp", specifier = ">=3.14.1" },
|
||||
{ name = "aioproxies", git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git" },
|
||||
{ name = "curl-cffi", specifier = ">=0.15.0" },
|
||||
{ name = "fake-useragent", specifier = ">=2.2.0" },
|
||||
{ name = "motor", specifier = ">=3.7.1" },
|
||||
|
||||
Reference in new issue
Block a user