added proxy managament from dsqls module
This commit is contained in:
1 parent
1cb39e2e0d
commit
4f7dd161ef
3 files changed
+21
-11
No files matched your search
@@ -8,6 +8,7 @@ version = "0.1.0"
|
|||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aiohttp>=3.14.1",
|
"aiohttp>=3.14.1",
|
||||||
|
"aioproxies",
|
||||||
"curl-cffi>=0.15.0",
|
"curl-cffi>=0.15.0",
|
||||||
"fake-useragent>=2.2.0",
|
"fake-useragent>=2.2.0",
|
||||||
"motor>=3.7.1",
|
"motor>=3.7.1",
|
||||||
@@ -20,3 +21,6 @@ dependencies = [
|
|||||||
[tool.setuptools.packages.find]
|
[tool.setuptools.packages.find]
|
||||||
where = ["src"]
|
where = ["src"]
|
||||||
include = ["monitor_utils*"]
|
include = ["monitor_utils*"]
|
||||||
|
|
||||||
|
[tool.uv.sources]
|
||||||
|
aioproxies = { git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git" }
|
||||||
@@ -8,6 +8,7 @@ from typing import AsyncGenerator
|
|||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
import requests
|
import requests
|
||||||
|
from aioproxies import AioProxies
|
||||||
from curl_cffi.requests import AsyncSession
|
from curl_cffi.requests import AsyncSession
|
||||||
from curl_cffi.requests.errors import RequestsError
|
from curl_cffi.requests.errors import RequestsError
|
||||||
from curl_cffi.requests.exceptions import Timeout as CurlTimeout
|
from curl_cffi.requests.exceptions import Timeout as CurlTimeout
|
||||||
@@ -21,7 +22,6 @@ from tenacity import (
|
|||||||
wait_fixed,
|
wait_fixed,
|
||||||
)
|
)
|
||||||
|
|
||||||
from monitor_utils.config import get_proxies
|
|
||||||
from monitor_utils.constants import (
|
from monitor_utils.constants import (
|
||||||
CLIENT_TIMEOUT,
|
CLIENT_TIMEOUT,
|
||||||
TIMEOUT_SECONDS,
|
TIMEOUT_SECONDS,
|
||||||
@@ -40,12 +40,14 @@ async def get_base_session(
|
|||||||
cookies: dict | None = None,
|
cookies: dict | None = None,
|
||||||
connector: aiohttp.TCPConnector | None = None,
|
connector: aiohttp.TCPConnector | None = None,
|
||||||
timeout: aiohttp.ClientTimeout = CLIENT_TIMEOUT,
|
timeout: aiohttp.ClientTimeout = CLIENT_TIMEOUT,
|
||||||
|
proxies: AioProxies | None = None,
|
||||||
) -> AsyncGenerator[aiohttp.ClientSession, None]:
|
) -> AsyncGenerator[aiohttp.ClientSession, None]:
|
||||||
headers = headers if headers else {}
|
headers = headers if headers else {}
|
||||||
headers.setdefault("User-Agent", UA.random)
|
headers.setdefault("User-Agent", UA.random)
|
||||||
session = aiohttp.ClientSession(
|
session = aiohttp.ClientSession(
|
||||||
headers=headers, cookies=cookies, connector=connector, timeout=timeout
|
headers=headers, cookies=cookies, connector=connector, timeout=timeout
|
||||||
)
|
)
|
||||||
|
session.proxy_pool = proxies
|
||||||
try:
|
try:
|
||||||
yield session
|
yield session
|
||||||
finally:
|
finally:
|
||||||
@@ -54,7 +56,9 @@ async def get_base_session(
|
|||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def get_tls_session(
|
async def get_tls_session(
|
||||||
headers: dict | None = None, timeout: int = TIMEOUT_SECONDS
|
headers: dict | None = None,
|
||||||
|
timeout: int = TIMEOUT_SECONDS,
|
||||||
|
proxies: AioProxies | None = None,
|
||||||
) -> AsyncGenerator[AsyncSession, None]:
|
) -> AsyncGenerator[AsyncSession, None]:
|
||||||
headers = headers if headers else {}
|
headers = headers if headers else {}
|
||||||
headers.setdefault("User-Agent", UA.chrome)
|
headers.setdefault("User-Agent", UA.chrome)
|
||||||
@@ -63,15 +67,15 @@ async def get_tls_session(
|
|||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
headers=headers,
|
headers=headers,
|
||||||
) as session:
|
) as session:
|
||||||
|
session.proxy_pool = proxies
|
||||||
yield session
|
yield session
|
||||||
|
|
||||||
|
|
||||||
def build_session_details(
|
def build_session_details(
|
||||||
session: aiohttp.ClientSession | AsyncSession, headers: dict = None
|
session: aiohttp.ClientSession | AsyncSession, headers: dict = None
|
||||||
):
|
):
|
||||||
# ik headers doesn't match tls i'm lazy and the antibots used don't care
|
|
||||||
headers = headers or {}
|
headers = headers or {}
|
||||||
headers.setdefault("User-Agent", FakeUserAgent().random)
|
headers.setdefault("User-Agent", UA.random)
|
||||||
session.headers.update(headers)
|
session.headers.update(headers)
|
||||||
|
|
||||||
|
|
||||||
@@ -106,7 +110,8 @@ async def send_request(
|
|||||||
proxy = development_proxy
|
proxy = development_proxy
|
||||||
verify = False
|
verify = False
|
||||||
else:
|
else:
|
||||||
proxy = random.choice(get_proxies())
|
manager = getattr(session, "proxy_pool", None)
|
||||||
|
proxy = manager.next().url() if manager else None
|
||||||
|
|
||||||
if isinstance(session, aiohttp.ClientSession):
|
if isinstance(session, aiohttp.ClientSession):
|
||||||
async with session.request(
|
async with session.request(
|
||||||
@@ -121,12 +126,6 @@ async def send_request(
|
|||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
text = await response.text()
|
text = await response.text()
|
||||||
else:
|
else:
|
||||||
if (
|
|
||||||
url
|
|
||||||
== "https://www.hp.com/wcs/resources/store/10151/component/configure/addtocartconfig"
|
|
||||||
):
|
|
||||||
pass
|
|
||||||
|
|
||||||
response = await session.request(
|
response = await session.request(
|
||||||
method=method.upper(),
|
method=method.upper(),
|
||||||
url=url,
|
url=url,
|
||||||
|
|||||||
@@ -148,6 +148,11 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" },
|
{ url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "aioproxies"
|
||||||
|
version = "0.2.2"
|
||||||
|
source = { git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git#eef4b25f07cba8f44c376228d4a1a85f526ce031" }
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aiosignal"
|
name = "aiosignal"
|
||||||
version = "1.4.0"
|
version = "1.4.0"
|
||||||
@@ -592,6 +597,7 @@ version = "0.1.0"
|
|||||||
source = { editable = "." }
|
source = { editable = "." }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "aiohttp" },
|
{ name = "aiohttp" },
|
||||||
|
{ name = "aioproxies" },
|
||||||
{ name = "curl-cffi" },
|
{ name = "curl-cffi" },
|
||||||
{ name = "fake-useragent" },
|
{ name = "fake-useragent" },
|
||||||
{ name = "motor" },
|
{ name = "motor" },
|
||||||
@@ -604,6 +610,7 @@ dependencies = [
|
|||||||
[package.metadata]
|
[package.metadata]
|
||||||
requires-dist = [
|
requires-dist = [
|
||||||
{ name = "aiohttp", specifier = ">=3.14.1" },
|
{ name = "aiohttp", specifier = ">=3.14.1" },
|
||||||
|
{ name = "aioproxies", git = "https://git.rethinkstudios.io/rethink-public/aioproxies.git" },
|
||||||
{ name = "curl-cffi", specifier = ">=0.15.0" },
|
{ name = "curl-cffi", specifier = ">=0.15.0" },
|
||||||
{ name = "fake-useragent", specifier = ">=2.2.0" },
|
{ name = "fake-useragent", specifier = ">=2.2.0" },
|
||||||
{ name = "motor", specifier = ">=3.7.1" },
|
{ name = "motor", specifier = ">=3.7.1" },
|
||||||
|
|||||||
Reference in new issue
Block a user