small bug fix

src/monitor_utils/config.py

@@ -0,0 +1,65 @@
+import logging
+import json
+import pathlib
+from functools import cache
+from enum import Enum
+
+from monitor_utils.src.constants import (
+    PROXIES_PATH,
+    PROJECT_ROOT,
+    PRODUCTION_BOT_TOKEN,
+    DEVELOPMENT_BOT_TOKEN,
+)
+
+RESERVED = frozenset(logging.LogRecord("", 0, "", 0, "", (), None).__dict__)
+
+
+class DevelopmentMode(str, Enum):
+    PRODUCTION = "PRODUCTION"
+    DEVELOPMENT = "DEVELOPMENT"
+
+
+def is_production(development_mode: DevelopmentMode) -> bool:
+    return development_mode == DevelopmentMode.PRODUCTION
+
+
+def get_bot_token(is_production: bool) -> str:
+    if is_production:
+        return PRODUCTION_BOT_TOKEN
+
+    return DEVELOPMENT_BOT_TOKEN
+
+
+@cache
+def get_proxies() -> list[str]:
+    with PROXIES_PATH.open() as f:
+        proxies = []
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+
+            parts = line.split(":", 3)
+            if len(parts) == 4 and not parts[1].startswith("//"):
+                host, port, user, password = parts
+                proxies.append(f"http://{user}:{password}@{host}:{port}")
+            else:
+                proxies.append(f"http://{line}")
+
+    if not proxies:
+        raise ValueError("Proxy list cannot be empty")
+
+    return proxies
+
+
+def get_config(
+    monitor_name: str, use_production_path: bool = False
+) -> tuple[dict, pathlib.Path]:
+    monitor_name = monitor_name.replace(".json", "")
+    if use_production_path:
+        config_path = PROJECT_ROOT / "configs" / f"{monitor_name}.json"
+    else:
+        config_path = PROJECT_ROOT / "tests" / "configs" / f"{monitor_name}.json"
+
+    with open(config_path) as f:
+        return json.load(f), config_path

src/monitor_utils/constants.py

@@ -0,0 +1,64 @@
+import asyncio
+import pathlib
+from json import JSONDecodeError
+
+import aiohttp
+import requests
+from curl_cffi.requests.errors import RequestsError
+from pymongo.errors import PyMongoError
+
+PROJECT_ROOT = pathlib.Path(".")
+PROXIES_PATH = pathlib.Path("proxies.txt")
+MONGO_DATABASE_NAME = "tpv_woot"
+MONGO_URL = (
+    "mongodb://tpv:n3%23QI07GnbZCbvGJ@zelda.disqualifier.me:27017/?authSource=tpv_woot"
+)
+PRODUCTION_BOT_TOKEN = (
+    "MTQwMDE4NDk2MzY4NDYzMDUyOQ.GOrqOb.6frT3upUQ5B9XubVOIT9pdzVcYVXSrnEMK7RlA"
+)
+DEVELOPMENT_BOT_TOKEN = (
+    "MTUxNzYwNDI4NDEwMTk1MTUxOA.GO3b1i.aXdPICnvlY37m67OqESVcQsZhFcGOcCrn_1F-U"
+)
+PROFIT_API_KEY = "dev-XvIu8S83BJ2N1C9ZCsoLAAav5jan7kj5"
+LOGGING_LEVEL = "INFO"
+
+
+RETRY_LIMIT = 5
+RETRY_WAIT = 3
+BACKOFF_DELAY = 0.001
+FAILURE_DELAY = 0.001
+
+TIMEOUT_SECONDS = 30
+CLIENT_IDENTIFIERS = ["chrome", "chrome_android", "safari", "safari_ios"]
+CLIENT_TIMEOUT = aiohttp.ClientTimeout(
+    total=TIMEOUT_SECONDS, sock_connect=30, sock_read=30
+)
+
+FLAG_WEBHOOK_MAP = (
+    ("member_pick", "memberPick"),
+    ("member_deal", "memberDeal"),
+)
+
+BUCKET_WEBHOOK_MAP = {
+    40: "40-49",
+    50: "50-60",
+    60: "60-70",
+    70: "70-80",
+    80: "80-90",
+    90: "90-100",
+    "price_drop": "40-49",
+}
+STATUS_PENDING = "pending"
+STATUS_SENT = "sent"
+STATUS_SKIPPED = "skipped"
+STATUS_FAILED = "failed"
+
+EXPECTED_TASK_ERRORS = (
+    RequestsError,
+    aiohttp.ClientError,
+    asyncio.TimeoutError,
+    requests.Timeout,
+    requests.HTTPError,
+    PyMongoError,
+    JSONDecodeError,
+)

src/monitor_utils/database.py

@@ -0,0 +1,118 @@
+import logging
+from dataclasses import asdict
+from datetime import datetime, timezone
+from functools import cache
+
+from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorCollection
+from pymongo import UpdateOne
+from pymongo.errors import PyMongoError
+
+from monitor_utils.src.models import Product
+from monitor_utils.src.constants import STATUS_PENDING, MONGO_DATABASE_NAME, MONGO_URL
+
+LOGGER = logging.getLogger(__name__)
+
+
+def get_collection(collection_name: str) -> AsyncIOMotorCollection:
+    return get_mongo_client()[MONGO_DATABASE_NAME][collection_name]
+
+
+@cache
+def get_mongo_client() -> AsyncIOMotorClient:
+    return AsyncIOMotorClient(
+        MONGO_URL,
+        maxPoolSize=200,
+        minPoolSize=10,
+        retryWrites=True,
+        retryReads=True,
+        tz_aware=True,
+        tzinfo=timezone.utc,
+    )
+
+
+async def get_product_from_database(
+    collection: AsyncIOMotorCollection,
+    product_id: str,
+) -> dict | None:
+    return await collection.find_one({"_id": product_id})
+
+
+async def add_products_to_database(
+    products: list[Product],
+    collection: AsyncIOMotorCollection,
+    category_identifier: str,
+) -> None:
+    if not products:
+        return
+
+    composite_ids = [f"{p.product_id}_{category_identifier}" for p in products]
+    cursor = collection.find({"_id": {"$in": composite_ids}}, {"currentPrice": 1})
+    existing_prices = {str(doc["_id"]): doc.get("currentPrice") async for doc in cursor}
+
+    ops = []
+    for product in products:
+        document = convert_product_to_mongo(product, category_identifier)
+        doc_id = document["_id"]
+        old_price = existing_prices.get(doc_id)
+        new_price = document.get("currentPrice")
+
+        price_dropped = old_price is None or (
+            new_price is not None and new_price < old_price
+        )
+
+        if price_dropped:
+            document["status"] = STATUS_PENDING
+            document["priceBeforeDrop"] = old_price
+
+        set_on_insert = {"discoveredDate": datetime.now(timezone.utc)}
+        if not price_dropped:
+            set_on_insert["status"] = STATUS_PENDING
+
+        ops.append(
+            UpdateOne(
+                {"_id": doc_id},
+                {"$set": document, "$setOnInsert": set_on_insert},
+                upsert=True,
+            )
+        )
+
+    try:
+        await collection.bulk_write(ops, ordered=False)
+    except PyMongoError:
+        LOGGER.exception(f"{category_identifier} - failed to add products to database")
+        raise
+
+
+def convert_product_to_mongo(product: Product, category_identifier: str) -> dict:
+    data = asdict(product)
+    metadata = data["metadata"]
+    timing = data["timing"]
+    pricing = data["pricing"]
+    availability = data["availability"]
+    current_price = pricing["current_price"]
+    original_price = pricing["original_price"]
+    if isinstance(current_price, str):
+        current_price = float(current_price.replace(",", ""))
+        original_price = float(original_price.replace(",", ""))
+
+    return {
+        "_id": f"{data['product_id']}_{category_identifier}",
+        "title": data["title"],
+        "url": data["url"],
+        "currentPrice": current_price,
+        "originalPrice": original_price,
+        "percentage": metadata["percentage"],
+        "imageUrl": metadata["image_url"],
+        "customizationIndex": metadata["customization_index"],
+        "memberNonMemberComparison": metadata["member_non_member_comparison"],
+        "overstockText": metadata["overstock_text"],
+        "startDate": timing["start_date"],
+        "endDate": timing["end_date"],
+        "priceBeforeDrop": pricing["price_before_drop"],
+        "eduPrice": pricing["edu_price"],
+        "available": availability["available"],
+        "condition": availability["condition"],
+        "appExclusive": availability["app_exclusive"],
+        "memberPick": availability["member_pick"],
+        "memberDeal": availability["member_deal"],
+    }

src/monitor_utils/dsql_shortlinks.py

@@ -0,0 +1,97 @@
+import asyncio
+from typing import Dict
+
+import httpx
+
+"""
+Minimal usage:
+
+    from dsql_shortlinks import generate_short_link_sync
+
+    result = generate_short_link_sync(
+        "https://example.com/product",
+        api_key="YOUR_DSQL_KEY",
+    )
+
+    print(result["short_url"])
+"""
+
+
+DEFAULT_BASE_URL = "https://api.profitlounge.us/link/affiliate"
+
+
+class DsqlShortlinks:
+    def __init__(
+        self,
+        api_key: str,
+        *,
+        base_url: str = DEFAULT_BASE_URL,
+        timeout: int = 60,
+    ) -> None:
+        self.api_key = api_key
+        self.base_url = base_url
+        self.timeout = timeout
+
+    async def create_short_link(self, url: str) -> Dict[str, str]:
+        headers = {"x-api-key": self.api_key}
+        payload = {"url": url}
+
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.post(
+                    self.base_url,
+                    headers=headers,
+                    json=payload,
+                    timeout=self.timeout,
+                )
+                response.raise_for_status()
+                data = response.json()
+                if isinstance(data, dict) and data.get("short_url"):
+                    return data
+        except Exception:
+            pass
+
+        return {"short_url": url, "long_url": url}
+
+    def create_short_link_sync(self, url: str) -> Dict[str, str]:
+        try:
+            running_loop = asyncio.get_running_loop()
+        except RuntimeError:
+            running_loop = None
+
+        if running_loop and running_loop.is_running():
+            raise RuntimeError(
+                "create_short_link_sync() cannot run inside an active event loop. "
+                "Use await create_short_link(...) instead."
+            )
+
+        return asyncio.run(self.create_short_link(url))
+
+
+async def generate_short_link(
+    url: str, api_key: str, timeout: int = 60
+) -> Dict[str, str]:
+    client = DsqlShortlinks(api_key, timeout=timeout)
+    return await client.create_short_link(url)
+
+
+def generate_short_link_sync(
+    url: str, api_key: str, timeout: int = 60
+) -> Dict[str, str]:
+    client = DsqlShortlinks(api_key, timeout=timeout)
+    return client.create_short_link_sync(url)
+
+
+if __name__ == "__main__":
+    EXAMPLE_URL = "https://woot.com/"
+    EXAMPLE_API_KEY = "YOUR_DSQL_KEY"
+
+    print("Example:")
+    print("from dsql_shortlinks import generate_short_link_sync")
+    print()
+    print("result = generate_short_link_sync(")
+    print(f'    "{EXAMPLE_URL}",')
+    print(f'    api_key="{EXAMPLE_API_KEY}",')
+    print(")")
+    print()
+    print('print(result["short_url"])')

src/monitor_utils/models.py

@@ -0,0 +1,72 @@
+from datetime import datetime
+from dataclasses import field
+
+from pydantic.dataclasses import dataclass
+
+
+@dataclass
+class ConsumerConfig:
+    # used claude to refactor the consumer building in the discord bot
+    """Configuration for a single retailer's consumer.
+
+    Attributes:
+        monitor_name: Retailer name (matches setup_config key).
+        collection: MongoDB collection for this retailer's products.
+        channel_ids: Discord webhook/channel routing config.
+        min_percentage: Minimum discount % to trigger a ping.
+        ping_threshold: Price floor for the secondary 40% rule (e.g. Woot pings
+            on 40%+ discounts if the item is >= this price). None disables
+            the secondary rule.
+    """
+
+    monitor_name: str
+    collection: object
+    channel_ids: dict
+    min_percentage: int
+    ping_threshold: int | None = None
+
+
+@dataclass
+class PricingInfo:
+    original_price: float
+    current_price: float
+    price_before_drop: float | None = None
+    edu_price: float | None = None
+
+
+@dataclass
+class AvailabilityInfo:
+    available: bool | None = None
+    condition: str | None = None
+    app_exclusive: bool | None = None
+    member_pick: bool | None = None
+    member_deal: bool | None = None
+
+
+@dataclass
+class ProductMetadata:
+    image_url: str | None = None
+    channel_id: str | None = None
+    customization_index: str | None = None
+    member_non_member_comparison: str | None = None
+    overstock_text: str | None = None
+    percentage: int | None = None
+    affiliate_url: str | None = None
+
+
+@dataclass
+class ProductTiming:
+    discovered_date: datetime | None = None
+    start_date: datetime | None = None
+    end_date: datetime | None = None
+
+
+@dataclass
+class Product:
+    product_id: str
+    title: str
+    url: str
+    pricing: PricingInfo
+    metadata: ProductMetadata = field(default_factory=ProductMetadata)
+    timing: ProductTiming = field(default_factory=ProductTiming)
+    availability: AvailabilityInfo = field(default_factory=AvailabilityInfo)

src/monitor_utils/runner.py

@@ -0,0 +1,315 @@
+import asyncio
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Awaitable, Callable, Iterable
+
+import aiohttp
+from curl_cffi.requests import AsyncSession
+from motor.motor_asyncio import AsyncIOMotorCollection
+
+from monitor_utils.src.constants import (
+    EXPECTED_TASK_ERRORS,
+    FLAG_WEBHOOK_MAP,
+    BUCKET_WEBHOOK_MAP,
+)
+from monitor_utils.src.config import get_config, is_production, DevelopmentMode
+from monitor_utils.src.models import (
+    Product,
+    ProductMetadata,
+    ProductTiming,
+    PricingInfo,
+    AvailabilityInfo,
+)
+from monitor_utils.src.sessions import get_base_session, get_tls_session
+from monitor_utils.src.database import get_collection
+
+LOGGER = logging.getLogger(__name__)
+
+
+def setup_config(
+    monitor_name: str, development_mode: DevelopmentMode
+) -> tuple[list[str], dict, AsyncIOMotorCollection]:
+    use_production = is_production(development_mode)
+    config, _ = get_config(
+        monitor_name=monitor_name,
+        use_production_path=use_production,
+    )
+
+    category_names = config["categories"]
+    collection = get_collection(
+        monitor_name if use_production else f"{monitor_name}_testing"
+    )
+    return category_names, config["channels"], collection
+
+
+def get_channel_id(
+    product: Product, channel_ids: dict, use_price_drop_bucket: bool = False
+) -> str:
+    # KeyError is not a issue here bc the percentage filtering per monitor handles it
+    if product.availability.app_exclusive:
+        return get_single_channel_id(channel_ids["appExclusive"])
+
+    if product.availability.available is False:
+        return get_single_channel_id(channel_ids["soldOut"])
+
+    if product.timing.end_date:
+        now = datetime.now(timezone.utc)
+        if (product.timing.end_date - now) <= timedelta(days=1):
+            return get_single_channel_id(channel_ids["event"])
+
+    for attr, key in FLAG_WEBHOOK_MAP:
+        if getattr(product.availability, attr, False):
+            return get_single_channel_id(channel_ids[key])
+
+    percentage = product.metadata.percentage or 0
+    percentage_bucket = min(90, (percentage // 10) * 10)
+    if use_price_drop_bucket:
+        return get_single_channel_id(channel_ids[BUCKET_WEBHOOK_MAP["price_drop"]])
+
+    return get_single_channel_id(channel_ids[BUCKET_WEBHOOK_MAP[percentage_bucket]])
+
+
+def get_single_channel_id(value: str | list[str]) -> str:
+    if isinstance(value, list):
+        return value[0]
+
+    return value
+
+
+def build_product_details(
+    product: dict,
+    channel_ids: dict,
+    min_percentage: int,
+    min_price_drop: float | None = None,
+) -> Product | None:
+    product_id = product["_id"]
+    current_price = product["currentPrice"]
+    original_price = product["originalPrice"]
+    if not original_price:
+        LOGGER.warning(f"{product_id} - Missing or zero original price, skipping")
+        return
+
+    percentage = int(((original_price - current_price) / original_price) * 100)
+    price_drop = original_price - current_price
+    passes_percentage = percentage >= min_percentage
+    passes_price_drop = min_price_drop is not None and price_drop >= min_price_drop
+    if not passes_percentage and not passes_price_drop:
+        return
+
+    built = Product(
+        product_id=product_id,
+        title=product["title"],
+        url=product["url"],
+        metadata=ProductMetadata(
+            image_url=product["imageUrl"],
+            customization_index=product["customizationIndex"],
+            member_non_member_comparison=product["memberNonMemberComparison"],
+            overstock_text=product["overstockText"],
+            percentage=percentage,
+            affiliate_url=product.get("affiliateUrl"),
+        ),
+        timing=ProductTiming(
+            discovered_date=product["discoveredDate"],
+            start_date=product["startDate"],
+            end_date=product["endDate"],
+        ),
+        pricing=PricingInfo(
+            current_price=current_price,
+            original_price=original_price,
+            price_before_drop=product["priceBeforeDrop"],
+            edu_price=product["eduPrice"],
+        ),
+        availability=AvailabilityInfo(
+            available=product["available"],
+            condition=product["condition"],
+            app_exclusive=product["appExclusive"],
+            member_pick=product["memberPick"],
+            member_deal=product["memberDeal"],
+        ),
+    )
+    built.metadata.channel_id = get_channel_id(
+        built, channel_ids, passes_price_drop and not passes_percentage
+    )
+    return built
+
+
+async def execute_monitor_logic(
+    category_identifiers: list[str] | dict[str, str],
+    task_func: Callable[..., Awaitable[None]],
+    headers: dict | None = None,
+    cookies: dict | None = None,
+    pre_task_func: Callable[..., Awaitable[None]] | None = None,
+    use_tls: bool = False,
+    **kwargs,
+):
+    while True:
+        session_ctx = (
+            get_tls_session(headers=headers)
+            if use_tls
+            else get_base_session(headers=headers, cookies=cookies)
+        )
+        async with session_ctx as session:
+            LOGGER.info("Starting tasks...")
+            if pre_task_func:
+                try:
+                    await pre_task_func(session=session, **kwargs)
+                except EXPECTED_TASK_ERRORS as error:
+                    LOGGER.warning("Failed to get products", exc_info=error)
+                except Exception:
+                    LOGGER.exception("Unexpected error")
+                    raise
+
+            if isinstance(category_identifiers, dict):
+                tasks = [
+                    task_func(category_id, session, query_category_id=query, **kwargs)
+                    for category_id, query in category_identifiers.items()
+                ]
+                category_keys = list(category_identifiers.keys())
+            else:
+                tasks = [
+                    task_func(category_identifier, session, **kwargs)
+                    for category_identifier in category_identifiers
+                ]
+                category_keys = category_identifiers
+
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+            for category_identifier, result in zip(category_keys, results):
+                if isinstance(result, EXPECTED_TASK_ERRORS):
+                    LOGGER.warning(
+                        f"{category_identifier} - Failed to get products",
+                        exc_info=result,
+                    )
+                elif isinstance(result, Exception):
+                    LOGGER.error(
+                        f"{category_identifier} - Unexpected error",
+                        exc_info=result,
+                    )
+                    raise result
+
+        LOGGER.info(
+            "Sleeping 0.30 seconds before next tasks"
+        )  # Might add option to increase/decrease this delay
+        await asyncio.sleep(0.30)
+
+
+async def execute_pagination(
+    category_identifier: str,
+    total_products: int,
+    products_per_page: int,
+    func: Callable[..., Awaitable[None]],
+    session: aiohttp.ClientSession | AsyncSession,
+    max_concurrent_tasks: int = 5,
+    start_offset: int = 24,
+    pagination_mode: str = "offset",
+    **kwargs,
+):
+    LOGGER.info(f"{category_identifier} - executing pagination")
+    total_pages = (int(total_products) + products_per_page - 1) // products_per_page
+    semaphore = asyncio.Semaphore(max_concurrent_tasks)
+    stop_event = asyncio.Event()
+    pagination_label = "offset" if pagination_mode == "offset" else "page"
+    page_range = range(2, total_pages + 1)
+
+    async def fetch_page(page_index: int):
+        async with semaphore:
+            if stop_event.is_set():
+                return
+
+            if pagination_mode == "offset":
+                current_offset = start_offset + page_index * products_per_page
+                await func(
+                    session=session,
+                    offset=current_offset,
+                    **kwargs,
+                )
+            else:
+                await func(
+                    session=session,
+                    page_number=page_index,
+                    **kwargs,
+                )
+
+    failed_pages = await fetch_all_pages(
+        page_range, fetch_page, category_identifier, pagination_label
+    )
+    if failed_pages and not stop_event.is_set():
+        failed_pages = await retry_failed_pages(
+            failed_pages, fetch_page, category_identifier, pagination_label
+        )
+
+    if failed_pages and not stop_event.is_set():
+        LOGGER.error(
+            f"{category_identifier} - failed to get products after retries: "
+            f"{pagination_label}s {failed_pages}"
+        )
+
+    if stop_event.is_set():
+        LOGGER.info(f"{category_identifier} - pagination stopped early")
+
+
+async def fetch_all_pages(
+    page_range: Iterable[int],
+    fetch_func: Callable[[int], Awaitable[None]],
+    category_identifier: str,
+    pagination_label: str,
+) -> list[int]:
+    tasks = [fetch_func(page) for page in page_range]
+    results = await asyncio.gather(*tasks, return_exceptions=True)
+    return handle_page_results(
+        list(page_range), results, category_identifier, pagination_label
+    )
+
+
+def handle_page_results(
+    page_numbers: list[int],
+    results: list,
+    category_identifier: str,
+    pagination_label: str,
+    retry_attempt: int | None = None,
+) -> list[int]:
+    failed_pages = []
+    retry_suffix = f" Retry {retry_attempt} " if retry_attempt else " "
+
+    for page_number, result in zip(page_numbers, results):
+        if isinstance(result, EXPECTED_TASK_ERRORS):
+            LOGGER.warning(
+                f"{category_identifier} - {pagination_label} {page_number} -{retry_suffix} failed",
+                exc_info=result,
+            )
+            failed_pages.append(page_number)
+        elif isinstance(result, Exception):
+            LOGGER.error(
+                f"{category_identifier} - {pagination_label} {page_number} -{retry_suffix} unexpected error",
+                exc_info=result,
+            )
+            raise result
+        elif retry_attempt:
+            LOGGER.info(
+                f"{category_identifier} - {pagination_label} {page_number} - retry {retry_attempt} succeeded"
+            )
+
+    return failed_pages
+
+
+async def retry_failed_pages(
+    failed_pages: list[int],
+    fetch_func: Callable[[int], Awaitable[None]],
+    category_identifier: str,
+    pagination_label: str,
+    max_retries: int = 3,
+) -> list[int]:
+    retry_attempt = 1
+    while failed_pages and retry_attempt <= max_retries:
+        await asyncio.sleep(2**retry_attempt)
+        LOGGER.info(
+            f"{category_identifier} - retrying {len(failed_pages)} failed {pagination_label}s "
+            f"(attempt {retry_attempt}/{max_retries})"
+        )
+        tasks = [fetch_func(page_number) for page_number in failed_pages]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        failed_pages = handle_page_results(
+            failed_pages, results, category_identifier, pagination_label, retry_attempt
+        )
+        retry_attempt += 1
+
+    return failed_pages

src/monitor_utils/sessions.py

@@ -0,0 +1,148 @@
+# ik ua/tls doesn't match headers/header order but targeted antibots don't seem to care might fix later
+import asyncio
+import random
+import json
+import logging
+from contextlib import asynccontextmanager
+from typing import AsyncGenerator
+
+import aiohttp
+import requests
+from curl_cffi.requests import AsyncSession
+from curl_cffi.requests.errors import RequestsError
+from curl_cffi.requests.exceptions import Timeout as CurlTimeout
+from fake_useragent import FakeUserAgent
+from tenacity import (
+    after_log,
+    before_log,
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_fixed,
+)
+
+from monitor_utils.src.config import get_proxies
+from monitor_utils.src.constants import (
+    CLIENT_TIMEOUT,
+    TIMEOUT_SECONDS,
+    CLIENT_IDENTIFIERS,
+    RETRY_LIMIT,
+    RETRY_WAIT,
+)
+
+LOGGER = logging.getLogger(__name__)
+UA = FakeUserAgent()
+
+
+@asynccontextmanager
+async def get_base_session(
+    headers: dict | None = None,
+    cookies: dict | None = None,
+    connector: aiohttp.TCPConnector | None = None,
+    timeout: aiohttp.ClientTimeout = CLIENT_TIMEOUT,
+) -> AsyncGenerator[aiohttp.ClientSession, None]:
+    headers = headers if headers else {}
+    headers.setdefault("User-Agent", UA.random)
+    session = aiohttp.ClientSession(
+        headers=headers, cookies=cookies, connector=connector, timeout=timeout
+    )
+    try:
+        yield session
+    finally:
+        await session.close()
+
+
+@asynccontextmanager
+async def get_tls_session(
+    headers: dict | None = None, timeout: int = TIMEOUT_SECONDS
+) -> AsyncGenerator[AsyncSession, None]:
+    headers = headers if headers else {}
+    headers.setdefault("User-Agent", UA.chrome)
+    async with AsyncSession(
+        impersonate=random.choice(CLIENT_IDENTIFIERS),
+        timeout=timeout,
+        headers=headers,
+    ) as session:
+        yield session
+
+
+def build_session_details(
+    session: aiohttp.ClientSession | AsyncSession, headers: dict = None
+):
+    # ik headers doesn't match tls i'm lazy and the antibots used don't care
+    headers = headers or {}
+    headers.setdefault("User-Agent", FakeUserAgent().random)
+    session.headers.update(headers)
+
+
+@retry(
+    stop=stop_after_attempt(RETRY_LIMIT),
+    wait=wait_fixed(RETRY_WAIT),
+    retry=retry_if_exception_type(
+        (
+            aiohttp.ClientError,
+            asyncio.TimeoutError,
+            RequestsError,
+            requests.Timeout,
+            CurlTimeout,
+        )
+    ),
+    before=before_log(LOGGER, logging.DEBUG),
+    after=after_log(LOGGER, logging.DEBUG),
+    reraise=True,
+)
+async def send_request(
+    session: aiohttp.ClientSession | AsyncSession,
+    url: str,
+    method: str,
+    params: dict | None = None,
+    json_data: dict | None = None,
+    data: dict | None = None,
+    reraise_json_error: bool = False,
+    development_proxy: str | None = None,
+) -> dict | str:
+    verify = True
+    if development_proxy:
+        proxy = development_proxy
+        verify = False
+    else:
+        proxy = random.choice(get_proxies())
+
+    if isinstance(session, aiohttp.ClientSession):
+        async with session.request(
+            method=method,
+            url=url,
+            params=params,
+            json=json_data,
+            data=data,
+            proxy=proxy,
+            ssl=verify,
+        ) as response:
+            response.raise_for_status()
+            text = await response.text()
+    else:
+        if (
+            url
+            == "https://www.hp.com/wcs/resources/store/10151/component/configure/addtocartconfig"
+        ):
+            pass
+
+        response = await session.request(
+            method=method.upper(),
+            url=url,
+            params=params,
+            json=json_data,
+            data=data,
+            proxy=proxy,
+            verify=verify,
+        )
+        response.raise_for_status()
+        text = response.text
+
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        if reraise_json_error:
+            raise
+
+        return text