import asyncio import logging from typing import Awaitable, Callable, Iterable import aiohttp from aioproxies import AioProxies from curl_cffi.requests import AsyncSession from log_setup import setup_logging from motor.motor_asyncio import AsyncIOMotorCollection from monitor_utils.config import ( DevelopmentMode, get_config, is_production, set_logger_levels, ) from monitor_utils.constants import EXPECTED_TASK_ERRORS, LOGGING_LEVEL from monitor_utils.database import get_collection from monitor_utils.models import ( AvailabilityInfo, PricingInfo, Product, ProductMetadata, ProductTiming, ) from monitor_utils.sessions import get_base_session, get_tls_session LOGGER = logging.getLogger(__name__) def setup_monitor_logging(monitor_name: str, logging_level=LOGGING_LEVEL): """Configure JSON logging for a monitor and quiet noisy libraries. Call once at module import, at global scope, before creating module loggers, so logging is configured synchronously at startup rather than inside the async init_monitor. Logs go to logs/latest.log with rotated history keyed by monitor_name. Args: monitor_name: Monitor name used as the rotated-history base. logging_level: Root logging level. """ setup_logging( name="logs/latest.log", history_name=monitor_name, level=logging_level, output="json", ) set_logger_levels() async def init_monitor( monitor_name: str, development_mode: DevelopmentMode, production_proxy_path: str = "proxies.txt", testing_proxy_path: str = "configs/proxies.txt", ) -> tuple[list[str] | dict[str, str], AsyncIOMotorCollection, AioProxies]: """Load a monitor's config, collection, and proxies for the mode. Async because config now lives in Mongo; call it from inside the monitor's event loop (in main), not at module import. Logging is configured separately by setup_monitor_logging at module scope and is not touched here. Args: monitor_name: Monitor identifier used for config and collection lookup. development_mode: Selects production or testing config, collection, and proxy path. production_proxy_path: Proxy file path used in production mode. testing_proxy_path: Proxy file path used in testing mode. Returns: Tuple of category names, Mongo collection, and loaded proxies. """ use_production = is_production(development_mode) category_names, _, collection = await setup_config( monitor_name=monitor_name, development_mode=development_mode ) proxy_path = production_proxy_path if use_production else testing_proxy_path proxies = AioProxies.from_file(path=proxy_path) return category_names, collection, proxies async def setup_config( monitor_name: str, development_mode: DevelopmentMode ) -> tuple[ list[str] | dict[str, str], dict[str, str | list[str]], AsyncIOMotorCollection ]: """Load monitor config and resolve the Mongo collection for the mode. Testing mode reads the configs_testing collection and appends a _testing suffix to the product collection name so production data is never touched during development. Args: monitor_name: Monitor identifier used for config and collection lookup. development_mode: Selects the production or testing config and collection. Returns: Tuple of category names, channel id mapping, and Mongo collection. """ use_production = is_production(development_mode) config = await get_config( monitor_name=monitor_name, use_production=use_production, ) category_names = config["categories"] collection = get_collection( monitor_name if use_production else f"{monitor_name}_testing" ) return category_names, config["channels"], collection def build_product_details( product: dict, min_percentage: int, min_price_drop: float | None = None, ) -> Product | None: """Build a Product from a Mongo document if it passes discount thresholds. Skips documents with a missing or zero original price. Passing requires meeting min_percentage, or min_price_drop when provided. Channel routing is owned by the consumer bot; no channel id is attached here. Args: product: Mongo document for the product. min_percentage: Minimum discount percentage to pass. min_price_drop: Optional minimum absolute price drop to pass. Returns: The built Product, or None when skipped or below thresholds. """ product_id = product["_id"] current_price = product["currentPrice"] original_price = product["originalPrice"] if not original_price: LOGGER.warning( "missing or zero original price, skipping", extra={"product_id": product_id}, ) return percentage = int(((original_price - current_price) / original_price) * 100) price_drop = original_price - current_price passes_percentage = percentage >= min_percentage passes_price_drop = min_price_drop is not None and price_drop >= min_price_drop if not passes_percentage and not passes_price_drop: return built = Product( product_id=product_id, title=product["title"], url=product["url"], metadata=ProductMetadata( image_url=product["imageUrl"], customization_index=product["customizationIndex"], member_non_member_comparison=product["memberNonMemberComparison"], overstock_text=product["overstockText"], percentage=percentage, affiliate_url=product.get("affiliateUrl"), ), timing=ProductTiming( discovered_date=product["discoveredDate"], start_date=product["startDate"], end_date=product["endDate"], ), pricing=PricingInfo( current_price=current_price, original_price=original_price, price_before_drop=product["priceBeforeDrop"], edu_price=product["eduPrice"], ), availability=AvailabilityInfo( available=product["available"], condition=product["condition"], app_exclusive=product["appExclusive"], member_pick=product["memberPick"], member_deal=product["memberDeal"], ), ) return built async def execute_monitor_logic( category_identifiers: list[str] | dict[str, str], task_func: Callable[..., Awaitable[None]], *, proxies: AioProxies, headers: dict | None = None, cookies: dict | None = None, concurrent_pre_task: Callable[..., Awaitable[None]] | None = None, use_tls: bool = False, **kwargs, ): """Run the monitor's category tasks in a loop with a fresh session per cycle. Each cycle opens a TLS or base session, dispatches task_func per category concurrently, logs expected failures, raises unexpected ones, then sleeps briefly before the next cycle. Args: category_identifiers: List of category ids, or a mapping of category id to query id; mapping entries pass query_category_id to task_func. task_func: Awaitable invoked per category with the identifier, session, and **kwargs. proxies: Proxies applied to the session. headers: Headers applied to the session. cookies: Cookies applied to the base session; ignored for TLS. concurrent_pre_task: Optional awaitable run once per cycle, gathered alongside the category tasks and receiving session and **kwargs. Category tasks must not depend on its side effects (session headers, cookies, fetched state) since no ordering is guaranteed. use_tls: Use the TLS fingerprint session instead of the base session. **kwargs: Forwarded verbatim to concurrent_pre_task and every task_func call. """ while True: session_ctx = ( get_tls_session( headers=headers, proxies=proxies, ) if use_tls else get_base_session(headers=headers, cookies=cookies, proxies=proxies) ) async with session_ctx as session: LOGGER.info("starting tasks") if isinstance(category_identifiers, dict): tasks = [ task_func(category_id, session, query_category_id=query, **kwargs) for category_id, query in category_identifiers.items() ] category_keys = list(category_identifiers.keys()) else: tasks = [ task_func(category_identifier, session, **kwargs) for category_identifier in category_identifiers ] category_keys = list(category_identifiers) if concurrent_pre_task: tasks.append(concurrent_pre_task(session=session, **kwargs)) category_keys.append("pre_task") results = await asyncio.gather(*tasks, return_exceptions=True) for category_identifier, result in zip(category_keys, results): if isinstance(result, EXPECTED_TASK_ERRORS): LOGGER.warning( "failed to get products", extra={"category": category_identifier}, exc_info=result, ) elif isinstance(result, Exception): LOGGER.error( "unexpected error", extra={"category": category_identifier}, exc_info=result, ) raise result LOGGER.info("sleeping before next tasks", extra={"delay": 0.30}) await asyncio.sleep(0.30) async def execute_pagination( category_identifier: str, total_products: int, products_per_page: int, func: Callable[..., Awaitable[None]], session: aiohttp.ClientSession | AsyncSession, *, max_concurrent_tasks: int = 5, start_offset: int = 0, pagination_mode: str = "offset", **kwargs, ): """Paginate a category by dispatching concurrent page or offset tasks to func. Runs pages 2..N under a semaphore, retries failed pages with exponential backoff, and logs any pages that still fail after retries. Page 1 is assumed to have been fetched by the caller. Args: category_identifier: Label used for logging only; not passed to func. total_products: Total product count used to derive page count. products_per_page: Page size used to derive page count and offsets. func: Awaitable task invoked per page; receives session plus either offset or page_number depending on pagination_mode. session: HTTP session forwarded to each func call. max_concurrent_tasks: Semaphore limit for concurrent page fetches. start_offset: Additive base applied to every offset, for APIs whose offset space does not begin at zero. Page N requests offset start_offset + (N - 1) * products_per_page. pagination_mode: Either offset or page, selecting the kwarg sent to func. **kwargs: Forwarded verbatim to every func call. Any identifier func needs (category_id, category_name, category_path, or bestbuy's query_category_id) must be supplied here, since naming varies per monitor. """ total_pages = (int(total_products) + products_per_page - 1) // products_per_page LOGGER.info( "executing pagination", extra={"category": category_identifier, "total": total_pages}, ) semaphore = asyncio.Semaphore(max_concurrent_tasks) stop_event = asyncio.Event() pagination_label = "offset" if pagination_mode == "offset" else "page" page_range = range(2, total_pages + 1) async def fetch_page(page_index: int): """Fetch a single page or offset under the shared semaphore.""" async with semaphore: if stop_event.is_set(): return if pagination_mode == "offset": current_offset = start_offset + (page_index - 1) * products_per_page await func( session=session, offset=current_offset, **kwargs, ) else: await func( session=session, page_number=page_index, **kwargs, ) failed_pages = await fetch_all_pages( page_range, fetch_page, category_identifier, pagination_label ) if failed_pages and not stop_event.is_set(): failed_pages = await retry_failed_pages( failed_pages, fetch_page, category_identifier, pagination_label ) if failed_pages and not stop_event.is_set(): LOGGER.error( "failed to get products after retries", extra={ "category": category_identifier, "label": pagination_label, "failed": failed_pages, }, ) if stop_event.is_set(): LOGGER.info("pagination stopped early", extra={"category": category_identifier}) async def fetch_all_pages( page_range: Iterable[int], fetch_func: Callable[[int], Awaitable[None]], category_identifier: str, pagination_label: str, ) -> list[int]: """Dispatch all page fetches concurrently and return failed page numbers. Args: page_range: Page numbers to fetch. fetch_func: Awaitable invoked per page number. category_identifier: Label used for logging. pagination_label: Either offset or page, used for logging. Returns: Page numbers whose fetch raised an expected error. """ tasks = [fetch_func(page) for page in page_range] results = await asyncio.gather(*tasks, return_exceptions=True) return handle_page_results( list(page_range), results, category_identifier, pagination_label ) def handle_page_results( page_numbers: list[int], results: list, category_identifier: str, pagination_label: str, retry_attempt: int | None = None, ) -> list[int]: """Log page results and collect failures, raising on unexpected errors. Expected errors are logged as warnings and their pages collected for retry. Unexpected exceptions are logged and re-raised. Successful pages are logged only when part of a retry attempt. Args: page_numbers: Page numbers matching results by position. results: Gathered results or exceptions per page. category_identifier: Label used for logging. pagination_label: Either offset or page, used for logging. retry_attempt: Retry attempt number when handling retry results. Returns: Page numbers whose fetch raised an expected error. """ failed_pages = [] for page_number, result in zip(page_numbers, results): if isinstance(result, EXPECTED_TASK_ERRORS): LOGGER.warning( "failed", extra={ "category": category_identifier, "label": pagination_label, "page": page_number, "retry": retry_attempt, }, exc_info=result, ) failed_pages.append(page_number) elif isinstance(result, Exception): LOGGER.error( "unexpected error", extra={ "category": category_identifier, "label": pagination_label, "page": page_number, "retry": retry_attempt, }, exc_info=result, ) raise result elif retry_attempt: LOGGER.info( "retry succeeded", extra={ "category": category_identifier, "label": pagination_label, "page": page_number, "retry": retry_attempt, }, ) return failed_pages async def retry_failed_pages( failed_pages: list[int], fetch_func: Callable[[int], Awaitable[None]], category_identifier: str, pagination_label: str, max_retries: int = 3, ) -> list[int]: """Retry failed pages with exponential backoff until exhausted or clear. Sleeps 2**attempt seconds before each round and re-dispatches only the pages that failed the previous round. Args: failed_pages: Page numbers to retry. fetch_func: Awaitable invoked per page number. category_identifier: Label used for logging. pagination_label: Either offset or page, used for logging. max_retries: Maximum retry rounds. Returns: Page numbers still failing after all retries. """ retry_attempt = 1 while failed_pages and retry_attempt <= max_retries: await asyncio.sleep(2**retry_attempt) LOGGER.info( "retrying failed pages", extra={ "category": category_identifier, "label": pagination_label, "count": len(failed_pages), "attempt": retry_attempt, "max_retries": max_retries, }, ) tasks = [fetch_func(page_number) for page_number in failed_pages] results = await asyncio.gather(*tasks, return_exceptions=True) failed_pages = handle_page_results( failed_pages, results, category_identifier, pagination_label, retry_attempt ) retry_attempt += 1 return failed_pages