migrate session handling to aioweb_tls/refactoring of helpers

src/monitor_utils/runner.py

@@ -1,6 +1,6 @@
 import asyncio
 import logging
-from typing import Awaitable, Callable, Iterable
+from collections.abc import Awaitable, Callable, Iterable
 
 import aiohttp
 from aioproxies import AioProxies
@@ -18,12 +18,12 @@ from monitor_utils.constants import EXPECTED_TASK_ERRORS, LOGGING_LEVEL
 from monitor_utils.database import get_collection
 from monitor_utils.models import (
     AvailabilityInfo,
+    CycleResult,
     PricingInfo,
     Product,
     ProductMetadata,
     ProductTiming,
 )
-from monitor_utils.sessions import get_base_session, get_tls_session
 
 LOGGER = logging.getLogger(__name__)
 
@@ -186,23 +186,23 @@ def build_product_details(
     return built
 
 
-async def execute_monitor_logic(
+async def run_monitor_cycle(
     category_identifiers: list[str] | dict[str, str],
     task_func: Callable[..., Awaitable[None]],
+    session: aiohttp.ClientSession | AsyncSession,
     *,
-    proxies: AioProxies,
-    headers: dict | None = None,
-    cookies: dict | None = None,
     pre_task_func: Callable[..., Awaitable[None]] | None = None,
     concurrent_pre_task_func: Callable[..., Awaitable[None]] | None = None,
-    use_tls: bool = False,
     **kwargs,
-):
-    """Run the monitor's category tasks in a loop with a fresh session per cycle.
+) -> CycleResult:
+    """Run one monitor cycle over a caller-supplied session and report the outcome.
 
-    Each cycle opens a TLS or base session, awaits pre_task_func if given,
-    dispatches task_func per category concurrently, logs expected failures,
-    raises unexpected ones, then sleeps briefly before the next cycle.
+    Awaits pre_task_func if given, dispatches task_func per category
+    concurrently, logs expected failures, and raises unexpected ones. The
+    caller owns the loop, the session lifetime, and the inter-cycle delay: wrap
+    this in a `while True` that opens a fresh session per cycle and sleeps
+    between cycles, using the returned CycleResult to drive backoff (reset the
+    delay when healthy, grow it when not).
 
     Args:
         category_identifiers: List of category ids, or a mapping of category
@@ -210,80 +210,78 @@ async def execute_monitor_logic(
             task_func.
         task_func: Awaitable invoked per category with the identifier,
             session, and **kwargs.
-        proxies: Proxies applied to the session.
-        headers: Headers applied to the session.
-        cookies: Cookies applied to the base session; ignored for TLS.
-        pre_task_func: Optional awaitable run once per cycle, awaited before any
-            category tasks are dispatched and receiving session and **kwargs.
-            Use it for setup the category tasks depend on (session headers,
-            cookies, fetched state); if it raises an expected task error the
-            cycle is skipped and retried after the usual delay.
+        session: Open HTTP session for this cycle, built and closed by the
+            caller. Proxies are the caller's responsibility, either bound to
+            the session or forwarded to task_func via **kwargs.
+        pre_task_func: Optional awaitable run once at the start of the cycle,
+            awaited before any category tasks are dispatched and receiving
+            session and **kwargs. Use it for setup the category tasks depend on
+            (session headers, cookies, fetched state); if it raises an expected
+            task error the cycle is skipped and a skipped CycleResult returned.
         concurrent_pre_task_func: Optional awaitable run once per cycle, gathered
             alongside the category tasks and receiving session and **kwargs.
             Category tasks must not depend on its side effects (session
             headers, cookies, fetched state) since no ordering is guaranteed.
-        use_tls: Use the TLS fingerprint session instead of the base session.
         **kwargs: Forwarded verbatim to pre_task_func, concurrent_pre_task_func,
             and every task_func call.
+
+    Returns:
+        A CycleResult tallying task successes and expected-error failures, or a
+        skipped result when pre_task_func failed.
+
+    Raises:
+        Exception: Any unexpected (non-EXPECTED_TASK_ERRORS) error raised by a
+            task or pre_task_func propagates, failing loud so the caller's loop
+            crashes rather than silently retrying a broken monitor.
     """
 
-    while True:
-        session_ctx = (
-            get_tls_session(
-                headers=headers,
-                proxies=proxies,
+    if pre_task_func:
+        try:
+            await pre_task_func(session=session, **kwargs)
+        except EXPECTED_TASK_ERRORS as error:
+            LOGGER.warning("pre task failed, skipping cycle", exc_info=error)
+            return CycleResult(skipped=True)
+
+    LOGGER.info("starting tasks")
+    if isinstance(category_identifiers, dict):
+        tasks = [
+            task_func(category_id, session, query_category_id=query, **kwargs)
+            for category_id, query in category_identifiers.items()
+        ]
+        category_keys = list(category_identifiers.keys())
+    else:
+        tasks = [
+            task_func(category_identifier, session, **kwargs)
+            for category_identifier in category_identifiers
+        ]
+        category_keys = list(category_identifiers)
+
+    if concurrent_pre_task_func:
+        tasks.append(concurrent_pre_task_func(session=session, **kwargs))
+        category_keys.append("pre_task")
+
+    results = await asyncio.gather(*tasks, return_exceptions=True)
+    successes = 0
+    failures = 0
+    for category_identifier, result in zip(category_keys, results):
+        if isinstance(result, EXPECTED_TASK_ERRORS):
+            LOGGER.warning(
+                "failed to get products",
+                extra={"category": category_identifier},
+                exc_info=result,
             )
-            if use_tls
-            else get_base_session(headers=headers, cookies=cookies, proxies=proxies)
-        )
-        async with session_ctx as session:
-            skip_cycle = False
-            if pre_task_func:
-                try:
-                    await pre_task_func(session=session, **kwargs)
-                except EXPECTED_TASK_ERRORS as error:
-                    LOGGER.warning("pre task failed, skipping cycle", exc_info=error)
-                    skip_cycle = True
+            failures += 1
+        elif isinstance(result, Exception):
+            LOGGER.error(
+                "unexpected error",
+                extra={"category": category_identifier},
+                exc_info=result,
+            )
+            raise result
+        else:
+            successes += 1
 
-            if not skip_cycle:
-                LOGGER.info("starting tasks")
-                if isinstance(category_identifiers, dict):
-                    tasks = [
-                        task_func(
-                            category_id, session, query_category_id=query, **kwargs
-                        )
-                        for category_id, query in category_identifiers.items()
-                    ]
-                    category_keys = list(category_identifiers.keys())
-                else:
-                    tasks = [
-                        task_func(category_identifier, session, **kwargs)
-                        for category_identifier in category_identifiers
-                    ]
-                    category_keys = list(category_identifiers)
-
-                if concurrent_pre_task_func:
-                    tasks.append(concurrent_pre_task_func(session=session, **kwargs))
-                    category_keys.append("pre_task")
-
-                results = await asyncio.gather(*tasks, return_exceptions=True)
-                for category_identifier, result in zip(category_keys, results):
-                    if isinstance(result, EXPECTED_TASK_ERRORS):
-                        LOGGER.warning(
-                            "failed to get products",
-                            extra={"category": category_identifier},
-                            exc_info=result,
-                        )
-                    elif isinstance(result, Exception):
-                        LOGGER.error(
-                            "unexpected error",
-                            extra={"category": category_identifier},
-                            exc_info=result,
-                        )
-                        raise result
-
-        LOGGER.info("sleeping before next tasks", extra={"delay": 0.30})
-        await asyncio.sleep(0.30)
+    return CycleResult(successes=successes, failures=failures)
 
 
 async def execute_pagination(
@@ -311,11 +309,10 @@ async def execute_pagination(
         func: Awaitable task invoked per page; receives session plus either
             offset or page_number depending on pagination_mode.
         session: HTTP session forwarded to each func call.
-        max_concurrent_tasks: Semaphore limit for concurrent page fetches, used
-            only when semaphore is not supplied. Each execute_pagination call
-            builds its own semaphore, so this bounds one category in isolation;
-            with many categories running concurrently the effective limit is
-            this value times the category count.
+        semaphore: Required bound on concurrent page fetches. Pass one shared
+            semaphore across all categories so the limit caps total fan-out; a
+            per-call semaphore would instead bound each category in isolation,
+            making the effective limit this value times the category count.
         start_offset: Additive base applied to every offset, for APIs whose
             offset space does not begin at zero. Page N requests offset
             start_offset + (N - 1) * products_per_page.