abstracted out bounded cycles

This commit is contained in:
TPV committed 2026-07-30 14:13:26 -07:00
1 parent aa6ed4cb06
commit 8903be2685
3 files changed
+229 -3

No files matched your search

+75 -3
View File
@@ -15,8 +15,15 @@ from monitor_utils.config import (
is_production, is_production,
set_logger_levels, set_logger_levels,
) )
from monitor_utils.constants import EXPECTED_TASK_ERRORS, LOGGING_LEVEL from monitor_utils.constants import (
CYCLE_TIMEOUT_SECONDS,
EXPECTED_TASK_ERRORS,
LOGGING_LEVEL,
MAX_CURL_HANDLES,
TIMEOUT_SECONDS,
)
from monitor_utils.database import get_collection from monitor_utils.database import get_collection
from monitor_utils.sessions import get_default_session
from monitor_utils.models import ( from monitor_utils.models import (
AvailabilityInfo, AvailabilityInfo,
CycleResult, CycleResult,
@@ -233,8 +240,10 @@ async def run_monitor_cycle(
concurrently, logs expected failures, and raises unexpected ones. The concurrently, logs expected failures, and raises unexpected ones. The
caller owns the loop, the session lifetime, and the inter-cycle delay: wrap caller owns the loop, the session lifetime, and the inter-cycle delay: wrap
this in a `while True` that opens a fresh session per cycle and sleeps this in a `while True` that opens a fresh session per cycle and sleeps
between cycles. The returned CycleResult is pure counts for the caller to between cycles — or use run_bounded_cycle, which owns the session and a
log or act on; this function applies no policy to it. hard cycle budget and leaves only the loop and the sleep to the caller.
The returned CycleResult is pure counts for the caller to log or act on;
this function applies no policy to it.
session and collection are the two per-cycle resources, passed to every session and collection are the two per-cycle resources, passed to every
callback (pre_task_func, concurrent_pre_task_func, and each task_func) just callback (pre_task_func, concurrent_pre_task_func, and each task_func) just
@@ -329,6 +338,69 @@ async def run_monitor_cycle(
return CycleResult(successes=successes, failures=failures) return CycleResult(successes=successes, failures=failures)
async def run_bounded_cycle(
category_identifiers: list[str] | dict[str, str],
task_func: Callable[..., Awaitable[None]],
proxies: AioProxies,
collection: AsyncIOMotorCollection,
*,
session_timeout: int = TIMEOUT_SECONDS,
max_clients: int = MAX_CURL_HANDLES,
cycle_timeout: int = CYCLE_TIMEOUT_SECONDS,
**kwargs,
) -> CycleResult | None:
"""Run one monitor cycle on a fresh session under a hard time budget.
The session is opened and closed inside the call, so curl multi-handle
state cannot outlive one cycle: a wedged transfer dies with its session
instead of parking the process. The budget turns an engine-level lost
wakeup into a caught TimeoutError and an abandoned cycle, never a
permanent wedge; the session's own close is bounded by
get_default_session, so the exit cannot park past the budget either.
The caller still owns the loop and the inter-cycle delay:
while True:
await run_bounded_cycle(ids, task_func, proxies, collection)
await asyncio.sleep(CYCLE_DELAY)
Args:
category_identifiers: Categories, passed through to run_monitor_cycle.
task_func: Awaitable invoked per category.
proxies: Pool bound to each cycle's fresh session.
collection: Mongo collection forwarded to every callback.
session_timeout: Per-request timeout on the cycle's session.
max_clients: Curl handle pool size. Keep at or above the monitor's
whole fan-out; see get_default_session for why.
cycle_timeout: Hard budget in seconds for the whole cycle.
**kwargs: Forwarded to run_monitor_cycle (pre_task_func,
concurrent_pre_task_func, and per-task kwargs).
Returns:
The cycle's CycleResult, or None when the budget expired and the
cycle was abandoned.
"""
async with get_default_session(
proxies, timeout=session_timeout, max_clients=max_clients
) as session:
try:
async with asyncio.timeout(cycle_timeout):
return await run_monitor_cycle(
category_identifiers,
task_func,
session=session,
collection=collection,
**kwargs,
)
except TimeoutError:
LOGGER.error(
"cycle exceeded budget, abandoning cycle",
extra={"timeout": cycle_timeout},
)
return None
async def execute_pagination( async def execute_pagination(
category_identifier: str, category_identifier: str,
total_products: int, total_products: int,
+19
View File
@@ -30,6 +30,9 @@ def _doc(**overrides) -> dict:
"endDate": None, "endDate": None,
"priceBeforeDrop": None, "priceBeforeDrop": None,
"eduPrice": None, "eduPrice": None,
"pastPrices": [
{"price": 80.0, "date": datetime(2026, 7, 26, tzinfo=timezone.utc)}
],
"available": True, "available": True,
"condition": None, "condition": None,
"appExclusive": None, "appExclusive": None,
@@ -103,3 +106,19 @@ def test_dollar_code_does_not_affect_threshold():
promoMinSpend=100, promoMinSpend=100,
) )
assert build_product_details(doc, min_percentage=40) is None assert build_product_details(doc, min_percentage=40) is None
# ----------------------------------------------------------------- pastPrices
def test_past_prices_read_through():
"""Price history passes through unchanged."""
history = [
{"price": 95.0, "date": datetime(2026, 7, 20, tzinfo=timezone.utc)},
{"price": 90.0, "date": datetime(2026, 7, 24, tzinfo=timezone.utc)},
]
product = build_product_details(_doc(pastPrices=history), min_percentage=0)
assert product.pricing.past_prices == history
+135
View File
@@ -0,0 +1,135 @@
"""unit test: run_bounded_cycle session lifetime, budget, and pass-through
run from the repo root:
uv run pytest tests/test_run_bounded_cycle.py
offline — get_default_session is faked at the runner's namespace so no proxies
or network are touched; run_monitor_cycle underneath runs real. Gap: the real
session wiring (timeout/max_clients reaching curl) is covered by the monitors'
own pool-size tests, not here.
"""
import asyncio
import logging
import aiohttp
import pytest
from monitor_utils import runner
from monitor_utils.models import CycleResult
from monitor_utils.runner import run_bounded_cycle
class FakeSession:
def __init__(self):
self.closed = False
async def __aenter__(self):
return self
async def __aexit__(self, *exc):
self.closed = True
def stub_session(monkeypatch) -> dict:
"""Fake get_default_session, recording each call's kwargs and session."""
calls = []
def fake(proxies, timeout=None, max_clients=None):
session = FakeSession()
calls.append(
{"timeout": timeout, "max_clients": max_clients, "session": session}
)
return session
monkeypatch.setattr(runner, "get_default_session", fake)
return calls
def test_returns_cycle_result_and_closes_session(monkeypatch):
calls = stub_session(monkeypatch)
async def task(category, session, collection, **kwargs):
pass
result = asyncio.run(run_bounded_cycle(["a", "b"], task, None, None))
assert result == CycleResult(successes=2, failures=0)
assert calls[0]["session"].closed
def test_session_params_reach_the_factory(monkeypatch):
calls = stub_session(monkeypatch)
async def task(category, session, collection, **kwargs):
pass
asyncio.run(
run_bounded_cycle(
["a"], task, None, None, session_timeout=8, max_clients=81
)
)
assert calls[0]["timeout"] == 8
assert calls[0]["max_clients"] == 81
def test_expired_budget_returns_none_not_raise(monkeypatch, caplog):
"""A cycle past its budget is abandoned and logged, and the loop lives on."""
calls = stub_session(monkeypatch)
async def parked(category, session, collection, **kwargs):
await asyncio.Event().wait()
with caplog.at_level(logging.ERROR, logger="monitor_utils.runner"):
result = asyncio.run(
run_bounded_cycle(["a"], parked, None, None, cycle_timeout=0.05)
)
assert result is None
assert calls[0]["session"].closed
assert any(
r.message == "cycle exceeded budget, abandoning cycle" for r in caplog.records
)
def test_expected_task_error_counts_as_failure(monkeypatch):
stub_session(monkeypatch)
async def flaky(category, session, collection, **kwargs):
raise aiohttp.ClientError("routine")
result = asyncio.run(run_bounded_cycle(["a"], flaky, None, None))
assert result == CycleResult(successes=0, failures=1)
def test_unexpected_error_propagates_and_closes_session(monkeypatch):
"""fail loud: a real bug must escape the helper, not vanish into a log."""
calls = stub_session(monkeypatch)
async def broken(category, session, collection, **kwargs):
raise RuntimeError("bug")
with pytest.raises(RuntimeError):
asyncio.run(run_bounded_cycle(["a"], broken, None, None))
assert calls[0]["session"].closed
def test_each_call_opens_a_fresh_session(monkeypatch):
calls = stub_session(monkeypatch)
async def task(category, session, collection, **kwargs):
pass
async def two_cycles():
await run_bounded_cycle(["a"], task, None, None)
await run_bounded_cycle(["a"], task, None, None)
asyncio.run(two_cycles())
assert len(calls) == 2
assert calls[0]["session"] is not calls[1]["session"]