abstracted out bounded cycles
This commit is contained in:
1 parent
aa6ed4cb06
commit
8903be2685
3 files changed
+229
-3
No files matched your search
@@ -15,8 +15,15 @@ from monitor_utils.config import (
|
||||
is_production,
|
||||
set_logger_levels,
|
||||
)
|
||||
from monitor_utils.constants import EXPECTED_TASK_ERRORS, LOGGING_LEVEL
|
||||
from monitor_utils.constants import (
|
||||
CYCLE_TIMEOUT_SECONDS,
|
||||
EXPECTED_TASK_ERRORS,
|
||||
LOGGING_LEVEL,
|
||||
MAX_CURL_HANDLES,
|
||||
TIMEOUT_SECONDS,
|
||||
)
|
||||
from monitor_utils.database import get_collection
|
||||
from monitor_utils.sessions import get_default_session
|
||||
from monitor_utils.models import (
|
||||
AvailabilityInfo,
|
||||
CycleResult,
|
||||
@@ -233,8 +240,10 @@ async def run_monitor_cycle(
|
||||
concurrently, logs expected failures, and raises unexpected ones. The
|
||||
caller owns the loop, the session lifetime, and the inter-cycle delay: wrap
|
||||
this in a `while True` that opens a fresh session per cycle and sleeps
|
||||
between cycles. The returned CycleResult is pure counts for the caller to
|
||||
log or act on; this function applies no policy to it.
|
||||
between cycles — or use run_bounded_cycle, which owns the session and a
|
||||
hard cycle budget and leaves only the loop and the sleep to the caller.
|
||||
The returned CycleResult is pure counts for the caller to log or act on;
|
||||
this function applies no policy to it.
|
||||
|
||||
session and collection are the two per-cycle resources, passed to every
|
||||
callback (pre_task_func, concurrent_pre_task_func, and each task_func) just
|
||||
@@ -329,6 +338,69 @@ async def run_monitor_cycle(
|
||||
return CycleResult(successes=successes, failures=failures)
|
||||
|
||||
|
||||
async def run_bounded_cycle(
|
||||
category_identifiers: list[str] | dict[str, str],
|
||||
task_func: Callable[..., Awaitable[None]],
|
||||
proxies: AioProxies,
|
||||
collection: AsyncIOMotorCollection,
|
||||
*,
|
||||
session_timeout: int = TIMEOUT_SECONDS,
|
||||
max_clients: int = MAX_CURL_HANDLES,
|
||||
cycle_timeout: int = CYCLE_TIMEOUT_SECONDS,
|
||||
**kwargs,
|
||||
) -> CycleResult | None:
|
||||
"""Run one monitor cycle on a fresh session under a hard time budget.
|
||||
|
||||
The session is opened and closed inside the call, so curl multi-handle
|
||||
state cannot outlive one cycle: a wedged transfer dies with its session
|
||||
instead of parking the process. The budget turns an engine-level lost
|
||||
wakeup into a caught TimeoutError and an abandoned cycle, never a
|
||||
permanent wedge; the session's own close is bounded by
|
||||
get_default_session, so the exit cannot park past the budget either.
|
||||
|
||||
The caller still owns the loop and the inter-cycle delay:
|
||||
|
||||
while True:
|
||||
await run_bounded_cycle(ids, task_func, proxies, collection)
|
||||
await asyncio.sleep(CYCLE_DELAY)
|
||||
|
||||
Args:
|
||||
category_identifiers: Categories, passed through to run_monitor_cycle.
|
||||
task_func: Awaitable invoked per category.
|
||||
proxies: Pool bound to each cycle's fresh session.
|
||||
collection: Mongo collection forwarded to every callback.
|
||||
session_timeout: Per-request timeout on the cycle's session.
|
||||
max_clients: Curl handle pool size. Keep at or above the monitor's
|
||||
whole fan-out; see get_default_session for why.
|
||||
cycle_timeout: Hard budget in seconds for the whole cycle.
|
||||
**kwargs: Forwarded to run_monitor_cycle (pre_task_func,
|
||||
concurrent_pre_task_func, and per-task kwargs).
|
||||
|
||||
Returns:
|
||||
The cycle's CycleResult, or None when the budget expired and the
|
||||
cycle was abandoned.
|
||||
"""
|
||||
|
||||
async with get_default_session(
|
||||
proxies, timeout=session_timeout, max_clients=max_clients
|
||||
) as session:
|
||||
try:
|
||||
async with asyncio.timeout(cycle_timeout):
|
||||
return await run_monitor_cycle(
|
||||
category_identifiers,
|
||||
task_func,
|
||||
session=session,
|
||||
collection=collection,
|
||||
**kwargs,
|
||||
)
|
||||
except TimeoutError:
|
||||
LOGGER.error(
|
||||
"cycle exceeded budget, abandoning cycle",
|
||||
extra={"timeout": cycle_timeout},
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def execute_pagination(
|
||||
category_identifier: str,
|
||||
total_products: int,
|
||||
|
||||
@@ -30,6 +30,9 @@ def _doc(**overrides) -> dict:
|
||||
"endDate": None,
|
||||
"priceBeforeDrop": None,
|
||||
"eduPrice": None,
|
||||
"pastPrices": [
|
||||
{"price": 80.0, "date": datetime(2026, 7, 26, tzinfo=timezone.utc)}
|
||||
],
|
||||
"available": True,
|
||||
"condition": None,
|
||||
"appExclusive": None,
|
||||
@@ -103,3 +106,19 @@ def test_dollar_code_does_not_affect_threshold():
|
||||
promoMinSpend=100,
|
||||
)
|
||||
assert build_product_details(doc, min_percentage=40) is None
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- pastPrices
|
||||
|
||||
|
||||
def test_past_prices_read_through():
|
||||
"""Price history passes through unchanged."""
|
||||
|
||||
history = [
|
||||
{"price": 95.0, "date": datetime(2026, 7, 20, tzinfo=timezone.utc)},
|
||||
{"price": 90.0, "date": datetime(2026, 7, 24, tzinfo=timezone.utc)},
|
||||
]
|
||||
|
||||
product = build_product_details(_doc(pastPrices=history), min_percentage=0)
|
||||
|
||||
assert product.pricing.past_prices == history
|
||||
@@ -0,0 +1,135 @@
|
||||
"""unit test: run_bounded_cycle session lifetime, budget, and pass-through
|
||||
|
||||
run from the repo root:
|
||||
uv run pytest tests/test_run_bounded_cycle.py
|
||||
|
||||
offline — get_default_session is faked at the runner's namespace so no proxies
|
||||
or network are touched; run_monitor_cycle underneath runs real. Gap: the real
|
||||
session wiring (timeout/max_clients reaching curl) is covered by the monitors'
|
||||
own pool-size tests, not here.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import aiohttp
|
||||
import pytest
|
||||
|
||||
from monitor_utils import runner
|
||||
from monitor_utils.models import CycleResult
|
||||
from monitor_utils.runner import run_bounded_cycle
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self):
|
||||
self.closed = False
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
self.closed = True
|
||||
|
||||
|
||||
def stub_session(monkeypatch) -> dict:
|
||||
"""Fake get_default_session, recording each call's kwargs and session."""
|
||||
calls = []
|
||||
|
||||
def fake(proxies, timeout=None, max_clients=None):
|
||||
session = FakeSession()
|
||||
calls.append(
|
||||
{"timeout": timeout, "max_clients": max_clients, "session": session}
|
||||
)
|
||||
return session
|
||||
|
||||
monkeypatch.setattr(runner, "get_default_session", fake)
|
||||
return calls
|
||||
|
||||
|
||||
def test_returns_cycle_result_and_closes_session(monkeypatch):
|
||||
calls = stub_session(monkeypatch)
|
||||
|
||||
async def task(category, session, collection, **kwargs):
|
||||
pass
|
||||
|
||||
result = asyncio.run(run_bounded_cycle(["a", "b"], task, None, None))
|
||||
|
||||
assert result == CycleResult(successes=2, failures=0)
|
||||
assert calls[0]["session"].closed
|
||||
|
||||
|
||||
def test_session_params_reach_the_factory(monkeypatch):
|
||||
calls = stub_session(monkeypatch)
|
||||
|
||||
async def task(category, session, collection, **kwargs):
|
||||
pass
|
||||
|
||||
asyncio.run(
|
||||
run_bounded_cycle(
|
||||
["a"], task, None, None, session_timeout=8, max_clients=81
|
||||
)
|
||||
)
|
||||
|
||||
assert calls[0]["timeout"] == 8
|
||||
assert calls[0]["max_clients"] == 81
|
||||
|
||||
|
||||
def test_expired_budget_returns_none_not_raise(monkeypatch, caplog):
|
||||
"""A cycle past its budget is abandoned and logged, and the loop lives on."""
|
||||
|
||||
calls = stub_session(monkeypatch)
|
||||
|
||||
async def parked(category, session, collection, **kwargs):
|
||||
await asyncio.Event().wait()
|
||||
|
||||
with caplog.at_level(logging.ERROR, logger="monitor_utils.runner"):
|
||||
result = asyncio.run(
|
||||
run_bounded_cycle(["a"], parked, None, None, cycle_timeout=0.05)
|
||||
)
|
||||
|
||||
assert result is None
|
||||
assert calls[0]["session"].closed
|
||||
assert any(
|
||||
r.message == "cycle exceeded budget, abandoning cycle" for r in caplog.records
|
||||
)
|
||||
|
||||
|
||||
def test_expected_task_error_counts_as_failure(monkeypatch):
|
||||
stub_session(monkeypatch)
|
||||
|
||||
async def flaky(category, session, collection, **kwargs):
|
||||
raise aiohttp.ClientError("routine")
|
||||
|
||||
result = asyncio.run(run_bounded_cycle(["a"], flaky, None, None))
|
||||
|
||||
assert result == CycleResult(successes=0, failures=1)
|
||||
|
||||
|
||||
def test_unexpected_error_propagates_and_closes_session(monkeypatch):
|
||||
"""fail loud: a real bug must escape the helper, not vanish into a log."""
|
||||
|
||||
calls = stub_session(monkeypatch)
|
||||
|
||||
async def broken(category, session, collection, **kwargs):
|
||||
raise RuntimeError("bug")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
asyncio.run(run_bounded_cycle(["a"], broken, None, None))
|
||||
|
||||
assert calls[0]["session"].closed
|
||||
|
||||
|
||||
def test_each_call_opens_a_fresh_session(monkeypatch):
|
||||
calls = stub_session(monkeypatch)
|
||||
|
||||
async def task(category, session, collection, **kwargs):
|
||||
pass
|
||||
|
||||
async def two_cycles():
|
||||
await run_bounded_cycle(["a"], task, None, None)
|
||||
await run_bounded_cycle(["a"], task, None, None)
|
||||
|
||||
asyncio.run(two_cycles())
|
||||
|
||||
assert len(calls) == 2
|
||||
assert calls[0]["session"] is not calls[1]["session"]
|
||||
Reference in new issue
Block a user