added pagination fan out pagination abstraction

This commit is contained in:
TPV committed 2026-07-30 11:52:36 -07:00
1 parent 3f823b3a6e
commit aa6ed4cb06
2 files changed
+162 -1

No files matched your search

+36 -1
View File
@@ -1,7 +1,7 @@
import asyncio
import logging
from collections.abc import Awaitable, Callable, Iterable
from contextlib import nullcontext
from contextlib import asynccontextmanager, nullcontext
import aiohttp
from aioproxies import AioProxies
@@ -568,3 +568,38 @@ def first_error(group_exc: BaseExceptionGroup) -> BaseException:
error = error.exceptions[0]
return error
@asynccontextmanager
async def fan_out_group():
"""A TaskGroup for page-one fan-out whose failure surfaces unwrapped.
Use it where one page's processing must be bound to the pagination that
page spawns: create the processing task on the yielded group, then await
execute_pagination in the body. The group cancels whichever sibling is
still running when the other fails, so a dead category stops doing work
nobody will use, and a cancelled cycle takes both down together.
The unwrapping is the reason this is shared rather than inlined. A bare
TaskGroup always raises an ExceptionGroup, and run_monitor_cycle classifies
task results with isinstance against EXPECTED_TASK_ERRORS -- an
ExceptionGroup matches neither branch as intended, so a routine page
failure would be reclassified as an unexpected one and kill the monitor.
first_error unwraps to a leaf, preferring an unexpected exception so a real
bug is never masked by a network error.
Note the caller's own body is inside the group, so a plain exception raised
there (not from a child) propagates unchanged rather than being wrapped,
and an early `return` still awaits the child tasks.
Example:
async with fan_out_group() as group:
group.create_task(process_page(data, ...))
await execute_pagination(...)
"""
try:
async with asyncio.TaskGroup() as group:
yield group
except BaseExceptionGroup as group_exc:
raise first_error(group_exc) from None
+126
View File
@@ -0,0 +1,126 @@
"""unit test: fan_out_group binds sibling tasks and surfaces errors unwrapped
run from the repo root:
uv run pytest tests/test_fan_out_group.py
offline — no network. These lock the contract monitors depend on: a bare
TaskGroup always raises an ExceptionGroup, which run_monitor_cycle would
misclassify (it matches neither the EXPECTED_TASK_ERRORS branch nor the
unexpected-Exception branch as intended), so the unwrapping is load-bearing and
not cosmetic.
"""
import asyncio
import aiohttp
import pytest
from monitor_utils.runner import fan_out_group
async def _boom(error: BaseException):
raise error
def test_child_error_surfaces_unwrapped():
"""A failing child arrives as itself, not wrapped in an ExceptionGroup."""
async def scenario():
async with fan_out_group() as group:
group.create_task(_boom(RuntimeError("child")))
await asyncio.sleep(0)
with pytest.raises(RuntimeError) as exc_info:
asyncio.run(scenario())
assert not isinstance(exc_info.value, BaseExceptionGroup)
assert str(exc_info.value) == "child"
def test_body_error_surfaces_unwrapped():
"""An error raised by the caller's own body propagates unchanged."""
async def ok():
await asyncio.sleep(0)
async def scenario():
async with fan_out_group() as group:
group.create_task(ok())
raise ValueError("body")
with pytest.raises(ValueError) as exc_info:
asyncio.run(scenario())
assert not isinstance(exc_info.value, BaseExceptionGroup)
def test_unexpected_error_is_preferred_over_an_expected_one():
"""first_error must not let a routine network error mask a real bug."""
async def scenario():
async with fan_out_group() as group:
group.create_task(_boom(aiohttp.ClientError("routine")))
await asyncio.sleep(0)
raise ValueError("real bug")
with pytest.raises(ValueError):
asyncio.run(scenario())
def test_early_return_still_awaits_children():
"""A `return` inside the body must not abandon an in-flight child task."""
finished = []
async def child():
await asyncio.sleep(0.01)
finished.append("done")
async def scenario():
async with fan_out_group() as group:
group.create_task(child())
return "returned"
assert asyncio.run(scenario()) == "returned"
assert finished == ["done"]
def test_parked_sibling_is_cancelled_when_the_body_fails():
"""The binding is the point: a child that never finishes must not hang.
Without the group, a create_task + `finally: await` shape would park here
forever. The deadline is the assertion — a hang fails this test.
"""
async def parked():
await asyncio.Event().wait()
async def scenario():
async with asyncio.timeout(2):
async with fan_out_group() as group:
group.create_task(parked())
await asyncio.sleep(0)
raise RuntimeError("body")
with pytest.raises(RuntimeError):
asyncio.run(scenario())
def test_children_run_concurrently_with_the_body():
"""Both siblings make progress; the group is not a sequencing device."""
order = []
async def child():
order.append("child-start")
await asyncio.sleep(0.01)
order.append("child-end")
async def scenario():
async with fan_out_group() as group:
group.create_task(child())
order.append("body")
await asyncio.sleep(0.02)
asyncio.run(scenario())
assert order == ["body", "child-start", "child-end"]