added pagination fan out pagination abstraction
This commit is contained in:
1 parent
3f823b3a6e
commit
aa6ed4cb06
2 files changed
+162
-1
No files matched your search
@@ -1,7 +1,7 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
from collections.abc import Awaitable, Callable, Iterable
|
from collections.abc import Awaitable, Callable, Iterable
|
||||||
from contextlib import nullcontext
|
from contextlib import asynccontextmanager, nullcontext
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from aioproxies import AioProxies
|
from aioproxies import AioProxies
|
||||||
@@ -568,3 +568,38 @@ def first_error(group_exc: BaseExceptionGroup) -> BaseException:
|
|||||||
error = error.exceptions[0]
|
error = error.exceptions[0]
|
||||||
|
|
||||||
return error
|
return error
|
||||||
|
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def fan_out_group():
|
||||||
|
"""A TaskGroup for page-one fan-out whose failure surfaces unwrapped.
|
||||||
|
|
||||||
|
Use it where one page's processing must be bound to the pagination that
|
||||||
|
page spawns: create the processing task on the yielded group, then await
|
||||||
|
execute_pagination in the body. The group cancels whichever sibling is
|
||||||
|
still running when the other fails, so a dead category stops doing work
|
||||||
|
nobody will use, and a cancelled cycle takes both down together.
|
||||||
|
|
||||||
|
The unwrapping is the reason this is shared rather than inlined. A bare
|
||||||
|
TaskGroup always raises an ExceptionGroup, and run_monitor_cycle classifies
|
||||||
|
task results with isinstance against EXPECTED_TASK_ERRORS -- an
|
||||||
|
ExceptionGroup matches neither branch as intended, so a routine page
|
||||||
|
failure would be reclassified as an unexpected one and kill the monitor.
|
||||||
|
first_error unwraps to a leaf, preferring an unexpected exception so a real
|
||||||
|
bug is never masked by a network error.
|
||||||
|
|
||||||
|
Note the caller's own body is inside the group, so a plain exception raised
|
||||||
|
there (not from a child) propagates unchanged rather than being wrapped,
|
||||||
|
and an early `return` still awaits the child tasks.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
async with fan_out_group() as group:
|
||||||
|
group.create_task(process_page(data, ...))
|
||||||
|
await execute_pagination(...)
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with asyncio.TaskGroup() as group:
|
||||||
|
yield group
|
||||||
|
except BaseExceptionGroup as group_exc:
|
||||||
|
raise first_error(group_exc) from None
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
"""unit test: fan_out_group binds sibling tasks and surfaces errors unwrapped
|
||||||
|
|
||||||
|
run from the repo root:
|
||||||
|
uv run pytest tests/test_fan_out_group.py
|
||||||
|
|
||||||
|
offline — no network. These lock the contract monitors depend on: a bare
|
||||||
|
TaskGroup always raises an ExceptionGroup, which run_monitor_cycle would
|
||||||
|
misclassify (it matches neither the EXPECTED_TASK_ERRORS branch nor the
|
||||||
|
unexpected-Exception branch as intended), so the unwrapping is load-bearing and
|
||||||
|
not cosmetic.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
import aiohttp
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from monitor_utils.runner import fan_out_group
|
||||||
|
|
||||||
|
|
||||||
|
async def _boom(error: BaseException):
|
||||||
|
raise error
|
||||||
|
|
||||||
|
|
||||||
|
def test_child_error_surfaces_unwrapped():
|
||||||
|
"""A failing child arrives as itself, not wrapped in an ExceptionGroup."""
|
||||||
|
|
||||||
|
async def scenario():
|
||||||
|
async with fan_out_group() as group:
|
||||||
|
group.create_task(_boom(RuntimeError("child")))
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError) as exc_info:
|
||||||
|
asyncio.run(scenario())
|
||||||
|
|
||||||
|
assert not isinstance(exc_info.value, BaseExceptionGroup)
|
||||||
|
assert str(exc_info.value) == "child"
|
||||||
|
|
||||||
|
|
||||||
|
def test_body_error_surfaces_unwrapped():
|
||||||
|
"""An error raised by the caller's own body propagates unchanged."""
|
||||||
|
|
||||||
|
async def ok():
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
|
||||||
|
async def scenario():
|
||||||
|
async with fan_out_group() as group:
|
||||||
|
group.create_task(ok())
|
||||||
|
raise ValueError("body")
|
||||||
|
|
||||||
|
with pytest.raises(ValueError) as exc_info:
|
||||||
|
asyncio.run(scenario())
|
||||||
|
|
||||||
|
assert not isinstance(exc_info.value, BaseExceptionGroup)
|
||||||
|
|
||||||
|
|
||||||
|
def test_unexpected_error_is_preferred_over_an_expected_one():
|
||||||
|
"""first_error must not let a routine network error mask a real bug."""
|
||||||
|
|
||||||
|
async def scenario():
|
||||||
|
async with fan_out_group() as group:
|
||||||
|
group.create_task(_boom(aiohttp.ClientError("routine")))
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
raise ValueError("real bug")
|
||||||
|
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
asyncio.run(scenario())
|
||||||
|
|
||||||
|
|
||||||
|
def test_early_return_still_awaits_children():
|
||||||
|
"""A `return` inside the body must not abandon an in-flight child task."""
|
||||||
|
|
||||||
|
finished = []
|
||||||
|
|
||||||
|
async def child():
|
||||||
|
await asyncio.sleep(0.01)
|
||||||
|
finished.append("done")
|
||||||
|
|
||||||
|
async def scenario():
|
||||||
|
async with fan_out_group() as group:
|
||||||
|
group.create_task(child())
|
||||||
|
return "returned"
|
||||||
|
|
||||||
|
assert asyncio.run(scenario()) == "returned"
|
||||||
|
assert finished == ["done"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_parked_sibling_is_cancelled_when_the_body_fails():
|
||||||
|
"""The binding is the point: a child that never finishes must not hang.
|
||||||
|
|
||||||
|
Without the group, a create_task + `finally: await` shape would park here
|
||||||
|
forever. The deadline is the assertion — a hang fails this test.
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def parked():
|
||||||
|
await asyncio.Event().wait()
|
||||||
|
|
||||||
|
async def scenario():
|
||||||
|
async with asyncio.timeout(2):
|
||||||
|
async with fan_out_group() as group:
|
||||||
|
group.create_task(parked())
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
raise RuntimeError("body")
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
asyncio.run(scenario())
|
||||||
|
|
||||||
|
|
||||||
|
def test_children_run_concurrently_with_the_body():
|
||||||
|
"""Both siblings make progress; the group is not a sequencing device."""
|
||||||
|
|
||||||
|
order = []
|
||||||
|
|
||||||
|
async def child():
|
||||||
|
order.append("child-start")
|
||||||
|
await asyncio.sleep(0.01)
|
||||||
|
order.append("child-end")
|
||||||
|
|
||||||
|
async def scenario():
|
||||||
|
async with fan_out_group() as group:
|
||||||
|
group.create_task(child())
|
||||||
|
order.append("body")
|
||||||
|
await asyncio.sleep(0.02)
|
||||||
|
|
||||||
|
asyncio.run(scenario())
|
||||||
|
assert order == ["body", "child-start", "child-end"]
|
||||||
Reference in new issue
Block a user