added pagination fan out pagination abstraction

This commit is contained in:
TPV committed 2026-07-30 11:52:36 -07:00
1 parent 3f823b3a6e
commit aa6ed4cb06
2 files changed
+162 -1

No files matched your search

+36 -1
View File
@@ -1,7 +1,7 @@
import asyncio
import logging
from collections.abc import Awaitable, Callable, Iterable
from contextlib import nullcontext
from contextlib import asynccontextmanager, nullcontext
import aiohttp
from aioproxies import AioProxies
@@ -568,3 +568,38 @@ def first_error(group_exc: BaseExceptionGroup) -> BaseException:
error = error.exceptions[0]
return error
@asynccontextmanager
async def fan_out_group():
"""A TaskGroup for page-one fan-out whose failure surfaces unwrapped.
Use it where one page's processing must be bound to the pagination that
page spawns: create the processing task on the yielded group, then await
execute_pagination in the body. The group cancels whichever sibling is
still running when the other fails, so a dead category stops doing work
nobody will use, and a cancelled cycle takes both down together.
The unwrapping is the reason this is shared rather than inlined. A bare
TaskGroup always raises an ExceptionGroup, and run_monitor_cycle classifies
task results with isinstance against EXPECTED_TASK_ERRORS -- an
ExceptionGroup matches neither branch as intended, so a routine page
failure would be reclassified as an unexpected one and kill the monitor.
first_error unwraps to a leaf, preferring an unexpected exception so a real
bug is never masked by a network error.
Note the caller's own body is inside the group, so a plain exception raised
there (not from a child) propagates unchanged rather than being wrapped,
and an early `return` still awaits the child tasks.
Example:
async with fan_out_group() as group:
group.create_task(process_page(data, ...))
await execute_pagination(...)
"""
try:
async with asyncio.TaskGroup() as group:
yield group
except BaseExceptionGroup as group_exc:
raise first_error(group_exc) from None