added pagination fan out pagination abstraction
This commit is contained in:
1 parent
3f823b3a6e
commit
aa6ed4cb06
2 files changed
+162
-1
No files matched your search
@@ -1,7 +1,7 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable, Iterable
|
||||
from contextlib import nullcontext
|
||||
from contextlib import asynccontextmanager, nullcontext
|
||||
|
||||
import aiohttp
|
||||
from aioproxies import AioProxies
|
||||
@@ -568,3 +568,38 @@ def first_error(group_exc: BaseExceptionGroup) -> BaseException:
|
||||
error = error.exceptions[0]
|
||||
|
||||
return error
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def fan_out_group():
|
||||
"""A TaskGroup for page-one fan-out whose failure surfaces unwrapped.
|
||||
|
||||
Use it where one page's processing must be bound to the pagination that
|
||||
page spawns: create the processing task on the yielded group, then await
|
||||
execute_pagination in the body. The group cancels whichever sibling is
|
||||
still running when the other fails, so a dead category stops doing work
|
||||
nobody will use, and a cancelled cycle takes both down together.
|
||||
|
||||
The unwrapping is the reason this is shared rather than inlined. A bare
|
||||
TaskGroup always raises an ExceptionGroup, and run_monitor_cycle classifies
|
||||
task results with isinstance against EXPECTED_TASK_ERRORS -- an
|
||||
ExceptionGroup matches neither branch as intended, so a routine page
|
||||
failure would be reclassified as an unexpected one and kill the monitor.
|
||||
first_error unwraps to a leaf, preferring an unexpected exception so a real
|
||||
bug is never masked by a network error.
|
||||
|
||||
Note the caller's own body is inside the group, so a plain exception raised
|
||||
there (not from a child) propagates unchanged rather than being wrapped,
|
||||
and an early `return` still awaits the child tasks.
|
||||
|
||||
Example:
|
||||
async with fan_out_group() as group:
|
||||
group.create_task(process_page(data, ...))
|
||||
await execute_pagination(...)
|
||||
"""
|
||||
|
||||
try:
|
||||
async with asyncio.TaskGroup() as group:
|
||||
yield group
|
||||
except BaseExceptionGroup as group_exc:
|
||||
raise first_error(group_exc) from None
|
||||
Reference in new issue
Block a user