added concurrency limiting
This commit is contained in:
1 parent
4aa0e43289
commit
3a8a12ac6e
2 files changed
+93
-3
No files matched your search
@@ -293,7 +293,7 @@ async def execute_pagination(
|
||||
func: Callable[..., Awaitable[None]],
|
||||
session: aiohttp.ClientSession | AsyncSession,
|
||||
*,
|
||||
max_concurrent_tasks: int = 5,
|
||||
semaphore: asyncio.Semaphore,
|
||||
start_offset: int = 0,
|
||||
pagination_mode: str = "offset",
|
||||
**kwargs,
|
||||
@@ -311,7 +311,11 @@ async def execute_pagination(
|
||||
func: Awaitable task invoked per page; receives session plus either
|
||||
offset or page_number depending on pagination_mode.
|
||||
session: HTTP session forwarded to each func call.
|
||||
max_concurrent_tasks: Semaphore limit for concurrent page fetches.
|
||||
max_concurrent_tasks: Semaphore limit for concurrent page fetches, used
|
||||
only when semaphore is not supplied. Each execute_pagination call
|
||||
builds its own semaphore, so this bounds one category in isolation;
|
||||
with many categories running concurrently the effective limit is
|
||||
this value times the category count.
|
||||
start_offset: Additive base applied to every offset, for APIs whose
|
||||
offset space does not begin at zero. Page N requests offset
|
||||
start_offset + (N - 1) * products_per_page.
|
||||
@@ -327,7 +331,6 @@ async def execute_pagination(
|
||||
"executing pagination",
|
||||
extra={"category": category_identifier, "total": total_pages},
|
||||
)
|
||||
semaphore = asyncio.Semaphore(max_concurrent_tasks)
|
||||
stop_event = asyncio.Event()
|
||||
pagination_label = "offset" if pagination_mode == "offset" else "page"
|
||||
page_range = range(2, total_pages + 1)
|
||||
|
||||
Reference in new issue
Block a user