added concurrency limiting

This commit is contained in:
TPV committed 2026-07-17 14:24:53 -07:00
1 parent 4aa0e43289
commit 3a8a12ac6e
2 files changed
+93 -3

No files matched your search

+6 -3
View File
@@ -293,7 +293,7 @@ async def execute_pagination(
func: Callable[..., Awaitable[None]],
session: aiohttp.ClientSession | AsyncSession,
*,
max_concurrent_tasks: int = 5,
semaphore: asyncio.Semaphore,
start_offset: int = 0,
pagination_mode: str = "offset",
**kwargs,
@@ -311,7 +311,11 @@ async def execute_pagination(
func: Awaitable task invoked per page; receives session plus either
offset or page_number depending on pagination_mode.
session: HTTP session forwarded to each func call.
max_concurrent_tasks: Semaphore limit for concurrent page fetches.
max_concurrent_tasks: Semaphore limit for concurrent page fetches, used
only when semaphore is not supplied. Each execute_pagination call
builds its own semaphore, so this bounds one category in isolation;
with many categories running concurrently the effective limit is
this value times the category count.
start_offset: Additive base applied to every offset, for APIs whose
offset space does not begin at zero. Page N requests offset
start_offset + (N - 1) * products_per_page.
@@ -327,7 +331,6 @@ async def execute_pagination(
"executing pagination",
extra={"category": category_identifier, "total": total_pages},
)
semaphore = asyncio.Semaphore(max_concurrent_tasks)
stop_event = asyncio.Event()
pagination_label = "offset" if pagination_mode == "offset" else "page"
page_range = range(2, total_pages + 1)