removed failed page retrying #11

Merged
TPV merged 1 commits from development into main 2026-08-08 20:51:09 +00:00
Showing only changes of commit 0b462aef06 - Show all commits
+2 -14
View File
@@ -461,16 +461,12 @@ async def execute_pagination(
"executing pagination",
extra={"category": category_identifier, "total": total_pages},
)
stop_event = asyncio.Event()
pagination_label = "offset" if pagination_mode == "offset" else "page"
page_range = range(2, total_pages + 1)
async def fetch_page(page_index: int):
"""Fetch a single page or offset, optionally under the shared semaphore."""
async with semaphore or nullcontext():
if stop_event.is_set():
return
if pagination_mode == "offset":
current_offset = start_offset + (page_index - 1) * products_per_page
await func(
@@ -488,14 +484,9 @@ async def execute_pagination(
failed_pages = await fetch_all_pages(
page_range, fetch_page, category_identifier, pagination_label
)
if failed_pages and not stop_event.is_set():
failed_pages = await retry_failed_pages(
failed_pages, fetch_page, category_identifier, pagination_label
)
if failed_pages and not stop_event.is_set():
if failed_pages:
LOGGER.error(
"failed to get products after retries",
"failed to get products from failed pages",
extra={
"category": category_identifier,
"label": pagination_label,
@@ -503,9 +494,6 @@ async def execute_pagination(
},
)
if stop_event.is_set():
LOGGER.info("pagination stopped early", extra={"category": category_identifier})
async def fetch_all_pages(
page_range: Iterable[int],