From 0b462aef06efc4500964363326290752a1c2acd2 Mon Sep 17 00:00:00 2001 From: TPV Date: Sat, 8 Aug 2026 13:49:35 -0700 Subject: [PATCH] removed failed page retrying --- src/monitor_utils/runner.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/monitor_utils/runner.py b/src/monitor_utils/runner.py index bc689d4..ba4f50a 100644 --- a/src/monitor_utils/runner.py +++ b/src/monitor_utils/runner.py @@ -461,16 +461,12 @@ async def execute_pagination( "executing pagination", extra={"category": category_identifier, "total": total_pages}, ) - stop_event = asyncio.Event() pagination_label = "offset" if pagination_mode == "offset" else "page" page_range = range(2, total_pages + 1) async def fetch_page(page_index: int): """Fetch a single page or offset, optionally under the shared semaphore.""" async with semaphore or nullcontext(): - if stop_event.is_set(): - return - if pagination_mode == "offset": current_offset = start_offset + (page_index - 1) * products_per_page await func( @@ -488,14 +484,9 @@ async def execute_pagination( failed_pages = await fetch_all_pages( page_range, fetch_page, category_identifier, pagination_label ) - if failed_pages and not stop_event.is_set(): - failed_pages = await retry_failed_pages( - failed_pages, fetch_page, category_identifier, pagination_label - ) - - if failed_pages and not stop_event.is_set(): + if failed_pages: LOGGER.error( - "failed to get products after retries", + "failed to get products from failed pages", extra={ "category": category_identifier, "label": pagination_label, @@ -503,9 +494,6 @@ async def execute_pagination( }, ) - if stop_event.is_set(): - LOGGER.info("pagination stopped early", extra={"category": category_identifier}) - async def fetch_all_pages( page_range: Iterable[int], -- 2.54.0