Compare commits

...
2 Commits
+2 -63
View File
@@ -525,20 +525,17 @@ def handle_page_results(
results: list, results: list,
category_identifier: str, category_identifier: str,
pagination_label: str, pagination_label: str,
retry_attempt: int | None = None,
) -> list[int]: ) -> list[int]:
"""Log page results and collect failures, raising on unexpected errors. """Log page results and collect failures, raising on unexpected errors.
Expected errors are logged as warnings and their pages collected for Expected errors are logged as warnings and their pages collected and
retry. Unexpected exceptions are logged and re-raised. Successful pages returned. Unexpected exceptions are logged and re-raised.
are logged only when part of a retry attempt.
Args: Args:
page_numbers: Page numbers matching results by position. page_numbers: Page numbers matching results by position.
results: Gathered results or exceptions per page. results: Gathered results or exceptions per page.
category_identifier: Label used for logging. category_identifier: Label used for logging.
pagination_label: Either offset or page, used for logging. pagination_label: Either offset or page, used for logging.
retry_attempt: Retry attempt number when handling retry results.
Returns: Returns:
Page numbers whose fetch raised an expected error. Page numbers whose fetch raised an expected error.
@@ -553,7 +550,6 @@ def handle_page_results(
"category": category_identifier, "category": category_identifier,
"label": pagination_label, "label": pagination_label,
"page": page_number, "page": page_number,
"retry": retry_attempt,
}, },
exc_info=result, exc_info=result,
) )
@@ -565,67 +561,10 @@ def handle_page_results(
"category": category_identifier, "category": category_identifier,
"label": pagination_label, "label": pagination_label,
"page": page_number, "page": page_number,
"retry": retry_attempt,
}, },
exc_info=result, exc_info=result,
) )
raise result raise result
elif retry_attempt:
LOGGER.info(
"retry succeeded",
extra={
"category": category_identifier,
"label": pagination_label,
"page": page_number,
"retry": retry_attempt,
},
)
return failed_pages
async def retry_failed_pages(
failed_pages: list[int],
fetch_func: Callable[[int], Awaitable[None]],
category_identifier: str,
pagination_label: str,
max_retries: int = 3,
) -> list[int]:
"""Retry failed pages with exponential backoff until exhausted or clear.
Sleeps 2**attempt seconds before each round and re-dispatches only the
pages that failed the previous round.
Args:
failed_pages: Page numbers to retry.
fetch_func: Awaitable invoked per page number.
category_identifier: Label used for logging.
pagination_label: Either offset or page, used for logging.
max_retries: Maximum retry rounds.
Returns:
Page numbers still failing after all retries.
"""
retry_attempt = 1
while failed_pages and retry_attempt <= max_retries:
await asyncio.sleep(2**retry_attempt)
LOGGER.info(
"retrying failed pages",
extra={
"category": category_identifier,
"label": pagination_label,
"count": len(failed_pages),
"attempt": retry_attempt,
"max_retries": max_retries,
},
)
tasks = [fetch_func(page_number) for page_number in failed_pages]
results = await asyncio.gather(*tasks, return_exceptions=True)
failed_pages = handle_page_results(
failed_pages, results, category_identifier, pagination_label, retry_attempt
)
retry_attempt += 1
return failed_pages return failed_pages