Merge pull request 'removed retry_failed_pages from docstrings' (#12) from development into main
Reviewed-on: #12
This commit was merged in pull request #12.
This commit is contained in:
commit
cc46e3d409
1 file changed
+2
-63
@@ -525,20 +525,17 @@ def handle_page_results(
|
||||
results: list,
|
||||
category_identifier: str,
|
||||
pagination_label: str,
|
||||
retry_attempt: int | None = None,
|
||||
) -> list[int]:
|
||||
"""Log page results and collect failures, raising on unexpected errors.
|
||||
|
||||
Expected errors are logged as warnings and their pages collected for
|
||||
retry. Unexpected exceptions are logged and re-raised. Successful pages
|
||||
are logged only when part of a retry attempt.
|
||||
Expected errors are logged as warnings and their pages collected and
|
||||
returned. Unexpected exceptions are logged and re-raised.
|
||||
|
||||
Args:
|
||||
page_numbers: Page numbers matching results by position.
|
||||
results: Gathered results or exceptions per page.
|
||||
category_identifier: Label used for logging.
|
||||
pagination_label: Either offset or page, used for logging.
|
||||
retry_attempt: Retry attempt number when handling retry results.
|
||||
|
||||
Returns:
|
||||
Page numbers whose fetch raised an expected error.
|
||||
@@ -553,7 +550,6 @@ def handle_page_results(
|
||||
"category": category_identifier,
|
||||
"label": pagination_label,
|
||||
"page": page_number,
|
||||
"retry": retry_attempt,
|
||||
},
|
||||
exc_info=result,
|
||||
)
|
||||
@@ -565,67 +561,10 @@ def handle_page_results(
|
||||
"category": category_identifier,
|
||||
"label": pagination_label,
|
||||
"page": page_number,
|
||||
"retry": retry_attempt,
|
||||
},
|
||||
exc_info=result,
|
||||
)
|
||||
raise result
|
||||
elif retry_attempt:
|
||||
LOGGER.info(
|
||||
"retry succeeded",
|
||||
extra={
|
||||
"category": category_identifier,
|
||||
"label": pagination_label,
|
||||
"page": page_number,
|
||||
"retry": retry_attempt,
|
||||
},
|
||||
)
|
||||
|
||||
return failed_pages
|
||||
|
||||
|
||||
async def retry_failed_pages(
|
||||
failed_pages: list[int],
|
||||
fetch_func: Callable[[int], Awaitable[None]],
|
||||
category_identifier: str,
|
||||
pagination_label: str,
|
||||
max_retries: int = 3,
|
||||
) -> list[int]:
|
||||
"""Retry failed pages with exponential backoff until exhausted or clear.
|
||||
|
||||
Sleeps 2**attempt seconds before each round and re-dispatches only the
|
||||
pages that failed the previous round.
|
||||
|
||||
Args:
|
||||
failed_pages: Page numbers to retry.
|
||||
fetch_func: Awaitable invoked per page number.
|
||||
category_identifier: Label used for logging.
|
||||
pagination_label: Either offset or page, used for logging.
|
||||
max_retries: Maximum retry rounds.
|
||||
|
||||
Returns:
|
||||
Page numbers still failing after all retries.
|
||||
"""
|
||||
|
||||
retry_attempt = 1
|
||||
while failed_pages and retry_attempt <= max_retries:
|
||||
await asyncio.sleep(2**retry_attempt)
|
||||
LOGGER.info(
|
||||
"retrying failed pages",
|
||||
extra={
|
||||
"category": category_identifier,
|
||||
"label": pagination_label,
|
||||
"count": len(failed_pages),
|
||||
"attempt": retry_attempt,
|
||||
"max_retries": max_retries,
|
||||
},
|
||||
)
|
||||
tasks = [fetch_func(page_number) for page_number in failed_pages]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
failed_pages = handle_page_results(
|
||||
failed_pages, results, category_identifier, pagination_label, retry_attempt
|
||||
)
|
||||
retry_attempt += 1
|
||||
|
||||
return failed_pages
|
||||
|
||||
|
||||
Reference in new issue
Block a user