Compare commits
2
Commits
5db82e3a59
...
cc46e3d409
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cc46e3d409 | ||
|
|
321546f9b9 |
@@ -525,20 +525,17 @@ def handle_page_results(
|
|||||||
results: list,
|
results: list,
|
||||||
category_identifier: str,
|
category_identifier: str,
|
||||||
pagination_label: str,
|
pagination_label: str,
|
||||||
retry_attempt: int | None = None,
|
|
||||||
) -> list[int]:
|
) -> list[int]:
|
||||||
"""Log page results and collect failures, raising on unexpected errors.
|
"""Log page results and collect failures, raising on unexpected errors.
|
||||||
|
|
||||||
Expected errors are logged as warnings and their pages collected for
|
Expected errors are logged as warnings and their pages collected and
|
||||||
retry. Unexpected exceptions are logged and re-raised. Successful pages
|
returned. Unexpected exceptions are logged and re-raised.
|
||||||
are logged only when part of a retry attempt.
|
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
page_numbers: Page numbers matching results by position.
|
page_numbers: Page numbers matching results by position.
|
||||||
results: Gathered results or exceptions per page.
|
results: Gathered results or exceptions per page.
|
||||||
category_identifier: Label used for logging.
|
category_identifier: Label used for logging.
|
||||||
pagination_label: Either offset or page, used for logging.
|
pagination_label: Either offset or page, used for logging.
|
||||||
retry_attempt: Retry attempt number when handling retry results.
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Page numbers whose fetch raised an expected error.
|
Page numbers whose fetch raised an expected error.
|
||||||
@@ -553,7 +550,6 @@ def handle_page_results(
|
|||||||
"category": category_identifier,
|
"category": category_identifier,
|
||||||
"label": pagination_label,
|
"label": pagination_label,
|
||||||
"page": page_number,
|
"page": page_number,
|
||||||
"retry": retry_attempt,
|
|
||||||
},
|
},
|
||||||
exc_info=result,
|
exc_info=result,
|
||||||
)
|
)
|
||||||
@@ -565,67 +561,10 @@ def handle_page_results(
|
|||||||
"category": category_identifier,
|
"category": category_identifier,
|
||||||
"label": pagination_label,
|
"label": pagination_label,
|
||||||
"page": page_number,
|
"page": page_number,
|
||||||
"retry": retry_attempt,
|
|
||||||
},
|
},
|
||||||
exc_info=result,
|
exc_info=result,
|
||||||
)
|
)
|
||||||
raise result
|
raise result
|
||||||
elif retry_attempt:
|
|
||||||
LOGGER.info(
|
|
||||||
"retry succeeded",
|
|
||||||
extra={
|
|
||||||
"category": category_identifier,
|
|
||||||
"label": pagination_label,
|
|
||||||
"page": page_number,
|
|
||||||
"retry": retry_attempt,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
return failed_pages
|
|
||||||
|
|
||||||
|
|
||||||
async def retry_failed_pages(
|
|
||||||
failed_pages: list[int],
|
|
||||||
fetch_func: Callable[[int], Awaitable[None]],
|
|
||||||
category_identifier: str,
|
|
||||||
pagination_label: str,
|
|
||||||
max_retries: int = 3,
|
|
||||||
) -> list[int]:
|
|
||||||
"""Retry failed pages with exponential backoff until exhausted or clear.
|
|
||||||
|
|
||||||
Sleeps 2**attempt seconds before each round and re-dispatches only the
|
|
||||||
pages that failed the previous round.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
failed_pages: Page numbers to retry.
|
|
||||||
fetch_func: Awaitable invoked per page number.
|
|
||||||
category_identifier: Label used for logging.
|
|
||||||
pagination_label: Either offset or page, used for logging.
|
|
||||||
max_retries: Maximum retry rounds.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Page numbers still failing after all retries.
|
|
||||||
"""
|
|
||||||
|
|
||||||
retry_attempt = 1
|
|
||||||
while failed_pages and retry_attempt <= max_retries:
|
|
||||||
await asyncio.sleep(2**retry_attempt)
|
|
||||||
LOGGER.info(
|
|
||||||
"retrying failed pages",
|
|
||||||
extra={
|
|
||||||
"category": category_identifier,
|
|
||||||
"label": pagination_label,
|
|
||||||
"count": len(failed_pages),
|
|
||||||
"attempt": retry_attempt,
|
|
||||||
"max_retries": max_retries,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
tasks = [fetch_func(page_number) for page_number in failed_pages]
|
|
||||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
||||||
failed_pages = handle_page_results(
|
|
||||||
failed_pages, results, category_identifier, pagination_label, retry_attempt
|
|
||||||
)
|
|
||||||
retry_attempt += 1
|
|
||||||
|
|
||||||
return failed_pages
|
return failed_pages
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user