fixed a small database issue with price drop tracking and added doc strings
This commit is contained in:
1 parent
a61dbdfaa9
commit
99d660ee52
6 files changed
+172
-4
No files matched your search
@@ -243,6 +243,27 @@ async def execute_pagination(
|
||||
pagination_mode: str = "offset",
|
||||
**kwargs,
|
||||
):
|
||||
"""Paginate a category by dispatching concurrent page or offset tasks to func.
|
||||
|
||||
Runs pages 2..N under a semaphore, retries failed pages with exponential
|
||||
backoff, and logs any pages that still fail after retries.
|
||||
|
||||
Args:
|
||||
category_identifier: Label used for logging only; not passed to func.
|
||||
total_products: Total product count used to derive page count.
|
||||
products_per_page: Page size used to derive page count and offsets.
|
||||
func: Awaitable task invoked per page; receives session plus either
|
||||
offset or page_number depending on pagination_mode.
|
||||
session: HTTP session forwarded to each func call.
|
||||
max_concurrent_tasks: Semaphore limit for concurrent page fetches.
|
||||
start_offset: Base offset added when pagination_mode is offset.
|
||||
pagination_mode: Either offset or page, selecting the kwarg sent to func.
|
||||
**kwargs: Forwarded verbatim to every func call. Any identifier func
|
||||
needs (category_id, category_name, category_path, or bestbuy's
|
||||
query_category_id) must be supplied here, since naming varies
|
||||
per monitor.
|
||||
"""
|
||||
|
||||
LOGGER.info("executing pagination", extra={"category": category_identifier})
|
||||
total_pages = (int(total_products) + products_per_page - 1) // products_per_page
|
||||
semaphore = asyncio.Semaphore(max_concurrent_tasks)
|
||||
|
||||
Reference in new issue
Block a user