diff --git a/src/monitor_utils/database.py b/src/monitor_utils/database.py index 16c6cec..ea26d6f 100644 --- a/src/monitor_utils/database.py +++ b/src/monitor_utils/database.py @@ -346,6 +346,7 @@ def convert_product_to_mongo(product: Product) -> dict: "priceBeforeDrop": pricing["price_before_drop"], "eduPrice": pricing["edu_price"], "available": availability["available"], + "availabilityLastChecked": availability["availability_last_checked"], "condition": availability["condition"], "appExclusive": availability["app_exclusive"], "memberPick": availability["member_pick"], diff --git a/src/monitor_utils/models.py b/src/monitor_utils/models.py index 50be55f..cb6c98b 100644 --- a/src/monitor_utils/models.py +++ b/src/monitor_utils/models.py @@ -54,6 +54,8 @@ class AvailabilityInfo: app_exclusive: Only purchasable through the retailer's app. member_pick: Flagged as a member pick offer. member_deal: Flagged as a member deal offer. + availability_last_checked: When available was last verified against + the retailer, letting monitors skip rechecks within an interval. """ available: bool | None = None @@ -61,6 +63,7 @@ class AvailabilityInfo: app_exclusive: bool | None = None member_pick: bool | None = None member_deal: bool | None = None + availability_last_checked: datetime | None = None @dataclass diff --git a/src/monitor_utils/runner.py b/src/monitor_utils/runner.py index 685db12..43b1ea6 100644 --- a/src/monitor_utils/runner.py +++ b/src/monitor_utils/runner.py @@ -84,7 +84,9 @@ async def init_monitor( async def setup_config( monitor_name: str, development_mode: DevelopmentMode -) -> tuple[list[str] | dict[str, str], dict[str, str | list[str]], AsyncIOMotorCollection]: +) -> tuple[ + list[str] | dict[str, str], dict[str, str | list[str]], AsyncIOMotorCollection +]: """Load monitor config and resolve the Mongo collection for the mode. Testing mode reads the configs_testing collection and appends a _testing @@ -275,14 +277,15 @@ async def execute_pagination( session: aiohttp.ClientSession | AsyncSession, *, max_concurrent_tasks: int = 5, - start_offset: int = 24, + start_offset: int = 0, pagination_mode: str = "offset", **kwargs, ): """Paginate a category by dispatching concurrent page or offset tasks to func. Runs pages 2..N under a semaphore, retries failed pages with exponential - backoff, and logs any pages that still fail after retries. + backoff, and logs any pages that still fail after retries. Page 1 is + assumed to have been fetched by the caller. Args: category_identifier: Label used for logging only; not passed to func. @@ -292,7 +295,9 @@ async def execute_pagination( offset or page_number depending on pagination_mode. session: HTTP session forwarded to each func call. max_concurrent_tasks: Semaphore limit for concurrent page fetches. - start_offset: Base offset added when pagination_mode is offset. + start_offset: Additive base applied to every offset, for APIs whose + offset space does not begin at zero. Page N requests offset + start_offset + (N - 1) * products_per_page. pagination_mode: Either offset or page, selecting the kwarg sent to func. **kwargs: Forwarded verbatim to every func call. Any identifier func needs (category_id, category_name, category_path, or bestbuy's @@ -317,7 +322,7 @@ async def execute_pagination( return if pagination_mode == "offset": - current_offset = start_offset + page_index * products_per_page + current_offset = start_offset + (page_index - 1) * products_per_page await func( session=session, offset=current_offset,