base monitor

configs/proxies.txt

@@ -0,0 +1,11 @@
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-wayekwk9:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-81fnraj7:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-tp9v8r3c:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-13jbes1u:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-dh2w4oce:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-4dbijnps:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-ll4sdjxd:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-h9ilno2y:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-0513pzra:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-n0vzkol3:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-m2ahxif4:6pbhpgi7eoeg

samsung_shop/samsung_shop_monitor.py

@@ -1,7 +1,5 @@
 import asyncio
 import logging
-import re
-from urllib.parse import quote
 
 from curl_cffi import AsyncSession
 from monitor_utils.config import DevelopmentMode
@@ -14,12 +12,13 @@ from motor.motor_asyncio import AsyncIOMotorCollection
 
 DEVELOPMENT_MODE = DevelopmentMode.DEVELOPMENT
 LOGGER = logging.getLogger(__name__)
+PRODUCTS_PER_PAGE = 16
 
 
 async def main():
     """Run the samsung shop monitor loop over all category ids using a base session."""
     category_paths, collection, proxies = await init_monitor(
-        monitor_name="hp", development_mode=DEVELOPMENT_MODE
+        monitor_name="samsung_shop", development_mode=DEVELOPMENT_MODE
     )
     await execute_monitor_logic(
         proxies=proxies,
@@ -33,9 +32,8 @@ async def get_category_products(
     category_id: str,
     session: AsyncSession,
     collection: AsyncIOMotorCollection,
-    offset=30,
+    offset=0,
 ):
-    # update offset to actual offset
     LOGGER.debug("getting products", extra={"category": category_id, "offset": offset})
     try:
         response = await send_request(
@@ -48,8 +46,8 @@ async def get_category_products(
                 "firstSearchYN": "true",
                 "countryCode": "us",
                 "storeID": "us_education",
-                "startIndex": "0",
-                "requestCount": "16",
+                "startIndex": str(offset),
+                "requestCount": str(PRODUCTS_PER_PAGE),
                 "category_code": category_id,
                 "sort": "newest",
             },
@@ -82,6 +80,22 @@ async def get_category_products(
         )
         raise
 
+    if offset == 0:
+        await execute_pagination(
+            category_identifier=category_id,
+            total_products=response["searchTotalCount"],
+            products_per_page=PRODUCTS_PER_PAGE,
+            func=get_category_products,
+            session=session,
+            # execute_pagination numbers pages from 2 and computes
+            # start_offset + page * products_per_page, so the base must be
+            # negative one page for offsets to continue from the page we just
+            # fetched. Overshooting searchTotalCount makes the API 500.
+            start_offset=-PRODUCTS_PER_PAGE,
+            category_id=category_id,
+            collection=collection,
+        )
+
 
 def extract_products(response: dict) -> list[Product]:
     products = []

test.py

@@ -25,7 +25,7 @@ data = {
     "storeID": "us_education",
     "startIndex": "0",
     "requestCount": "16",
-    "category_code": "08080000",
+    "category_code": "08030000",
     # "filters": '[{"name":"Type","values":["Wall Ovens"],"value_norm":["Wall Ovens"]}]',
     "sort": "newest",
 }