added fly out pagination

.gitattributes

@@ -0,0 +1,2 @@
+# shell hooks must stay LF or they fail to execute on Linux
+.githooks/** text eol=lf

.githooks/pre-commit.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+# block committing the bot/monitor configured in development mode
+#
+# the deployed branch must always run in production. this rejects any staged
+# python that sets DEVELOPMENT_MODE = DevelopmentMode.DEVELOPMENT. keep
+# DEVELOPMENT in your working tree while developing, just don't stage that line.
+#
+# activate once per clone:  git config core.hooksPath .githooks
+
+pattern='^[[:space:]]*DEVELOPMENT_MODE[[:space:]]*=[[:space:]]*DevelopmentMode\.DEVELOPMENT\b'
+offenders=""
+
+for file in $(git diff --cached --name-only --diff-filter=d -- '*.py'); do
+    if git show ":$file" | grep -qE "$pattern"; then
+        offenders="$offenders $file"
+    fi
+done
+
+if [ -n "$offenders" ]; then
+    echo "pre-commit: refusing to commit DEVELOPMENT mode in:$offenders" >&2
+    echo "  set DEVELOPMENT_MODE = DevelopmentMode.PRODUCTION before committing," >&2
+    echo "  or 'git restore --staged <file>' to leave it out of this commit." >&2
+    exit 1
+fi
+
+exit 0

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.13-slim
+ENV PYTHONUNBUFFERED=1 \
+    HOME=/tmp
+COPY --from=ghcr.io/astral-sh/uv:0.5 /uv /uvx /bin/
+RUN apt-get update && apt-get install -y --no-install-recommends git \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /app
+COPY pyproject.toml ./
+RUN uv pip install --system --no-cache -r pyproject.toml
+COPY samsung_shop/ ./samsung_shop/
+COPY configs/proxies.txt ./proxies.txt
+RUN mkdir -p logs && chmod -R a+rwX /app
+CMD ["python", "-m", "samsung_shop.samsung_shop_monitor"]

docker-compose.yml

@@ -0,0 +1,14 @@
+services:
+  svc:
+    build: .
+    user: "1337:1337"
+    restart: unless-stopped
+    environment:
+      HOME: /tmp
+    volumes:
+      - ${LOGS_DIR:-./logs}:/app/logs
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"

samsung_shop/samsung_shop_monitor.py

@@ -6,12 +6,19 @@ from monitor_utils.config import DevelopmentMode
 from monitor_utils.constants import EXPECTED_TASK_ERRORS
 from monitor_utils.database import add_products_to_database
 from monitor_utils.models import AvailabilityInfo, PricingInfo, Product, ProductMetadata
-from monitor_utils.runner import execute_monitor_logic, execute_pagination, init_monitor
+from monitor_utils.runner import (
+    execute_monitor_logic,
+    execute_pagination,
+    init_monitor,
+    setup_monitor_logging,
+)
 from monitor_utils.sessions import send_request
 from motor.motor_asyncio import AsyncIOMotorCollection
 
-DEVELOPMENT_MODE = DevelopmentMode.DEVELOPMENT
+setup_monitor_logging(monitor_name="samsung_shop")
+DEVELOPMENT_MODE = DevelopmentMode.PRODUCTION
 LOGGER = logging.getLogger(__name__)
+
 PRODUCTS_PER_PAGE = 16
 
 
@@ -52,6 +59,48 @@ async def get_category_products(
                 "sort": "newest",
             },
         )
+    except EXPECTED_TASK_ERRORS:
+        LOGGER.warning(
+            "failed to get products",
+            extra={"category": category_id, "offset": offset},
+        )
+        raise
+
+    if offset != 0:
+        await process_page(response, category_id, offset, collection)
+        return
+
+    page_task = asyncio.create_task(
+        process_page(response, category_id, offset, collection)
+    )
+    try:
+        total_products = response.get("searchTotalCount")
+        if total_products is None:
+            LOGGER.warning("missing searchTotalCount", extra={"category": category_id})
+            return
+
+        await execute_pagination(
+            category_identifier=category_id,
+            total_products=total_products,
+            products_per_page=PRODUCTS_PER_PAGE,
+            func=get_category_products,
+            session=session,
+            category_id=category_id,
+            collection=collection,
+        )
+    finally:
+        await page_task
+
+
+async def process_page(
+    response: dict,
+    category_id: str,
+    offset: int,
+    collection: AsyncIOMotorCollection,
+):
+    """Extract and persist one page of category search results."""
+
+    try:
         products = extract_products(response)
         if products:
             await add_products_to_database(
@@ -75,27 +124,11 @@ async def get_category_products(
             )
     except EXPECTED_TASK_ERRORS:
         LOGGER.warning(
-            "failed to get products",
+            "failed to build products",
             extra={"category": category_id, "offset": offset},
         )
         raise
 
-    if offset == 0:
-        await execute_pagination(
-            category_identifier=category_id,
-            total_products=response["searchTotalCount"],
-            products_per_page=PRODUCTS_PER_PAGE,
-            func=get_category_products,
-            session=session,
-            # execute_pagination numbers pages from 2 and computes
-            # start_offset + page * products_per_page, so the base must be
-            # negative one page for offsets to continue from the page we just
-            # fetched. Overshooting searchTotalCount makes the API 500.
-            start_offset=-PRODUCTS_PER_PAGE,
-            category_id=category_id,
-            collection=collection,
-        )
-
 
 def extract_products(response: dict) -> list[Product]:
     products = []

test.py

@@ -1,42 +0,0 @@
-import requests
-
-headers = {
-    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0",
-    "Accept": "*/*",
-    "Accept-Language": "en-US,en;q=0.9",
-    # 'Accept-Encoding': 'gzip, deflate, br, zstd',
-    "Content-Type": "application/x-www-form-urlencoded",
-    "Origin": "https://www.samsung.com",
-    "Connection": "keep-alive",
-    "Referer": "https://www.samsung.com/",
-    "Sec-Fetch-Dest": "empty",
-    "Sec-Fetch-Mode": "cors",
-    "Sec-Fetch-Site": "same-site",
-    "Priority": "u=0",
-    # Requests doesn't support trailers
-    # 'TE': 'trailers',
-}
-
-data = {
-    "clientCode": "b2c",
-    "clientName": "scom_pf",
-    "firstSearchYN": "true",
-    "countryCode": "us",
-    "storeID": "us_education",
-    "startIndex": "0",
-    "requestCount": "16",
-    "category_code": "08030000",
-    # "filters": '[{"name":"Type","values":["Wall Ovens"],"value_norm":["Wall Ovens"]}]',
-    "sort": "newest",
-}
-
-response = requests.post(
-    "https://sribsrch.ecom.samsung.com/estoresearch-api/v1/scom/us/pf_search",
-    headers=headers,
-    data=data,
-)
-print(response.json())
-with open("resp.json", "w") as write_resp:
-    import json
-
-    json.dump(response.json(), write_resp, indent=4)

tests/configs/proxies.txt

@@ -0,0 +1,11 @@
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-wayekwk9:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-81fnraj7:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-tp9v8r3c:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-13jbes1u:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-dh2w4oce:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-4dbijnps:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-ll4sdjxd:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-h9ilno2y:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-0513pzra:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-n0vzkol3:6pbhpgi7eoeg
+v3ispunlimited.hellworld.io:30:26033123vukg4js-country-US-time-30-session-m2ahxif4:6pbhpgi7eoeg