bug fix to avoid swallowing exceptions/failed pages inside fetch_all_pages/small refactor

src/monitor_utils/database.py

@@ -24,6 +24,9 @@ LOGGER = logging.getLogger(__name__)
 # without this they sort dead last and starve behind any backlog. 6 puts them
 # directly under the 70%+ band, tying with 60-69% and broken by createdAt.
 NO_MSRP_PRIORITY = 6
+# Floor rank for new listings, which send regardless of discount and would
+# otherwise sort near 0 and starve the same way MSRP-less products did.
+NEW_LISTING_PRIORITY = 6
 
 
 def get_collection(collection_name: str) -> AsyncIOMotorCollection:
@@ -104,7 +107,9 @@ async def add_products_to_database(
     Rewrites current fields on each upsert. priceBeforeDrop is derived here,
     not by monitors: it is set to the previous stored currentPrice when the new
     price is lower (a drop) and cleared otherwise, so any price_before_drop on
-    the incoming product is ignored. Prices are recorded in a pastPrices array
+    the incoming product is ignored. newListing is likewise stamped here, not
+    by monitors: True only on the cycle whose write inserts the document, and
+    copied onto the discovery alert, whose copy never flips. Prices are recorded in a pastPrices array
     of price/date entries; a price already present in the array is never
     recorded again, so each distinct price keeps only its first-seen timestamp.
     New products and never-before-recorded prices
@@ -148,6 +153,10 @@ async def add_products_to_database(
         document = convert_product_to_mongo(product)
         document["_id"] = doc_id
         existing_doc = existing.get(doc_id)
+        # Stamped here, not by monitors: only the database knows whether the
+        # doc already exists. Must precede build_alert so the alert's copy of
+        # the flag is taken from the stamped value.
+        document["newListing"] = existing_doc is None
         new_price = document.get("currentPrice")
         previous_price = existing_doc.get("currentPrice") if existing_doc else None
         document["priceBeforeDrop"] = resolve_price_before_drop(
@@ -260,11 +269,16 @@ def extract_recorded_prices(existing_doc: dict | None) -> set[float]:
 def build_alert(monitor_name: str, document: dict, now: datetime) -> dict:
     """Build an alert referencing a product for consumer delivery.
 
+    The alert carries its own copy of the document's newListing flag: the
+    product doc is rewritten every cycle, so its flag flips False one cycle
+    after discovery, while a discovery alert may sit in a backed-up queue
+    longer than that. The alert's copy is written once and cannot rot.
+
     Args:
         monitor_name: Monitor that produced the product, used by the
             consumer to resolve the owning collection and thresholds.
-        document: Product document supplying the id and pricing for the
-            priority calculation.
+        document: Product document supplying the id, pricing for the
+            priority calculation, and the stamped newListing flag.
         now: Detection timestamp stored as createdAt.
 
     Returns:
@@ -278,6 +292,7 @@ def build_alert(monitor_name: str, document: dict, now: datetime) -> dict:
         "claimed": False,
         "attempts": 0,
         "createdAt": now,
+        "newListing": document.get("newListing", False),
     }
 
 
@@ -289,7 +304,9 @@ def calculate_priority(document: dict) -> int:
     percentage. Missing prices, zero original prices, and price increases
     resolve to priority 0. A product with no MSRP has no percentage to rank
     on and takes NO_MSRP_PRIORITY instead, since a real 0 would leave it
-    permanently behind every other alert in the claim query's sort.
+    permanently behind every other alert in the claim query's sort. A new
+    listing is floored at NEW_LISTING_PRIORITY for the same reason: it may
+    carry any discount at all, and its percentage rank still wins when higher.
 
     Args:
         document: Product document supplying the prices.
@@ -307,7 +324,11 @@ def calculate_priority(document: dict) -> int:
         return NO_MSRP_PRIORITY
 
     percentage = int(((original_price - current_price) / original_price) * 100)
-    return min(max(percentage // 10, 0), 10)
+    priority = min(max(percentage // 10, 0), 10)
+    if document.get("newListing"):
+        return max(priority, NEW_LISTING_PRIORITY)
+
+    return priority
 
 
 def build_doc_id(

src/monitor_utils/models.py

@@ -123,6 +123,7 @@ class ProductMetadata:
     promo_percentage: int | None = None
     promo_dollar_off: int | None = None
     promo_min_spend: int | None = None
+    new_listing: bool | None = None
 
 
 @dataclass

src/monitor_utils/runner.py

@@ -148,17 +148,24 @@ def build_product_details(
     product: dict,
     min_percentage: int,
     min_price_drop: float | None = None,
+    bypass_thresholds: bool = False,
 ) -> Product | None:
     """Build a Product from a Mongo document if it passes discount thresholds.
 
     Skips documents with a missing or zero original price. Passing requires
-    meeting min_percentage, or min_price_drop when provided. Channel routing
-    is owned by the consumer bot; no channel id is attached here.
+    meeting min_percentage, or min_price_drop when provided. The caller may
+    bypass both thresholds (new listings send at any discount for monitors
+    that opt in); which alerts qualify is the caller's decision, not read
+    from the document, so per-monitor rules stay out of this function.
+    Channel routing is owned by the consumer bot; no channel id is attached
+    here.
 
     Args:
         product: Mongo document for the product.
         min_percentage: Minimum discount percentage to pass.
         min_price_drop: Optional minimum absolute price drop to pass.
+        bypass_thresholds: Build the product even when it fails both
+            thresholds. The missing-price skip still applies.
 
     Returns:
         The built Product, or None when skipped or below thresholds.
@@ -167,6 +174,14 @@ def build_product_details(
     product_id = product["_id"]
     current_price = product["currentPrice"]
     original_price = product["originalPrice"]
+    new_listing = product.get("newListing")
+
+    if not original_price:
+        LOGGER.warning(
+            "missing or zero original price, skipping",
+            extra={"product_id": product_id},
+        )
+        return
 
     percentage = 0
     # Defaulted, not indexed: documents written before the field existed have no
@@ -175,13 +190,6 @@ def build_product_details(
     # pre-existing threshold path rather than silently routing them as MSRP-less.
     has_msrp = product.get("hasMsrp", True)
     if has_msrp:
-        if not original_price:
-            LOGGER.warning(
-                "missing or zero original price, skipping",
-                extra={"product_id": product_id},
-            )
-            return
-
         percentage = int(((original_price - current_price) / original_price) * 100)
         price_drop = original_price - current_price
         # A percentage code compounds onto the sale price, so an item can clear the
@@ -193,7 +201,7 @@ def build_product_details(
         )
         passes_percentage = effective_percentage >= min_percentage
         passes_price_drop = min_price_drop is not None and price_drop >= min_price_drop
-        if not passes_percentage and not passes_price_drop:
+        if not bypass_thresholds and not passes_percentage and not passes_price_drop:
             return
 
     built = Product(
@@ -211,6 +219,7 @@ def build_product_details(
             promo_percentage=product.get("promoPercentage"),
             promo_dollar_off=product.get("promoDollarOff"),
             promo_min_spend=product.get("promoMinSpend"),
+            new_listing=new_listing,
         ),
         timing=ProductTiming(
             discovered_date=product["discoveredDate"],
@@ -513,11 +522,11 @@ async def fetch_all_pages(
         Page numbers whose fetch raised an expected error.
     """
 
-    tasks = [fetch_func(page) for page in page_range]
+    # list once: a one-shot iterable would be exhausted before the result pairing
+    pages = list(page_range)
+    tasks = [fetch_func(page) for page in pages]
     results = await asyncio.gather(*tasks, return_exceptions=True)
-    return handle_page_results(
-        list(page_range), results, category_identifier, pagination_label
-    )
+    return handle_page_results(pages, results, category_identifier, pagination_label)
 
 
 def handle_page_results(