bug fix to avoid swallowing exceptions/failed pages inside fetch_all_pages/small refactor

src/monitor_utils/database.py

@@ -24,6 +24,9 @@ LOGGER = logging.getLogger(__name__)
 # without this they sort dead last and starve behind any backlog. 6 puts them
 # directly under the 70%+ band, tying with 60-69% and broken by createdAt.
 NO_MSRP_PRIORITY = 6
+# Floor rank for new listings, which send regardless of discount and would
+# otherwise sort near 0 and starve the same way MSRP-less products did.
+NEW_LISTING_PRIORITY = 6
 
 
 def get_collection(collection_name: str) -> AsyncIOMotorCollection:
@@ -104,7 +107,9 @@ async def add_products_to_database(
     Rewrites current fields on each upsert. priceBeforeDrop is derived here,
     not by monitors: it is set to the previous stored currentPrice when the new
     price is lower (a drop) and cleared otherwise, so any price_before_drop on
-    the incoming product is ignored. Prices are recorded in a pastPrices array
+    the incoming product is ignored. newListing is likewise stamped here, not
+    by monitors: True only on the cycle whose write inserts the document, and
+    copied onto the discovery alert, whose copy never flips. Prices are recorded in a pastPrices array
     of price/date entries; a price already present in the array is never
     recorded again, so each distinct price keeps only its first-seen timestamp.
     New products and never-before-recorded prices
@@ -148,6 +153,10 @@ async def add_products_to_database(
         document = convert_product_to_mongo(product)
         document["_id"] = doc_id
         existing_doc = existing.get(doc_id)
+        # Stamped here, not by monitors: only the database knows whether the
+        # doc already exists. Must precede build_alert so the alert's copy of
+        # the flag is taken from the stamped value.
+        document["newListing"] = existing_doc is None
         new_price = document.get("currentPrice")
         previous_price = existing_doc.get("currentPrice") if existing_doc else None
         document["priceBeforeDrop"] = resolve_price_before_drop(
@@ -260,11 +269,16 @@ def extract_recorded_prices(existing_doc: dict | None) -> set[float]:
 def build_alert(monitor_name: str, document: dict, now: datetime) -> dict:
     """Build an alert referencing a product for consumer delivery.
 
+    The alert carries its own copy of the document's newListing flag: the
+    product doc is rewritten every cycle, so its flag flips False one cycle
+    after discovery, while a discovery alert may sit in a backed-up queue
+    longer than that. The alert's copy is written once and cannot rot.
+
     Args:
         monitor_name: Monitor that produced the product, used by the
             consumer to resolve the owning collection and thresholds.
-        document: Product document supplying the id and pricing for the
-            priority calculation.
+        document: Product document supplying the id, pricing for the
+            priority calculation, and the stamped newListing flag.
         now: Detection timestamp stored as createdAt.
 
     Returns:
@@ -278,6 +292,7 @@ def build_alert(monitor_name: str, document: dict, now: datetime) -> dict:
         "claimed": False,
         "attempts": 0,
         "createdAt": now,
+        "newListing": document.get("newListing", False),
     }
 
 
@@ -289,7 +304,9 @@ def calculate_priority(document: dict) -> int:
     percentage. Missing prices, zero original prices, and price increases
     resolve to priority 0. A product with no MSRP has no percentage to rank
     on and takes NO_MSRP_PRIORITY instead, since a real 0 would leave it
-    permanently behind every other alert in the claim query's sort.
+    permanently behind every other alert in the claim query's sort. A new
+    listing is floored at NEW_LISTING_PRIORITY for the same reason: it may
+    carry any discount at all, and its percentage rank still wins when higher.
 
     Args:
         document: Product document supplying the prices.
@@ -307,7 +324,11 @@ def calculate_priority(document: dict) -> int:
         return NO_MSRP_PRIORITY
 
     percentage = int(((original_price - current_price) / original_price) * 100)
-    return min(max(percentage // 10, 0), 10)
+    priority = min(max(percentage // 10, 0), 10)
+    if document.get("newListing"):
+        return max(priority, NEW_LISTING_PRIORITY)
+
+    return priority
 
 
 def build_doc_id(