Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6ea3187b33 | ||
|
|
f51aed2c9f | ||
|
|
6720592685 | ||
|
|
4563cdf920 |
No files matched your search
@@ -33,17 +33,17 @@ FAILURE_DELAY = 3
|
|||||||
|
|
||||||
TIMEOUT_SECONDS = 30
|
TIMEOUT_SECONDS = 30
|
||||||
CYCLE_TIMEOUT_SECONDS = 600
|
CYCLE_TIMEOUT_SECONDS = 600
|
||||||
# curl_cffi's AsyncSession defaults to 10 handles and blocks on an untimed
|
|
||||||
# pool.get() past that, so a monitor fanning out wider silently serialises with
|
|
||||||
# no timeout covering the wait. Handles are allocated lazily (the pool holds
|
|
||||||
# None placeholders until popped), so a generous ceiling costs a low-concurrency
|
|
||||||
# monitor nothing. Monitors fanning out past this pass max_clients explicitly.
|
|
||||||
MAX_CURL_HANDLES = 64
|
MAX_CURL_HANDLES = 64
|
||||||
# Budget for closing a session. curl_cffi's close awaits the multi handle with
|
|
||||||
# no timeout of its own, so a wedged transfer would otherwise park whatever is
|
|
||||||
# closing the session, including an `async with` exit.
|
|
||||||
SESSION_CLOSE_TIMEOUT_SECONDS = 10
|
SESSION_CLOSE_TIMEOUT_SECONDS = 10
|
||||||
CLIENT_IDENTIFIERS = ["chrome", "chrome_android", "safari", "safari_ios"]
|
CLIENT_IDENTIFIERS = [
|
||||||
|
"chrome119",
|
||||||
|
"chrome120",
|
||||||
|
"chrome124",
|
||||||
|
"chrome131",
|
||||||
|
"chrome131_android",
|
||||||
|
"safari",
|
||||||
|
"safari_ios",
|
||||||
|
]
|
||||||
CLIENT_TIMEOUT = aiohttp.ClientTimeout(
|
CLIENT_TIMEOUT = aiohttp.ClientTimeout(
|
||||||
total=TIMEOUT_SECONDS, sock_connect=30, sock_read=30
|
total=TIMEOUT_SECONDS, sock_connect=30, sock_read=30
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ alert references for new products and price changes.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import math
|
||||||
from dataclasses import asdict
|
from dataclasses import asdict
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from functools import cache
|
from functools import cache
|
||||||
@@ -318,10 +319,13 @@ def calculate_priority(document: dict) -> int:
|
|||||||
|
|
||||||
Always recalculated from current and original price so the priority
|
Always recalculated from current and original price so the priority
|
||||||
reflects the prices at alert time rather than any stale stored
|
reflects the prices at alert time rather than any stale stored
|
||||||
percentage. Missing prices, zero original prices, and price increases
|
percentage. Missing prices, zero original prices, non-finite prices, a
|
||||||
resolve to priority 0. A product with no MSRP has no percentage to rank
|
discount too large to represent, and price increases resolve to priority
|
||||||
on and takes NO_MSRP_PRIORITY instead, since a real 0 would leave it
|
0, so one product's bad pricing can never raise out of the whole cycle (a
|
||||||
permanently behind every other alert in the claim query's sort. A new
|
retailer field like "Infinity" survives float() and used to crash the
|
||||||
|
monitor with an OverflowError). A product with no MSRP has no percentage
|
||||||
|
to rank on and takes NO_MSRP_PRIORITY instead, since a real 0 would leave
|
||||||
|
it permanently behind every other alert in the claim query's sort. A new
|
||||||
listing is floored at NEW_LISTING_PRIORITY for the same reason: it may
|
listing is floored at NEW_LISTING_PRIORITY for the same reason: it may
|
||||||
carry any discount at all, and its percentage rank still wins when higher.
|
carry any discount at all, and its percentage rank still wins when higher.
|
||||||
|
|
||||||
@@ -337,10 +341,17 @@ def calculate_priority(document: dict) -> int:
|
|||||||
if current_price is None or not original_price:
|
if current_price is None or not original_price:
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
if not (math.isfinite(current_price) and math.isfinite(original_price)):
|
||||||
|
return 0
|
||||||
|
|
||||||
if not document.get("hasMsrp", True):
|
if not document.get("hasMsrp", True):
|
||||||
return NO_MSRP_PRIORITY
|
return NO_MSRP_PRIORITY
|
||||||
|
|
||||||
percentage = int(((original_price - current_price) / original_price) * 100)
|
discount = ((original_price - current_price) / original_price) * 100
|
||||||
|
if not math.isfinite(discount):
|
||||||
|
return 0
|
||||||
|
|
||||||
|
percentage = int(discount)
|
||||||
priority = min(max(percentage // 10, 0), 10)
|
priority = min(max(percentage // 10, 0), 10)
|
||||||
if document.get("newListing"):
|
if document.get("newListing"):
|
||||||
return max(priority, NEW_LISTING_PRIORITY)
|
return max(priority, NEW_LISTING_PRIORITY)
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
|
import math
|
||||||
from collections.abc import Awaitable, Callable, Iterable
|
from collections.abc import Awaitable, Callable, Iterable
|
||||||
from contextlib import asynccontextmanager, nullcontext
|
from contextlib import asynccontextmanager, nullcontext
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from aioproxies import AioProxies
|
from aioproxies import AioProxies
|
||||||
from curl_cffi.requests import AsyncSession
|
from curl_cffi.requests import AsyncSession
|
||||||
|
from curl_cffi.requests.impersonate import BrowserTypeLiteral
|
||||||
from log_setup import setup_logging
|
from log_setup import setup_logging
|
||||||
from motor.motor_asyncio import AsyncIOMotorCollection
|
from motor.motor_asyncio import AsyncIOMotorCollection
|
||||||
from motor.motor_asyncio import AsyncIOMotorClient
|
from motor.motor_asyncio import AsyncIOMotorClient
|
||||||
@@ -159,7 +161,10 @@ def build_product_details(
|
|||||||
) -> Product | None:
|
) -> Product | None:
|
||||||
"""Build a Product from a Mongo document if it passes discount thresholds.
|
"""Build a Product from a Mongo document if it passes discount thresholds.
|
||||||
|
|
||||||
Skips documents with a missing or zero original price. Passing requires
|
Skips documents with a missing or zero original price, a non-finite
|
||||||
|
price, or a discount too large to represent: a retailer field like
|
||||||
|
"Infinity" survives float(), and the percentage math used to raise
|
||||||
|
OverflowError out of every caller. Passing requires
|
||||||
meeting min_percentage, or min_price_drop when provided. The caller may
|
meeting min_percentage, or min_price_drop when provided. The caller may
|
||||||
bypass both thresholds (new listings send at any discount for monitors
|
bypass both thresholds (new listings send at any discount for monitors
|
||||||
that opt in); which alerts qualify is the caller's decision, not read
|
that opt in); which alerts qualify is the caller's decision, not read
|
||||||
@@ -172,7 +177,7 @@ def build_product_details(
|
|||||||
min_percentage: Minimum discount percentage to pass.
|
min_percentage: Minimum discount percentage to pass.
|
||||||
min_price_drop: Optional minimum absolute price drop to pass.
|
min_price_drop: Optional minimum absolute price drop to pass.
|
||||||
bypass_thresholds: Build the product even when it fails both
|
bypass_thresholds: Build the product even when it fails both
|
||||||
thresholds. The missing-price skip still applies.
|
thresholds. The missing- and non-finite-price skips still apply.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The built Product, or None when skipped or below thresholds.
|
The built Product, or None when skipped or below thresholds.
|
||||||
@@ -190,6 +195,19 @@ def build_product_details(
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Skipped on BOTH paths, not just the percentage one: an MSRP-less product
|
||||||
|
# would otherwise carry the infinite price into its payload. Only floats can
|
||||||
|
# be non-finite, so ints and other types keep their existing behaviour.
|
||||||
|
if any(
|
||||||
|
isinstance(price, float) and not math.isfinite(price)
|
||||||
|
for price in (current_price, original_price)
|
||||||
|
):
|
||||||
|
LOGGER.warning(
|
||||||
|
"non-finite price, skipping",
|
||||||
|
extra={"product_id": product_id},
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
percentage = 0
|
percentage = 0
|
||||||
# Defaulted, not indexed: documents written before the field existed have no
|
# Defaulted, not indexed: documents written before the field existed have no
|
||||||
# hasMsrp at all, and stale ones that drop out of a monitor's feed are never
|
# hasMsrp at all, and stale ones that drop out of a monitor's feed are never
|
||||||
@@ -197,7 +215,15 @@ def build_product_details(
|
|||||||
# pre-existing threshold path rather than silently routing them as MSRP-less.
|
# pre-existing threshold path rather than silently routing them as MSRP-less.
|
||||||
has_msrp = product.get("hasMsrp", True)
|
has_msrp = product.get("hasMsrp", True)
|
||||||
if has_msrp:
|
if has_msrp:
|
||||||
percentage = int(((original_price - current_price) / original_price) * 100)
|
discount = ((original_price - current_price) / original_price) * 100
|
||||||
|
if not math.isfinite(discount):
|
||||||
|
LOGGER.warning(
|
||||||
|
"discount not representable, skipping",
|
||||||
|
extra={"product_id": product_id},
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
percentage = int(discount)
|
||||||
price_drop = original_price - current_price
|
price_drop = original_price - current_price
|
||||||
# A percentage code compounds onto the sale price, so an item can clear the
|
# A percentage code compounds onto the sale price, so an item can clear the
|
||||||
# threshold on the combined discount even when the base discount would not.
|
# threshold on the combined discount even when the base discount would not.
|
||||||
@@ -377,6 +403,7 @@ async def run_bounded_cycle(
|
|||||||
session_timeout: int = TIMEOUT_SECONDS,
|
session_timeout: int = TIMEOUT_SECONDS,
|
||||||
max_clients: int = MAX_CURL_HANDLES,
|
max_clients: int = MAX_CURL_HANDLES,
|
||||||
cycle_timeout: int = CYCLE_TIMEOUT_SECONDS,
|
cycle_timeout: int = CYCLE_TIMEOUT_SECONDS,
|
||||||
|
impersonate: BrowserTypeLiteral | None = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> CycleResult | None:
|
) -> CycleResult | None:
|
||||||
"""Run one monitor cycle on a fresh session under a hard time budget.
|
"""Run one monitor cycle on a fresh session under a hard time budget.
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import random
|
|||||||
from aioproxies import AioProxies
|
from aioproxies import AioProxies
|
||||||
from aioweb import Response
|
from aioweb import Response
|
||||||
from aioweb_tls import CurlCffi, TLSSession
|
from aioweb_tls import CurlCffi, TLSSession
|
||||||
|
from curl_cffi.requests.impersonate import BrowserTypeLiteral
|
||||||
from monitor_utils.constants import (
|
from monitor_utils.constants import (
|
||||||
CLIENT_IDENTIFIERS,
|
CLIENT_IDENTIFIERS,
|
||||||
MAX_CURL_HANDLES,
|
MAX_CURL_HANDLES,
|
||||||
@@ -70,6 +71,7 @@ def get_default_session(
|
|||||||
headers: dict | None = None,
|
headers: dict | None = None,
|
||||||
timeout: int = TIMEOUT_SECONDS,
|
timeout: int = TIMEOUT_SECONDS,
|
||||||
max_clients: int = MAX_CURL_HANDLES,
|
max_clients: int = MAX_CURL_HANDLES,
|
||||||
|
impersonate: BrowserTypeLiteral | None = None,
|
||||||
) -> TLSSession:
|
) -> TLSSession:
|
||||||
"""Return a TLSSession impersonating a random client fingerprint.
|
"""Return a TLSSession impersonating a random client fingerprint.
|
||||||
|
|
||||||
@@ -98,13 +100,17 @@ def get_default_session(
|
|||||||
checkout. Keep it at or above the monitor's real fan-out (the sum
|
checkout. Keep it at or above the monitor's real fan-out (the sum
|
||||||
of its pagination semaphores plus any unsemaphored per-cycle
|
of its pagination semaphores plus any unsemaphored per-cycle
|
||||||
requests) so the wait is never reachable.
|
requests) so the wait is never reachable.
|
||||||
|
impersonate: A curl_cffi profile to pin instead of drawing one from
|
||||||
|
CLIENT_IDENTIFIERS, for a retailer whose bot protection only
|
||||||
|
tolerates some profiles from a given IP (B&H's Cloudflare refuses
|
||||||
|
every chrome profile on a flagged IP but still accepts safari).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A BoundedCloseSession (a TLSSession whose close is bounded) over a
|
A BoundedCloseSession (a TLSSession whose close is bounded) over a
|
||||||
curl_cffi backend with a random impersonation, with the proxy pool
|
curl_cffi backend with a random impersonation, with the proxy pool
|
||||||
attached as proxy_pool.
|
attached as proxy_pool.
|
||||||
"""
|
"""
|
||||||
impersonate = random.choice(CLIENT_IDENTIFIERS)
|
impersonate = impersonate or random.choice(CLIENT_IDENTIFIERS)
|
||||||
session = BoundedCloseSession(
|
session = BoundedCloseSession(
|
||||||
backend=CurlCffi(impersonate=impersonate),
|
backend=CurlCffi(impersonate=impersonate),
|
||||||
headers=headers,
|
headers=headers,
|
||||||
|
|||||||
@@ -50,6 +50,23 @@ def test_broken_prices_still_outrank_nothing():
|
|||||||
assert calculate_priority(document(None, 100.0, hasMsrp=False)) == 0
|
assert calculate_priority(document(None, 100.0, hasMsrp=False)) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_finite_prices_resolve_to_zero_instead_of_raising():
|
||||||
|
"""float("Infinity") from a retailer field once crashed the whole monitor."""
|
||||||
|
inf, nan = float("inf"), float("nan")
|
||||||
|
assert calculate_priority(document(50.0, inf)) == 0
|
||||||
|
assert calculate_priority(document(inf, 100.0)) == 0
|
||||||
|
assert calculate_priority(document(-inf, 100.0)) == 0
|
||||||
|
assert calculate_priority(document(nan, 100.0)) == 0
|
||||||
|
assert calculate_priority(document(50.0, nan)) == 0
|
||||||
|
assert calculate_priority(document(50.0, inf, newListing=True)) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_discount_too_large_to_represent_resolves_to_zero():
|
||||||
|
"""Finite prices can still overflow the ratio (a subnormal original)."""
|
||||||
|
assert calculate_priority(document(-1e308, 5e-324)) == 0
|
||||||
|
assert calculate_priority(document(1e308, 5e-324)) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_new_listing_is_floored_not_buried():
|
def test_new_listing_is_floored_not_buried():
|
||||||
"""A low-discount new listing sends regardless, so it must not sort last."""
|
"""A low-discount new listing sends regardless, so it must not sort last."""
|
||||||
assert (
|
assert (
|
||||||
|
|||||||
@@ -203,3 +203,41 @@ def test_msrpless_doc_with_prices_builds_unthresholded():
|
|||||||
product = build_product_details(doc, min_percentage=40)
|
product = build_product_details(doc, min_percentage=40)
|
||||||
assert product is not None
|
assert product is not None
|
||||||
assert product.pricing.has_msrp is False
|
assert product.pricing.has_msrp is False
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ non-finite prices
|
||||||
|
|
||||||
|
|
||||||
|
def test_infinite_price_skips_instead_of_raising():
|
||||||
|
"""Regression (2026-10-09 06:25Z): float("Infinity") from a retailer field
|
||||||
|
raised OverflowError here and stopped the consumer's API deal loop."""
|
||||||
|
|
||||||
|
inf = float("inf")
|
||||||
|
assert build_product_details(_doc(originalPrice=inf), min_percentage=40) is None
|
||||||
|
assert build_product_details(_doc(currentPrice=inf), min_percentage=40) is None
|
||||||
|
assert build_product_details(_doc(currentPrice=-inf), min_percentage=40) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_nan_price_skips_instead_of_raising():
|
||||||
|
nan = float("nan")
|
||||||
|
assert build_product_details(_doc(currentPrice=nan), min_percentage=40) is None
|
||||||
|
assert build_product_details(_doc(originalPrice=nan), min_percentage=40) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_msrpless_infinite_price_skips_rather_than_reaching_the_payload():
|
||||||
|
"""Without the skip the MSRP-less path would carry inf into the Product."""
|
||||||
|
|
||||||
|
doc = _doc(hasMsrp=False, currentPrice=float("inf"))
|
||||||
|
assert build_product_details(doc, min_percentage=40) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_bypass_does_not_rescue_a_non_finite_price():
|
||||||
|
doc = _doc(originalPrice=float("inf"))
|
||||||
|
assert build_product_details(doc, min_percentage=40, bypass_thresholds=True) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_unrepresentable_discount_skips_instead_of_raising():
|
||||||
|
"""Finite prices can still overflow the ratio (a subnormal original)."""
|
||||||
|
|
||||||
|
doc = _doc(currentPrice=-1e308, originalPrice=5e-324)
|
||||||
|
assert build_product_details(doc, min_percentage=40) is None
|
||||||
Reference in new issue
Block a user