250 lines
8.1 KiB
Python
250 lines
8.1 KiB
Python
"""Database helpers for upserting monitored products and tracking price history.
|
|
|
|
Provides the shared Motor client, collection access, and the upsert path that
|
|
rewrites current product fields while preserving prior prices in a pastPrices
|
|
array deduplicated by price, stamping lastUpdated on every write, and emitting
|
|
alert references for new products and price changes.
|
|
"""
|
|
|
|
import logging
|
|
from dataclasses import asdict
|
|
from datetime import datetime, timezone
|
|
from functools import cache
|
|
|
|
from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorCollection
|
|
from pymongo import UpdateOne
|
|
from pymongo.errors import PyMongoError
|
|
|
|
from monitor_utils.constants import (
|
|
ALERTS_COLLECTION_NAME,
|
|
MONGO_DATABASE_NAME,
|
|
MONGO_URL,
|
|
)
|
|
from monitor_utils.models import Product
|
|
|
|
LOGGER = logging.getLogger(__name__)
|
|
|
|
|
|
def get_collection(collection_name: str) -> AsyncIOMotorCollection:
|
|
"""Return the named collection from the shared client's database.
|
|
|
|
Args:
|
|
collection_name: Name of the collection to return.
|
|
|
|
Returns:
|
|
The requested collection.
|
|
"""
|
|
|
|
return get_mongo_client()[MONGO_DATABASE_NAME][collection_name]
|
|
|
|
|
|
@cache
|
|
def get_mongo_client() -> AsyncIOMotorClient:
|
|
"""Return a cached Motor client configured for the shared cluster.
|
|
|
|
Returns:
|
|
The pooled, timezone-aware Motor client.
|
|
"""
|
|
|
|
return AsyncIOMotorClient(
|
|
MONGO_URL,
|
|
maxPoolSize=200,
|
|
minPoolSize=10,
|
|
retryWrites=True,
|
|
retryReads=True,
|
|
tz_aware=True,
|
|
tzinfo=timezone.utc,
|
|
)
|
|
|
|
|
|
async def get_product_from_database(
|
|
collection: AsyncIOMotorCollection,
|
|
product_id: str,
|
|
) -> dict:
|
|
"""Fetch a single product document by its _id."""
|
|
|
|
return await collection.find_one({"_id": product_id})
|
|
|
|
|
|
async def add_products_to_database(
|
|
products: list[Product],
|
|
collection: AsyncIOMotorCollection,
|
|
category_identifier: str,
|
|
monitor_name: str,
|
|
use_composite: bool = False,
|
|
):
|
|
"""Upsert products, tracking deduplicated price history and emitting alerts.
|
|
|
|
Rewrites current fields on each upsert. Prices are recorded in a
|
|
pastPrices array of price/date entries; a price already present in the
|
|
array is never recorded again, so each distinct price keeps only its
|
|
first-seen timestamp. New products and price changes
|
|
insert an alert referencing the product; alerts insert only after the
|
|
product write succeeds so a failed upsert cannot produce an orphaned
|
|
alert.
|
|
|
|
Args:
|
|
products: Products to upsert.
|
|
collection: Target MongoDB collection.
|
|
category_identifier: Category suffix appended to composite ids.
|
|
monitor_name: Monitor name stamped onto emitted alerts for consumer
|
|
routing.
|
|
use_composite: Build composite ids so multi-variant monitors (Woot)
|
|
keep one doc per variant; other monitors collapse to one doc per
|
|
product_id.
|
|
"""
|
|
|
|
doc_ids = [
|
|
build_doc_id(p.product_id, category_identifier, use_composite) for p in products
|
|
]
|
|
cursor = collection.find(
|
|
{"_id": {"$in": doc_ids}}, {"currentPrice": 1, "pastPrices": 1}
|
|
)
|
|
existing = {str(doc["_id"]): doc async for doc in cursor}
|
|
|
|
ops = []
|
|
alerts = []
|
|
now = datetime.now(timezone.utc)
|
|
for product, doc_id in zip(products, doc_ids):
|
|
document = convert_product_to_mongo(product)
|
|
document["_id"] = doc_id
|
|
existing_doc = existing.get(doc_id)
|
|
old_price = existing_doc.get("currentPrice") if existing_doc else None
|
|
new_price = document.get("currentPrice")
|
|
price_changed = new_price is not None and new_price != old_price
|
|
set_on_insert = {"discoveredDate": now}
|
|
if existing_doc is None or price_changed:
|
|
alerts.append(build_alert(monitor_name, document, now))
|
|
|
|
update = {"$set": document, "$setOnInsert": set_on_insert}
|
|
recorded_prices = extract_recorded_prices(existing_doc)
|
|
if new_price is not None and new_price not in recorded_prices:
|
|
update["$push"] = {"pastPrices": {"price": new_price, "date": now}}
|
|
|
|
ops.append(UpdateOne({"_id": doc_id}, update, upsert=True))
|
|
|
|
try:
|
|
await collection.bulk_write(ops, ordered=False)
|
|
if alerts:
|
|
await get_collection(ALERTS_COLLECTION_NAME).insert_many(
|
|
alerts, ordered=False
|
|
)
|
|
except PyMongoError:
|
|
LOGGER.exception(
|
|
"failed to add products to database",
|
|
extra={"category": category_identifier},
|
|
)
|
|
raise
|
|
|
|
|
|
def extract_recorded_prices(existing_doc: dict | None) -> set[float]:
|
|
"""Return the set of prices already recorded in a document's pastPrices.
|
|
|
|
Args:
|
|
existing_doc: Existing product document, or None for new products.
|
|
|
|
Returns:
|
|
Prices present in the pastPrices array, empty when none exist.
|
|
"""
|
|
|
|
if not existing_doc:
|
|
return set()
|
|
|
|
return {entry["price"] for entry in existing_doc.get("pastPrices", [])}
|
|
|
|
|
|
def build_alert(monitor_name: str, document: dict, now: datetime) -> dict:
|
|
"""Build an alert referencing a product for consumer delivery.
|
|
|
|
Args:
|
|
monitor_name: Monitor that produced the product, used by the
|
|
consumer to resolve the owning collection and thresholds.
|
|
document: Product document supplying the id and priority percentage.
|
|
now: Detection timestamp stored as createdAt.
|
|
|
|
Returns:
|
|
The alert document for insertion.
|
|
"""
|
|
|
|
percentage = document.get("percentage") or 0
|
|
data = {
|
|
"monitor": monitor_name,
|
|
"product_id": document["_id"],
|
|
"priority": min(percentage // 10, 10),
|
|
"claimed": False,
|
|
"attempts": 0,
|
|
"createdAt": now,
|
|
}
|
|
return {
|
|
"monitor": monitor_name,
|
|
"product_id": document["_id"],
|
|
"priority": min(percentage // 10, 10),
|
|
"claimed": False,
|
|
"attempts": 0,
|
|
"createdAt": now,
|
|
}
|
|
|
|
|
|
def build_doc_id(
|
|
product_id: str, category_identifier: str, use_composite: bool = False
|
|
) -> str:
|
|
"""Return the composite id when use_composite is set, else the bare product id.
|
|
|
|
Args:
|
|
product_id: Retailer-specific product identifier.
|
|
category_identifier: Category suffix for multi-variant monitors.
|
|
use_composite: Append category_identifier to disambiguate variants.
|
|
|
|
Returns:
|
|
The composite id when use_composite is set, otherwise product_id.
|
|
"""
|
|
|
|
return f"{product_id}_{category_identifier}" if use_composite else product_id
|
|
|
|
|
|
def convert_product_to_mongo(product: Product) -> dict:
|
|
"""Flatten a Product into its MongoDB document representation.
|
|
|
|
Coerces string prices to floats and stamps lastUpdated with the current
|
|
UTC time. The _id is assigned by the caller via build_doc_id.
|
|
|
|
Args:
|
|
product: Product to convert.
|
|
|
|
Returns:
|
|
The MongoDB document dict for the product.
|
|
"""
|
|
|
|
data = asdict(product)
|
|
metadata = data["metadata"]
|
|
timing = data["timing"]
|
|
pricing = data["pricing"]
|
|
availability = data["availability"]
|
|
current_price = pricing["current_price"]
|
|
original_price = pricing["original_price"]
|
|
if isinstance(current_price, str):
|
|
current_price = float(current_price.replace(",", ""))
|
|
original_price = float(original_price.replace(",", ""))
|
|
|
|
return {
|
|
"title": data["title"],
|
|
"url": data["url"],
|
|
"currentPrice": current_price,
|
|
"originalPrice": original_price,
|
|
"percentage": metadata["percentage"],
|
|
"imageUrl": metadata["image_url"],
|
|
"customizationIndex": metadata["customization_index"],
|
|
"memberNonMemberComparison": metadata["member_non_member_comparison"],
|
|
"overstockText": metadata["overstock_text"],
|
|
"startDate": timing["start_date"],
|
|
"endDate": timing["end_date"],
|
|
"priceBeforeDrop": pricing["price_before_drop"],
|
|
"eduPrice": pricing["edu_price"],
|
|
"available": availability["available"],
|
|
"condition": availability["condition"],
|
|
"appExclusive": availability["app_exclusive"],
|
|
"memberPick": availability["member_pick"],
|
|
"memberDeal": availability["member_deal"],
|
|
"lastUpdated": datetime.now(timezone.utc),
|
|
}
|