From 03cc3a35cbe04e85254b0445097087d405257bb7 Mon Sep 17 00:00:00 2001 From: TPV Date: Tue, 8 Sep 2026 20:14:49 -0700 Subject: [PATCH] added asins for woot listings --- .gitignore | 5 +- scripts/bootstrap_db.py | 319 +++++++++++++++++++ scripts/db_seed/control.json | 53 +++ scripts/db_seed/production/apple.json | 74 +++++ scripts/db_seed/production/bestbuy.json | 144 +++++++++ scripts/db_seed/production/corsair.json | 15 + scripts/db_seed/production/hp.json | 35 ++ scripts/db_seed/production/lg.json | 30 ++ scripts/db_seed/production/northerntool.json | 46 +++ scripts/db_seed/production/samsung.json | 44 +++ scripts/db_seed/production/samsung_shop.json | 36 +++ scripts/db_seed/production/woot.json | 21 ++ scripts/db_seed/testing/apple.json | 65 ++++ scripts/db_seed/testing/bestbuy.json | 144 +++++++++ scripts/db_seed/testing/corsair.json | 22 ++ scripts/db_seed/testing/hp.json | 35 ++ scripts/db_seed/testing/lg.json | 29 ++ scripts/db_seed/testing/northerntool.json | 46 +++ scripts/db_seed/testing/samsung.json | 43 +++ scripts/db_seed/testing/samsung_shop.json | 36 +++ scripts/db_seed/testing/woot.json | 21 ++ src/monitor_utils/database.py | 1 + src/monitor_utils/models.py | 1 + tests/test_bootstrap_db.py | 302 ++++++++++++++++++ 24 files changed, 1565 insertions(+), 2 deletions(-) create mode 100644 scripts/bootstrap_db.py create mode 100644 scripts/db_seed/control.json create mode 100644 scripts/db_seed/production/apple.json create mode 100644 scripts/db_seed/production/bestbuy.json create mode 100644 scripts/db_seed/production/corsair.json create mode 100644 scripts/db_seed/production/hp.json create mode 100644 scripts/db_seed/production/lg.json create mode 100644 scripts/db_seed/production/northerntool.json create mode 100644 scripts/db_seed/production/samsung.json create mode 100644 scripts/db_seed/production/samsung_shop.json create mode 100644 scripts/db_seed/production/woot.json create mode 100644 scripts/db_seed/testing/apple.json create mode 100644 scripts/db_seed/testing/bestbuy.json create mode 100644 scripts/db_seed/testing/corsair.json create mode 100644 scripts/db_seed/testing/hp.json create mode 100644 scripts/db_seed/testing/lg.json create mode 100644 scripts/db_seed/testing/northerntool.json create mode 100644 scripts/db_seed/testing/samsung.json create mode 100644 scripts/db_seed/testing/samsung_shop.json create mode 100644 scripts/db_seed/testing/woot.json create mode 100644 tests/test_bootstrap_db.py diff --git a/.gitignore b/.gitignore index a4fbbb8..980645b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,6 @@ __pycache__ .idea monitor_utils.egg-info uv.lock -.claude -build/ \ No newline at end of file +.claude/ +build/ +logs/ \ No newline at end of file diff --git a/scripts/bootstrap_db.py b/scripts/bootstrap_db.py new file mode 100644 index 0000000..c053e7e --- /dev/null +++ b/scripts/bootstrap_db.py @@ -0,0 +1,319 @@ +"""Bootstrap the shared Mongo database: collections, indexes, and config seeds. + +Rebuilds what the monitors and the discord consumer expect to find: the +configs/control/alerts collections in both modes, the alerts indexes the +consumer relies on for claim ordering and dedupe, one empty product +collection per monitor and mode, the per-monitor config documents from +scripts/db_seed//.json, and the control docs from +scripts/db_seed/control.json (the monitors registry; log_channel and +bot_owner hold Discord ids only the bot's commands know, and migrations +must stay absent so the consumer re-runs them on a fresh database). + +Idempotent: existing collections and indexes are left alone and seed +documents use $setOnInsert, so re-running never clobbers operator edits. + +Usage: + python scripts/bootstrap_db.py [--dry-run] [--url URL] [--database NAME] +""" + +import argparse +import json +import logging +import sys +from pathlib import Path + +from pymongo import ASCENDING, DESCENDING, MongoClient +from pymongo.collection import Collection +from pymongo.database import Database + +from monitor_utils.constants import MONGO_DATABASE_NAME, MONGO_URL + +LOGGER = logging.getLogger("bootstrap_db") + +SEED_ROOT = Path(__file__).resolve().parent / "db_seed" +# collection suffix -> seed directory +MODES = {"": "production", "_testing": "testing"} +# monitors that call init_monitor and so own a product collection per mode +MONITORS = [ + "apple", + "bestbuy", + "corsair", + "hp", + "lg", + "northerntool", + "samsung", + "samsung_shop", + "woot", +] +SEED_KEYS = {"categories", "channels"} +# categories shape each monitor's task function accepts: run_monitor_cycle +# passes query_category_id= only for a mapping, and only bestbuy (required +# query) and apple (node -> route) take that argument; the rest take a list +CATEGORY_SHAPES = { + "apple": dict, + "bestbuy": dict, + "corsair": list, + "hp": list, + "lg": list, + "northerntool": list, + "samsung": list, + "samsung_shop": list, + "woot": list, +} +CONTROL_SEED = SEED_ROOT / "control.json" + +# shape setup_config (runner.py) reads: _id is the monitor name, categories is +# a list of names or a name -> query mapping, channels maps a bucket to ONE +# channel id as a digit string — the consumer's find_invalid_channel_ids +# rejects lists (the pre-07-16 spreading shape). Extra fields stay allowed. +CHANNEL_ID = {"bsonType": "string", "pattern": "^[0-9]+$"} +CONFIG_VALIDATOR = { + "$jsonSchema": { + "bsonType": "object", + "required": ["_id", "categories", "channels"], + "properties": { + "_id": {"bsonType": "string"}, + "categories": { + "anyOf": [ + {"bsonType": "array", "items": {"bsonType": "string"}}, + { + "bsonType": "object", + "additionalProperties": {"bsonType": "string"}, + }, + ] + }, + "channels": { + "bsonType": "object", + "additionalProperties": CHANNEL_ID, + }, + }, + } +} +# identical to the consumer's cog_load create_index calls; a differing spec +# under the same key pattern would make its startup fail +ALERT_INDEXES = [ + ([("claimed", ASCENDING), ("priority", DESCENDING), ("createdAt", ASCENDING)], {}), + ( + [("product_id", ASCENDING)], + {"unique": True, "partialFilterExpression": {"claimed": False}}, + ), +] + + +def ensure_collection( + database: Database, name: str, dry_run: bool = False, **options +) -> bool: + """Create a collection when absent; return True when it was created.""" + + if name in database.list_collection_names(): + LOGGER.info("collection exists", extra={"collection": name}) + return False + + LOGGER.info("creating collection", extra={"collection": name, "dry_run": dry_run}) + if not dry_run: + database.create_collection(name, **options) + + return True + + +def ensure_config_validator( + database: Database, name: str, dry_run: bool = False +) -> bool: + """Apply CONFIG_VALIDATOR to an existing collection; True when it changed.""" + + info = next(iter(database.list_collections(filter={"name": name})), None) + if info is None or info["options"].get("validator") == CONFIG_VALIDATOR: + return False + + LOGGER.info("updating validator", extra={"collection": name, "dry_run": dry_run}) + if not dry_run: + database.command("collMod", name, validator=CONFIG_VALIDATOR) + + return True + + +def ensure_alert_indexes(collection: Collection, dry_run: bool = False) -> list[str]: + """Create the consumer's alert indexes; return the index names.""" + + names = [] + for keys, options in ALERT_INDEXES: + if dry_run: + names.append("+".join(f"{field}_{direction}" for field, direction in keys)) + continue + + names.append(collection.create_index(keys, **options)) + + LOGGER.info( + "alert indexes ensured", + extra={"collection": collection.name, "indexes": names, "dry_run": dry_run}, + ) + return names + + +def load_seed_configs(seed_dir: Path) -> dict[str, dict]: + """Load .json seed docs keyed by monitor name. + + Raises: + ValueError: When a seed file lacks categories or channels. + """ + + docs = {} + for path in sorted(seed_dir.glob("*.json")): + with path.open() as seed_file: + doc = json.load(seed_file) + + missing = SEED_KEYS - set(doc) + if missing: + raise ValueError(f"{path}: missing {sorted(missing)}") + + bad_channels = { + bucket: value + for bucket, value in doc["channels"].items() + if not (isinstance(value, str) and value.isdigit()) + } + if bad_channels: + raise ValueError( + f"{path}: channels must be single id strings {bad_channels}" + ) + + expected = CATEGORY_SHAPES.get(path.stem) + if expected is not None and not isinstance(doc["categories"], expected): + raise ValueError( + f"{path}: categories must be {expected.__name__} for {path.stem}" + ) + + docs[path.stem] = {key: doc[key] for key in SEED_KEYS} + + return docs + + +def seed_configs( + collection: Collection, docs: dict[str, dict], dry_run: bool = False +) -> list[str]: + """Insert absent config docs via $setOnInsert; return the ids inserted.""" + + inserted = [] + for monitor_name, doc in docs.items(): + if dry_run: + if collection.find_one({"_id": monitor_name}, {"_id": 1}) is None: + inserted.append(monitor_name) + continue + + result = collection.update_one( + {"_id": monitor_name}, {"$setOnInsert": doc}, upsert=True + ) + if result.upserted_id is not None: + inserted.append(monitor_name) + + LOGGER.info( + "configs seeded", + extra={ + "collection": collection.name, + "inserted": inserted, + "skipped": sorted(set(docs) - set(inserted)), + "dry_run": dry_run, + }, + ) + return inserted + + +def load_control_seed(path: Path) -> dict[str, dict]: + """Load control docs keyed by _id from one JSON file.""" + + with path.open() as seed_file: + docs = json.load(seed_file) + + for doc_id, doc in docs.items(): + if not isinstance(doc, dict): + raise ValueError(f"{path}: {doc_id} must be an object") + + return docs + + +def seed_control( + collection: Collection, docs: dict[str, dict], dry_run: bool = False +) -> list[str]: + """Insert absent control docs via $setOnInsert; return the ids inserted.""" + + inserted = [] + for doc_id, doc in docs.items(): + if dry_run: + if collection.find_one({"_id": doc_id}, {"_id": 1}) is None: + inserted.append(doc_id) + continue + + result = collection.update_one( + {"_id": doc_id}, {"$setOnInsert": doc}, upsert=True + ) + if result.upserted_id is not None: + inserted.append(doc_id) + + LOGGER.info( + "control seeded", + extra={ + "collection": collection.name, + "inserted": inserted, + "skipped": sorted(set(docs) - set(inserted)), + "dry_run": dry_run, + }, + ) + return inserted + + +def bootstrap( + database: Database, seed_root: Path = SEED_ROOT, dry_run: bool = False +) -> dict: + """Run every step for both modes and return a summary of what changed.""" + + summary = {"created": [], "validators": [], "indexes": {}, "seeded": {}} + for suffix, seed_mode in MODES.items(): + configs_name = f"configs{suffix}" + if ensure_collection( + database, configs_name, dry_run, validator=CONFIG_VALIDATOR + ): + summary["created"].append(configs_name) + elif ensure_config_validator(database, configs_name, dry_run): + summary["validators"].append(configs_name) + + for base in ("control", "alerts", *MONITORS): + name = f"{base}{suffix}" + if ensure_collection(database, name, dry_run): + summary["created"].append(name) + + alerts_name = f"alerts{suffix}" + summary["indexes"][alerts_name] = ensure_alert_indexes( + database[alerts_name], dry_run + ) + docs = load_seed_configs(seed_root / seed_mode) + summary["seeded"][configs_name] = seed_configs( + database[configs_name], docs, dry_run + ) + control_name = f"control{suffix}" + summary["seeded"][control_name] = seed_control( + database[control_name], + load_control_seed(seed_root / "control.json"), + dry_run, + ) + + return summary + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--url", default=MONGO_URL, help="Mongo connection url") + parser.add_argument("--database", default=MONGO_DATABASE_NAME) + parser.add_argument( + "--dry-run", action="store_true", help="report actions without writing" + ) + args = parser.parse_args(argv) + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + + client = MongoClient(args.url, serverSelectionTimeoutMS=30_000) + LOGGER.info("connected", extra={"server": client.server_info()["version"]}) + summary = bootstrap(client[args.database], dry_run=args.dry_run) + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/db_seed/control.json b/scripts/db_seed/control.json new file mode 100644 index 0000000..fa29de5 --- /dev/null +++ b/scripts/db_seed/control.json @@ -0,0 +1,53 @@ +{ + "monitors": { + "names": [ + "bestbuy", + "woot", + "lg", + "hp", + "samsung", + "northerntool", + "corsair", + "apple", + "samsung_shop" + ], + "settings": { + "bestbuy": { + "min_percentage": 50, + "ping_threshold": null + }, + "woot": { + "min_percentage": 50, + "ping_threshold": 800 + }, + "lg": { + "min_percentage": 40, + "ping_threshold": null + }, + "hp": { + "min_percentage": 50, + "ping_threshold": null + }, + "samsung": { + "min_percentage": 40, + "ping_threshold": null + }, + "northerntool": { + "min_percentage": 40, + "ping_threshold": null + }, + "corsair": { + "min_percentage": 40, + "ping_threshold": null + }, + "apple": { + "min_percentage": 40, + "ping_threshold": null + }, + "samsung_shop": { + "min_percentage": 40, + "ping_threshold": null + } + } + } +} diff --git a/scripts/db_seed/production/apple.json b/scripts/db_seed/production/apple.json new file mode 100644 index 0000000..8807ad6 --- /dev/null +++ b/scripts/db_seed/production/apple.json @@ -0,0 +1,74 @@ +{ + "categories": { + "home/shop_iphone/family/iphone_17_pro": "html", + "home/shop_iphone/family/iphone_air": "html", + "home/shop_iphone/family/iphone_17": "html", + "home/shop_iphone/family/iphone_17e": "html", + "home/shop_iphone/family/iphone_16": "html", + "home/shop_mac/family/macbook_neo": "html", + "home/shop_mac/family/macbook_air_13_inch": "html", + "home/shop_mac/family/macbook_air_15_inch": "html", + "home/shop_mac/family/macbook_pro_14_inch": "html", + "home/shop_mac/family/macbook_pro_16_inch": "html", + "home/shop_mac/family/imac": "html", + "home/shop_mac/family/mac_mini": "html", + "home/shop_mac/family/mac_studio": "html", + "home/shop_mac/family/studio_display": "html", + "home/shop_mac/family/studio_display_xdr": "html", + "/shop/accessories/all/accessibility": "grid", + "/shop/accessories/all/cases-protection": "grid", + "home/shop_ipad/family/ipad_pro": "html", + "home/shop_ipad/family/ipad_air": "html", + "home/shop_ipad/family/ipad": "html", + "home/shop_ipad/family/ipad_mini": "html", + "home/shop_watch/family/apple_watch": "html", + "home/shop_watch/family/apple_watch_se": "html", + "home/shop_watch/family/apple_watch_ultra": "html", + "home/shop_watch/family/apple_watch_hermes": "html", + "home/shop_watch/family/apple_watch_hermes_ultra": "html", + "home/shop_vision/family/apple_vision_pro": "html", + "home/shop_airpods/family/airpods_4": "html", + "home/shop_airpods/family/airpods_pro_3": "html", + "home/shop_airpods/family/airpods_max_2": "html", + "home/shop_homepod/family/homepod": "html", + "home/shop_homepod/family/homepod_mini": "html", + "home/shop_tv/family/apple_tv_4k": "html", + "/shop/accessories/all/chargers-adapters": "grid", + "/shop/accessories/all/travel-essentials": "grid", + "/shop/accessories/all/seasonal-colors": "grid", + "/shop/airtag/accessories/airtag-essentials": "grid", + "/shop/ipad/accessories/content-creation": "grid", + "/shop/mac/accessories/content-creation": "grid", + "/shop/vision/accessories/apple-vision-pro-essentials": "grid", + "/shop/accessories/all/college-essentials": "grid", + "/shop/accessories/all/content-creation": "grid", + "/shop/accessories/all/drives-storage": "grid", + "/shop/accessories/all/gaming": "grid", + "/shop/accessories/all/headphones-speakers": "grid", + "/shop/accessories/all/health-fitness": "grid", + "/shop/accessories/all/homekit": "grid", + "/shop/accessories/all/made-by-apple": "grid", + "/shop/accessories/all/mice-keyboards": "grid", + "/shop/accessories/all/new-arrivals": "grid", + "/shop/accessories/all/office": "grid", + "/shop/accessories/all/software": "grid", + "/shop/refurbished/mac": "grid", + "/shop/refurbished/ipad": "grid", + "/shop/refurbished/iphone": "grid", + "/shop/refurbished/watch": "grid", + "/shop/refurbished/appletv": "grid", + "/shop/refurbished/accessories": "grid", + "/shop/refurbished/airpods": "grid", + "/shop/refurbished/homepod": "grid" + }, + "channels": { + "40-49": "1542905433235267694", + "50-60": "1542905485722779738", + "60-70": "1542905516047732947", + "70-80": "1542905555402629151", + "80-90": "1542905590832037932", + "90-100": "1542905643671756820", + "soldOut": "1542905707781816340", + "newListing": "1542905774601281677" + } +} diff --git a/scripts/db_seed/production/bestbuy.json b/scripts/db_seed/production/bestbuy.json new file mode 100644 index 0000000..f8e30f0 --- /dev/null +++ b/scripts/db_seed/production/bestbuy.json @@ -0,0 +1,144 @@ +{ + "categories": { + "pcmcat1764872919906": "categoryid$pcmcat1764872919906", + "abcat0715000": "categoryid$abcat0715000", + "pcmcat1662993575810": "categoryid$pcmcat1662993575810", + "pcmcat287600050002": "categoryid$pcmcat287600050002", + "pcmcat287600050003": "categoryid$pcmcat287600050003", + "pcmcat1763397583811": "pcmcat1763397583811_categoryid$pcmcat241600050001", + "abcat0208000": "categoryid$abcat0208000", + "pcmcat1591132221892": "pcmcat1591132221892_categoryid$abcat0500000", + "pcmcat304600050011": "categoryid$pcmcat304600050011", + "abcat0507000": "categoryid$abcat0507000", + "pcmcat243800050005": "categoryid$pcmcat243800050005", + "abcat0401000": "categoryid$abcat0401000", + "abcat0410000": "categoryid$abcat0410000", + "pcmcat369900050001": "categoryid$pcmcat369900050001", + "pcmcat1762875326350": "pcmcat1762875326350_categoryid$abcat0915000", + "abcat0915001": "categoryid$abcat0915001", + "pcmcat1762453391793": "pcmcat1762453391793_categoryid$abcat0915004", + "pcmcat1717008056314": "categoryid$pcmcat1717008056314", + "pcmcat244100050013": "categoryid$pcmcat244100050013", + "pcmcat333300050027": "categoryid$pcmcat333300050027", + "abcat0901004": "categoryid$abcat0901004", + "abcat0901005": "categoryid$abcat0901005", + "abcat0901006": "categoryid$abcat0901006", + "abcat0901001": "categoryid$abcat0901001", + "abcat0901003": "categoryid$abcat0901003", + "pcmcat313100050031": "categoryid$pcmcat313100050031", + "pcmcat144700050004": "categoryid$pcmcat144700050004", + "pcmcat308100050020": "categoryid$pcmcat308100050020", + "pcmcat748301598672": "categoryid$pcmcat748301598672", + "pcmcat1477674238305": "categoryid$pcmcat1477674238305", + "pcmcat1602773246733": "categoryid$abcat0503000", + "pcmcat345400050002": "categoryid$pcmcat345400050002", + "pcmcat308100050014": "categoryid$pcmcat308100050014", + "pcmcat308100050018": "categoryid$pcmcat308100050018", + "pcmcat308100050017": "categoryid$pcmcat308100050017", + "pcmcat1515445788580": "categoryid$pcmcat1515445788580", + "pcmcat254000050007": "categoryid$pcmcat254000050007", + "pcmcat1586207767722": "categoryid$pcmcat1586207767722", + "pcmcat179200050010": "categoryid$pcmcat179200050010", + "pcmcat1651764083616": "categoryid$pcmcat1651764083616", + "pcmcat1615397403616": "categoryid$pcmcat1615397403616", + "pcmcat270300050004": "categoryid$pcmcat270300050004", + "pcmcat1651764174444": "categoryid$pcmcat1651764174444", + "pcmcat179200050004": "categoryid$pcmcat179200050004", + "pcmcat270300050005": "categoryid$pcmcat270300050005", + "pcmcat179200050007": "categoryid$pcmcat179200050007", + "pcmcat1631303385635": "categoryid$pcmcat1631303385635", + "pcmcat179200050008": "categoryid$pcmcat179200050008", + "pcmcat1597939198513": "categoryid$pcmcat1597939198513", + "pcmcat179200050003": "categoryid$pcmcat179200050003", + "pcmcat1491416706156": "categoryid$pcmcat1491416706156", + "pcmcat1592600995496": "categoryid$pcmcat1592600995496", + "pcmcat334800050011": "categoryid$pcmcat334800050011", + "abcat0205006": "categoryid$abcat0205006", + "pcmcat273800050032": "categoryid$pcmcat273800050032", + "pcmcat1545417879756": "pcmcat1545417879756_categoryid$pcmcat250300050003", + "pcmcat250300050008": "categoryid$pcmcat250300050008", + "pcmcat265100050011": "categoryid$pcmcat265100050011", + "abcat0302005": "categoryid$abcat0302005", + "abcat0302012": "categoryid$abcat0302012", + "abcat0302034": "categoryid$abcat0302034", + "abcat0302001": "categoryid$abcat0302001", + "abcat0302011": "categoryid$abcat0302011", + "pcmcat326300050013": "categoryid$pcmcat326300050013", + "abcat0302019": "categoryid$abcat0302019", + "pcmcat1493671011172": "categoryid$pcmcat1493671011172", + "pcmcat165900050023": "categoryid$pcmcat165900050023", + "pcmcat330300050001": "categoryid$pcmcat330300050001", + "abcat0305000": "categoryid$abcat0305000", + "pcmcat1762288382250": "pcmcat1762288382250_categoryid$abcat0301000", + "pcmcat748301308133": "categoryid$pcmcat748301308133", + "pcmcat1496180344205": "categoryid$pcmcat1496180344205", + "pcmcat1754489958914": "categoryid$pcmcat1754489958914", + "pcmcat1754490913800": "categoryid$pcmcat1754490913800", + "pcmcat156300050006": "categoryid$pcmcat156300050006", + "pcmcat311200050005": "categoryid$pcmcat311200050005", + "pcmcat384300050013": "categoryid$pcmcat384300050013", + "pcmcat214700050000": "categoryid$pcmcat214700050000", + "pcmcat171900050026": "categoryid$pcmcat171900050026", + "pcmcat214700050002": "categoryid$pcmcat214700050002", + "pcmcat326000050010": "categoryid$pcmcat326000050010", + "pcmcat326000050008": "categoryid$pcmcat326000050008", + "wallets-grips-straps-mounts-travel-accessories": "wallets-grips-straps-mounts-travel-accessories", + "pcmcat1631123340977": "pcmcat1631123340977_categoryid$pcmcat748301221690", + "magsafe-multiple-station-chargers": "magsafe-multiple-station-chargers", + "pcmcat326000050006": "categoryid$pcmcat326000050006", + "pcmcat326300050011": "categoryid$pcmcat326300050011", + "pcmcat367400050001": "categoryid$pcmcat367400050001", + "pcmcat1484596902850": "categoryid$pcmcat1484596902850", + "abcat0916004": "categoryid$abcat0916004", + "abcat0905001": "categoryid$abcat0905001", + "abcat0916001": "categoryid$abcat0916001", + "pcmcat1684266033737": "categoryid$pcmcat1684266033737", + "pcmcat1707935982082": "pcmcat1707935982082_categoryid$abcat0905000", + "pcmcat1710866322171": "categoryid$pcmcat1710866322171", + "pcmcat196400050016": "pcmcat196400050016_categoryid$abcat0904003", + "pcmcat196400050015": "pcmcat196400050015_categoryid$abcat0904003", + "pcmcat1660745100989": "pcmcat1660745100989_categoryid$abcat0904003", + "pcmcat196400050017": "pcmcat196400050017_categoryid$abcat0904003", + "pcmcat1710865851607": "categoryid$pcmcat1710865851607", + "pcmcat1680897792908": "pcmcat1680897792908_categoryid$abcat0904003", + "abcat0904003": "categoryid$abcat0904003", + "pcmcat1564413660184": "categoryid$pcmcat1564413660184", + "pcmcat272800050011": "categoryid$pcmcat272800050011", + "pcmcat1597762271403": "categoryid$pcmcat1597762271403", + "pcmcat249700050007": "categoryid$pcmcat249700050007", + "pcmcat748300655035": "categoryid$pcmcat748300655035", + "abcat0106001": "categoryid$abcat0106001", + "pcmcat748300652083": "categoryid$pcmcat748300652083", + "pcmcat748300654236": "categoryid$pcmcat748300654236", + "pcmcat272800050000": "categoryid$pcmcat272800050000", + "pcmcat312500050006": "categoryid$pcmcat312500050006", + "pcmcat138200050001": "categoryid$pcmcat138200050001", + "abcat0809004": "categoryid$abcat0809004", + "abcat0809002": "categoryid$abcat0809002", + "abcat0106021": "categoryid$abcat0106021", + "pcmcat1760724127414": "categoryid$pcmcat1760724127414", + "pcmcat1688401595784": "categoryid$pcmcat1688401595784", + "pcmcat219100050010": "categoryid$pcmcat219100050010", + "pcmcat748300353430": "categoryid$pcmcat748300353430", + "pcmcat286300050020": "pcmcat286300050020", + "pcmcat1563557714651": "categoryid$pcmcat1563557714651", + "pcmcat1578693425373": "categoryid$pcmcat1578693425373", + "pcmcat367400050002": "categoryid$pcmcat367400050002", + "pcmcat1547762619778": "categoryid$pcmcat1547762619778", + "abcat0912011": "categoryid$abcat0912011", + "abcat0912022": "categoryid$abcat0912022", + "abcat0912013": "categoryid$abcat0912013", + "abcat0912012": "categoryid$abcat0912012" + }, + "channels": { + "event": "1452471895047012535", + "memberPick": "1452811066823278696", + "memberDeal": "1452811121776922794", + "soldOut": "1451799526540050512", + "60-70": "1451747974156914799", + "50-60": "1451747992297013318", + "70-80": "1451747949313921094", + "80-90": "1451747927923101737", + "90-100": "1451747737501564938" + } +} diff --git a/scripts/db_seed/production/corsair.json b/scripts/db_seed/production/corsair.json new file mode 100644 index 0000000..4122c07 --- /dev/null +++ b/scripts/db_seed/production/corsair.json @@ -0,0 +1,15 @@ +{ + "categories": [ + "pc-cases", + "wireless-keyboards" + ], + "channels": { + "50-60": "1525179737700306944", + "60-70": "1525179526898913280", + "70-80": "1525179285382369412", + "80-90": "1525180350035394590", + "90-100": "1525178672510664894", + "40-49": "1537283810452832357", + "newListing": "1537179654849568950" + } +} diff --git a/scripts/db_seed/production/hp.json b/scripts/db_seed/production/hp.json new file mode 100644 index 0000000..32ee626 --- /dev/null +++ b/scripts/db_seed/production/hp.json @@ -0,0 +1,35 @@ +{ + "categories": [ + "vwa/business-solutions/bizcat=Workstation", + "vwa/business-solutions/bizcat=Laptop", + "vwa/business-solutions/form=Mobile-workstation", + "vwa/laptops/usage=Gaming", + "vwa/accessories-88342--1/newprod=Yes", + "vwa/business-solutions/bizcat=Thin-Client", + "vwa/business-solutions/bizcat=Desktop", + "vwa/desktops/usage=Gaming", + "vwa/accessories-88342--1/subcat=Audio-Video-Multimedia&useaudvid=Microphones,Webcams-and-cameras", + "vwa/printers/segm=Business,Enterprise,Small-medium-business", + "vwa/printers/segm=Home,Home-and-home-office,Small-Office", + "vwa/laptops/segm=Home", + "vwa/accessories-88342--1/subcat=Headsets", + "vwa/accessories-88342--1/subcat=Bags-Cases", + "vwa/accessories-88342--1/subcat=Docking-Stations-Hubs", + "vwa/gaming-3074457345617980168--1/gamingcat=Accessories,Monitors", + "vwa/ink--toner---paper/inktyp=High-Yield-Ink-Cartridges", + "vwa/accessories-88342--1/typemicekbd=Stylus-Pens&subcat=Mice-Keyboards", + "vwa/accessories-88342--1/subcat=Home-Office-Connectivity", + "vwa/accessories-88342--1/subcat=PC-Components", + "vwa/desktops/segm=Home", + "vwa/accessories-88342--1/brandaccessories=OMEN", + "vwa/accessories-88342--1/subcat=Microphones-Webcams" + ], + "channels": { + "50-60": "1440486150723604551", + "60-70": "1440486181329703013", + "70-80": "1440486200929419335", + "80-90": "1440486221221724160", + "90-100": "1440486248895611103", + "soldOut": "1440486290490523700" + } +} diff --git a/scripts/db_seed/production/lg.json b/scripts/db_seed/production/lg.json new file mode 100644 index 0000000..0146a67 --- /dev/null +++ b/scripts/db_seed/production/lg.json @@ -0,0 +1,30 @@ +{ + "categories": [ + "window_air_conditioners", + "portable_air_conditioners", + "tvs", + "4k_uhd_tvs", + "monitors", + "speakers", + "microwaves", + "refrigerators", + "washers", + "dryers", + "ovens", + "dishwashers", + "lg_gram_pro_2_in_1_laptops", + "sound_bars", + "cooking_appliances", + "tv_and_home_theater_accessories", + "computing_accessories" + ], + "channels": { + "soldOut": "1497368064126095671", + "40-49": "1495254588520005683", + "50-60": "1495254526784049242", + "60-70": "1495254479891595304", + "70-80": "1497367798882504945", + "80-90": "1495254455644455012", + "90-100": "1495254418533388399" + } +} diff --git a/scripts/db_seed/production/northerntool.json b/scripts/db_seed/production/northerntool.json new file mode 100644 index 0000000..c9645ec --- /dev/null +++ b/scripts/db_seed/production/northerntool.json @@ -0,0 +1,46 @@ +{ + "categories": [ + "1733", + "1381", + "6970", + "1354", + "1385", + "1382", + "1357", + "1352", + "1378", + "1387", + "1380", + "1362", + "1353", + "1377", + "1355", + "1369", + "1384", + "1363", + "1376", + "1388", + "1383", + "1367", + "1371", + "1366", + "1360", + "1356", + "1368", + "1389", + "1358", + "1365", + "1386", + "1361", + "1364" + ], + "channels": { + "soldOut": "1510403235632906262", + "40-49": "1510403793576136814", + "50-60": "1510403590504841326", + "60-70": "1510266064305651752", + "70-80": "1510266139018531048", + "80-90": "1513214343620329572", + "90-100": "1510403299596173374" + } +} diff --git a/scripts/db_seed/production/samsung.json b/scripts/db_seed/production/samsung.json new file mode 100644 index 0000000..c15a4f8 --- /dev/null +++ b/scripts/db_seed/production/samsung.json @@ -0,0 +1,44 @@ +{ + "categories": [ + "01010000", + "01020000", + "01030000", + "01040000", + "01050000", + "01090000", + "01120000", + "02010000", + "03010000", + "04010000", + "04020000", + "04030000", + "04050000", + "04060000", + "04080000", + "05010000", + "05020000", + "05030000", + "07010000", + "07020000", + "07030000", + "07040000", + "08010000", + "08030000", + "08070000", + "08080000", + "08090000", + "08110000", + "08120000", + "09010000", + "11010000" + ], + "channels": { + "40-49": "1470224395514417276", + "50-60": "1467755134514761728", + "60-70": "1467755157805596692", + "70-80": "1467755178731114528", + "80-90": "1467755178731114528", + "90-100": "1467755197148172484", + "soldOut": "1467757497023729755" + } +} diff --git a/scripts/db_seed/production/samsung_shop.json b/scripts/db_seed/production/samsung_shop.json new file mode 100644 index 0000000..4a8613b --- /dev/null +++ b/scripts/db_seed/production/samsung_shop.json @@ -0,0 +1,36 @@ +{ + "categories": [ + "01010000", + "01020000", + "01030000", + "01040000", + "01050000", + "01090000", + "01120000", + "02010000", + "03010000", + "04010000", + "04020000", + "04030000", + "04050000", + "04060000", + "04080000", + "05010000", + "05020000", + "05030000", + "07010000", + "07020000", + "07030000", + "07040000", + "08010000", + "08030000", + "08070000", + "08080000", + "08090000", + "08110000", + "08120000", + "09010000", + "11010000" + ], + "channels": {} +} diff --git a/scripts/db_seed/production/woot.json b/scripts/db_seed/production/woot.json new file mode 100644 index 0000000..d435034 --- /dev/null +++ b/scripts/db_seed/production/woot.json @@ -0,0 +1,21 @@ +{ + "categories": [ + "tools", + "sport", + "home", + "grocery", + "pc", + "tech" + ], + "channels": { + "50-60": "1428512471723016283", + "60-70": "1428512958081798254", + "70-80": "1428512446678827099", + "80-90": "1428512446678827099", + "90-100": "1428512424176259124", + "event": "1428512391775387789", + "appExclusive": "1428512332446695434", + "soldOut": "1428512362763128933", + "40-49": "1479229158411468890" + } +} diff --git a/scripts/db_seed/testing/apple.json b/scripts/db_seed/testing/apple.json new file mode 100644 index 0000000..cf6c737 --- /dev/null +++ b/scripts/db_seed/testing/apple.json @@ -0,0 +1,65 @@ +{ + "categories": { + "home/shop_iphone/family/iphone_17_pro": "html", + "home/shop_iphone/family/iphone_air": "html", + "home/shop_iphone/family/iphone_17": "html", + "home/shop_iphone/family/iphone_17e": "html", + "home/shop_iphone/family/iphone_16": "html", + "home/shop_mac/family/macbook_neo": "html", + "home/shop_mac/family/macbook_air_13_inch": "html", + "home/shop_mac/family/macbook_air_15_inch": "html", + "home/shop_mac/family/macbook_pro_14_inch": "html", + "home/shop_mac/family/macbook_pro_16_inch": "html", + "home/shop_mac/family/imac": "html", + "home/shop_mac/family/mac_mini": "html", + "home/shop_mac/family/mac_studio": "html", + "home/shop_mac/family/studio_display": "html", + "home/shop_mac/family/studio_display_xdr": "html", + "/shop/accessories/all/accessibility": "grid", + "/shop/accessories/all/cases-protection": "grid", + "home/shop_ipad/family/ipad_pro": "html", + "home/shop_ipad/family/ipad_air": "html", + "home/shop_ipad/family/ipad": "html", + "home/shop_ipad/family/ipad_mini": "html", + "home/shop_watch/family/apple_watch": "html", + "home/shop_watch/family/apple_watch_se": "html", + "home/shop_watch/family/apple_watch_ultra": "html", + "home/shop_watch/family/apple_watch_hermes": "html", + "home/shop_watch/family/apple_watch_hermes_ultra": "html", + "home/shop_vision/family/apple_vision_pro": "html", + "home/shop_airpods/family/airpods_4": "html", + "home/shop_airpods/family/airpods_pro_3": "html", + "home/shop_airpods/family/airpods_max_2": "html", + "home/shop_homepod/family/homepod": "html", + "home/shop_homepod/family/homepod_mini": "html", + "home/shop_tv/family/apple_tv_4k": "html", + "/shop/accessories/all/chargers-adapters": "grid", + "/shop/accessories/all/travel-essentials": "grid", + "/shop/accessories/all/seasonal-colors": "grid", + "/shop/airtag/accessories/airtag-essentials": "grid", + "/shop/ipad/accessories/content-creation": "grid", + "/shop/mac/accessories/content-creation": "grid", + "/shop/vision/accessories/apple-vision-pro-essentials": "grid", + "/shop/accessories/all/college-essentials": "grid", + "/shop/accessories/all/content-creation": "grid", + "/shop/accessories/all/drives-storage": "grid", + "/shop/accessories/all/gaming": "grid", + "/shop/accessories/all/headphones-speakers": "grid", + "/shop/accessories/all/health-fitness": "grid", + "/shop/accessories/all/homekit": "grid", + "/shop/accessories/all/made-by-apple": "grid", + "/shop/accessories/all/mice-keyboards": "grid", + "/shop/accessories/all/new-arrivals": "grid", + "/shop/accessories/all/office": "grid", + "/shop/accessories/all/software": "grid", + "/shop/refurbished/mac": "grid", + "/shop/refurbished/ipad": "grid", + "/shop/refurbished/iphone": "grid", + "/shop/refurbished/watch": "grid", + "/shop/refurbished/appletv": "grid", + "/shop/refurbished/accessories": "grid", + "/shop/refurbished/airpods": "grid", + "/shop/refurbished/homepod": "grid" + }, + "channels": {} +} diff --git a/scripts/db_seed/testing/bestbuy.json b/scripts/db_seed/testing/bestbuy.json new file mode 100644 index 0000000..c285551 --- /dev/null +++ b/scripts/db_seed/testing/bestbuy.json @@ -0,0 +1,144 @@ +{ + "categories": { + "pcmcat1764872919906": "categoryid$pcmcat1764872919906", + "abcat0715000": "categoryid$abcat0715000", + "pcmcat1662993575810": "categoryid$pcmcat1662993575810", + "pcmcat287600050002": "categoryid$pcmcat287600050002", + "pcmcat287600050003": "categoryid$pcmcat287600050003", + "pcmcat1763397583811": "pcmcat1763397583811_categoryid$pcmcat241600050001", + "abcat0208000": "categoryid$abcat0208000", + "pcmcat1591132221892": "pcmcat1591132221892_categoryid$abcat0500000", + "pcmcat304600050011": "categoryid$pcmcat304600050011", + "abcat0507000": "categoryid$abcat0507000", + "pcmcat243800050005": "categoryid$pcmcat243800050005", + "abcat0401000": "categoryid$abcat0401000", + "abcat0410000": "categoryid$abcat0410000", + "pcmcat369900050001": "categoryid$pcmcat369900050001", + "pcmcat1762875326350": "pcmcat1762875326350_categoryid$abcat0915000", + "abcat0915001": "categoryid$abcat0915001", + "pcmcat1762453391793": "pcmcat1762453391793_categoryid$abcat0915004", + "pcmcat1717008056314": "categoryid$pcmcat1717008056314", + "pcmcat244100050013": "categoryid$pcmcat244100050013", + "pcmcat333300050027": "categoryid$pcmcat333300050027", + "abcat0901004": "categoryid$abcat0901004", + "abcat0901005": "categoryid$abcat0901005", + "abcat0901006": "categoryid$abcat0901006", + "abcat0901001": "categoryid$abcat0901001", + "abcat0901003": "categoryid$abcat0901003", + "pcmcat313100050031": "categoryid$pcmcat313100050031", + "pcmcat144700050004": "categoryid$pcmcat144700050004", + "pcmcat308100050020": "categoryid$pcmcat308100050020", + "pcmcat748301598672": "categoryid$pcmcat748301598672", + "pcmcat1477674238305": "categoryid$pcmcat1477674238305", + "pcmcat1602773246733": "categoryid$abcat0503000", + "pcmcat345400050002": "categoryid$pcmcat345400050002", + "pcmcat308100050014": "categoryid$pcmcat308100050014", + "pcmcat308100050018": "categoryid$pcmcat308100050018", + "pcmcat308100050017": "categoryid$pcmcat308100050017", + "pcmcat1515445788580": "categoryid$pcmcat1515445788580", + "pcmcat254000050007": "categoryid$pcmcat254000050007", + "pcmcat1586207767722": "categoryid$pcmcat1586207767722", + "pcmcat179200050010": "categoryid$pcmcat179200050010", + "pcmcat1651764083616": "categoryid$pcmcat1651764083616", + "pcmcat1615397403616": "categoryid$pcmcat1615397403616", + "pcmcat270300050004": "categoryid$pcmcat270300050004", + "pcmcat1651764174444": "categoryid$pcmcat1651764174444", + "pcmcat179200050004": "categoryid$pcmcat179200050004", + "pcmcat270300050005": "categoryid$pcmcat270300050005", + "pcmcat179200050007": "categoryid$pcmcat179200050007", + "pcmcat1631303385635": "categoryid$pcmcat1631303385635", + "pcmcat179200050008": "categoryid$pcmcat179200050008", + "pcmcat1597939198513": "categoryid$pcmcat1597939198513", + "pcmcat179200050003": "categoryid$pcmcat179200050003", + "pcmcat1491416706156": "categoryid$pcmcat1491416706156", + "pcmcat1592600995496": "categoryid$pcmcat1592600995496", + "pcmcat334800050011": "categoryid$pcmcat334800050011", + "abcat0205006": "categoryid$abcat0205006", + "pcmcat273800050032": "categoryid$pcmcat273800050032", + "pcmcat1545417879756": "pcmcat1545417879756_categoryid$pcmcat250300050003", + "pcmcat250300050008": "categoryid$pcmcat250300050008", + "pcmcat265100050011": "categoryid$pcmcat265100050011", + "abcat0302005": "categoryid$abcat0302005", + "abcat0302012": "categoryid$abcat0302012", + "abcat0302034": "categoryid$abcat0302034", + "abcat0302001": "categoryid$abcat0302001", + "abcat0302011": "categoryid$abcat0302011", + "pcmcat326300050013": "categoryid$pcmcat326300050013", + "abcat0302019": "categoryid$abcat0302019", + "pcmcat1493671011172": "categoryid$pcmcat1493671011172", + "pcmcat165900050023": "categoryid$pcmcat165900050023", + "pcmcat330300050001": "categoryid$pcmcat330300050001", + "abcat0305000": "categoryid$abcat0305000", + "pcmcat1762288382250": "pcmcat1762288382250_categoryid$abcat0301000", + "pcmcat748301308133": "categoryid$pcmcat748301308133", + "pcmcat1496180344205": "categoryid$pcmcat1496180344205", + "pcmcat1754489958914": "categoryid$pcmcat1754489958914", + "pcmcat1754490913800": "categoryid$pcmcat1754490913800", + "pcmcat156300050006": "categoryid$pcmcat156300050006", + "pcmcat311200050005": "categoryid$pcmcat311200050005", + "pcmcat384300050013": "categoryid$pcmcat384300050013", + "pcmcat214700050000": "categoryid$pcmcat214700050000", + "pcmcat171900050026": "categoryid$pcmcat171900050026", + "pcmcat214700050002": "categoryid$pcmcat214700050002", + "pcmcat326000050010": "categoryid$pcmcat326000050010", + "pcmcat326000050008": "categoryid$pcmcat326000050008", + "wallets-grips-straps-mounts-travel-accessories": "wallets-grips-straps-mounts-travel-accessories", + "pcmcat1631123340977": "pcmcat1631123340977_categoryid$pcmcat748301221690", + "magsafe-multiple-station-chargers": "magsafe-multiple-station-chargers", + "pcmcat326000050006": "categoryid$pcmcat326000050006", + "pcmcat326300050011": "categoryid$pcmcat326300050011", + "pcmcat367400050001": "categoryid$pcmcat367400050001", + "pcmcat1484596902850": "categoryid$pcmcat1484596902850", + "abcat0916004": "categoryid$abcat0916004", + "abcat0905001": "categoryid$abcat0905001", + "abcat0916001": "categoryid$abcat0916001", + "pcmcat1684266033737": "categoryid$pcmcat1684266033737", + "pcmcat1707935982082": "pcmcat1707935982082_categoryid$abcat0905000", + "pcmcat1710866322171": "categoryid$pcmcat1710866322171", + "pcmcat196400050016": "pcmcat196400050016_categoryid$abcat0904003", + "pcmcat196400050015": "pcmcat196400050015_categoryid$abcat0904003", + "pcmcat1660745100989": "pcmcat1660745100989_categoryid$abcat0904003", + "pcmcat196400050017": "pcmcat196400050017_categoryid$abcat0904003", + "pcmcat1710865851607": "categoryid$pcmcat1710865851607", + "pcmcat1680897792908": "pcmcat1680897792908_categoryid$abcat0904003", + "abcat0904003": "categoryid$abcat0904003", + "pcmcat1564413660184": "categoryid$pcmcat1564413660184", + "pcmcat272800050011": "categoryid$pcmcat272800050011", + "pcmcat1597762271403": "categoryid$pcmcat1597762271403", + "pcmcat249700050007": "categoryid$pcmcat249700050007", + "pcmcat748300655035": "categoryid$pcmcat748300655035", + "abcat0106001": "categoryid$abcat0106001", + "pcmcat748300652083": "categoryid$pcmcat748300652083", + "pcmcat748300654236": "categoryid$pcmcat748300654236", + "pcmcat272800050000": "categoryid$pcmcat272800050000", + "pcmcat312500050006": "categoryid$pcmcat312500050006", + "pcmcat138200050001": "categoryid$pcmcat138200050001", + "abcat0809004": "categoryid$abcat0809004", + "abcat0809002": "categoryid$abcat0809002", + "abcat0106021": "categoryid$abcat0106021", + "pcmcat1760724127414": "categoryid$pcmcat1760724127414", + "pcmcat1688401595784": "categoryid$pcmcat1688401595784", + "pcmcat219100050010": "categoryid$pcmcat219100050010", + "pcmcat748300353430": "categoryid$pcmcat748300353430", + "pcmcat286300050020": "pcmcat286300050020", + "pcmcat1563557714651": "categoryid$pcmcat1563557714651", + "pcmcat1578693425373": "categoryid$pcmcat1578693425373", + "pcmcat367400050002": "categoryid$pcmcat367400050002", + "pcmcat1547762619778": "categoryid$pcmcat1547762619778", + "abcat0912011": "categoryid$abcat0912011", + "abcat0912022": "categoryid$abcat0912022", + "abcat0912013": "categoryid$abcat0912013", + "abcat0912012": "categoryid$abcat0912012" + }, + "channels": { + "event": "1524040238064865393", + "memberPick": "1524040239369420841", + "memberDeal": "1524040240246030336", + "soldOut": "1524040240879374397", + "60-70": "1524040241902518414", + "50-60": "1524040242632458412", + "70-80": "1524040243450482779", + "80-90": "1524040244406517811", + "90-100": "1524040245136592957" + } +} diff --git a/scripts/db_seed/testing/corsair.json b/scripts/db_seed/testing/corsair.json new file mode 100644 index 0000000..da87ecc --- /dev/null +++ b/scripts/db_seed/testing/corsair.json @@ -0,0 +1,22 @@ +{ + "categories": [ + "pc-cases", + "keyboards", + "gaming-headsets", + "mousepads", + "leverless-controllers", + "ambient-lights", + "gaming-gear-accessories", + "cpu-coolers", + "gaming-computers", + "icue-link-components", + "gaming-furniture" + ], + "channels": { + "50-60": "1525179737700306944", + "60-70": "1525179526898913280", + "70-80": "1525179285382369412", + "80-90": "1525180350035394590", + "90-100": "1525178672510664894" + } +} diff --git a/scripts/db_seed/testing/hp.json b/scripts/db_seed/testing/hp.json new file mode 100644 index 0000000..2ee3d2d --- /dev/null +++ b/scripts/db_seed/testing/hp.json @@ -0,0 +1,35 @@ +{ + "categories": [ + "vwa/business-solutions/bizcat=Workstation", + "vwa/business-solutions/bizcat=Laptop", + "vwa/business-solutions/form=Mobile-workstation", + "vwa/laptops/usage=Gaming", + "vwa/accessories-88342--1/newprod=Yes", + "vwa/business-solutions/bizcat=Thin-Client", + "vwa/business-solutions/bizcat=Desktop", + "vwa/desktops/usage=Gaming", + "vwa/accessories-88342--1/subcat=Audio-Video-Multimedia&useaudvid=Microphones,Webcams-and-cameras", + "vwa/printers/segm=Business,Enterprise,Small-medium-business", + "vwa/printers/segm=Home,Home-and-home-office,Small-Office", + "vwa/laptops/segm=Home", + "vwa/accessories-88342--1/subcat=Headsets", + "vwa/accessories-88342--1/subcat=Bags-Cases", + "vwa/accessories-88342--1/subcat=Docking-Stations-Hubs", + "vwa/gaming-3074457345617980168--1/gamingcat=Accessories,Monitors", + "vwa/ink--toner---paper/inktyp=High-Yield-Ink-Cartridges", + "vwa/accessories-88342--1/typemicekbd=Stylus-Pens&subcat=Mice-Keyboards", + "vwa/accessories-88342--1/subcat=Home-Office-Connectivity", + "vwa/accessories-88342--1/subcat=PC-Components", + "vwa/desktops/segm=Home", + "vwa/accessories-88342--1/brandaccessories=OMEN", + "vwa/accessories-88342--1/subcat=Microphones-Webcams" + ], + "channels": { + "soldOut": "1523153716566560859", + "60-70": "1523153305897926776", + "50-60": "1523153047579136010", + "70-80": "1523153377587105892", + "80-90": "1523153568054513765", + "90-100": "1523153638971801710" + } +} diff --git a/scripts/db_seed/testing/lg.json b/scripts/db_seed/testing/lg.json new file mode 100644 index 0000000..8366acf --- /dev/null +++ b/scripts/db_seed/testing/lg.json @@ -0,0 +1,29 @@ +{ + "categories": [ + "window_air_conditioners", + "portable_air_conditioners", + "tvs", + "4k_uhd_tvs", + "monitors", + "speakers", + "microwaves", + "refrigerators", + "washers", + "dryers", + "ovens", + "dishwashers", + "lg_gram_pro_2_in_1_laptops", + "sound_bars", + "cooking_appliances", + "tv_and_home_theater_accessories", + "computing_accessories" + ], + "channels": { + "soldOut": "1524233737431421028", + "60-70": "1524233738748428328", + "50-60": "1524233739503407174", + "70-80": "1524233740417630238", + "80-90": "1524233741185187950", + "90-100": "1524233742036635852" + } +} diff --git a/scripts/db_seed/testing/northerntool.json b/scripts/db_seed/testing/northerntool.json new file mode 100644 index 0000000..943d82f --- /dev/null +++ b/scripts/db_seed/testing/northerntool.json @@ -0,0 +1,46 @@ +{ + "categories": [ + "1733", + "1381", + "6970", + "1354", + "1385", + "1382", + "1357", + "1352", + "1378", + "1387", + "1380", + "1362", + "1353", + "1377", + "1355", + "1369", + "1384", + "1363", + "1376", + "1388", + "1383", + "1367", + "1371", + "1366", + "1360", + "1356", + "1368", + "1389", + "1358", + "1365", + "1386", + "1361", + "1364" + ], + "channels": { + "soldOut": "1524233828661727292", + "60-70": "1524233830301433926", + "50-60": "1524233832838987827", + "70-80": "1524233834655387738", + "80-90": "1524233836714791003", + "90-100": "1524233837846991072", + "40-49": "1524839581055324160" + } +} diff --git a/scripts/db_seed/testing/samsung.json b/scripts/db_seed/testing/samsung.json new file mode 100644 index 0000000..75f3a17 --- /dev/null +++ b/scripts/db_seed/testing/samsung.json @@ -0,0 +1,43 @@ +{ + "categories": [ + "01010000", + "01020000", + "01030000", + "01040000", + "01050000", + "01090000", + "01120000", + "02010000", + "03010000", + "04010000", + "04020000", + "04030000", + "04050000", + "04060000", + "04080000", + "05010000", + "05020000", + "05030000", + "07010000", + "07020000", + "07030000", + "07040000", + "08010000", + "08030000", + "08070000", + "08080000", + "08090000", + "08110000", + "08120000", + "09010000", + "11010000" + ], + "channels": { + "soldOut": "1524233898702278741", + "60-70": "1524233899897520272", + "50-60": "1524233900866539642", + "70-80": "1524233901961117816", + "80-90": "1524233902611234856", + "90-100": "1524233903945154794" + } +} diff --git a/scripts/db_seed/testing/samsung_shop.json b/scripts/db_seed/testing/samsung_shop.json new file mode 100644 index 0000000..4a8613b --- /dev/null +++ b/scripts/db_seed/testing/samsung_shop.json @@ -0,0 +1,36 @@ +{ + "categories": [ + "01010000", + "01020000", + "01030000", + "01040000", + "01050000", + "01090000", + "01120000", + "02010000", + "03010000", + "04010000", + "04020000", + "04030000", + "04050000", + "04060000", + "04080000", + "05010000", + "05020000", + "05030000", + "07010000", + "07020000", + "07030000", + "07040000", + "08010000", + "08030000", + "08070000", + "08080000", + "08090000", + "08110000", + "08120000", + "09010000", + "11010000" + ], + "channels": {} +} diff --git a/scripts/db_seed/testing/woot.json b/scripts/db_seed/testing/woot.json new file mode 100644 index 0000000..a80dfea --- /dev/null +++ b/scripts/db_seed/testing/woot.json @@ -0,0 +1,21 @@ +{ + "categories": [ + "tools", + "sport", + "home", + "grocery", + "pc", + "tech" + ], + "channels": { + "event": "1523804538702794892", + "appExclusive": "1523804539302449342", + "priceDrop800": "1523804540271333530", + "soldOut": "1523804541164978266", + "60-70": "1523804541802385484", + "50-60": "1523804542783717652", + "70-80": "1523804545342242838", + "80-90": "1523804546286223482", + "90-100": "1523804547003453574" + } +} diff --git a/src/monitor_utils/database.py b/src/monitor_utils/database.py index 95335fe..19157f7 100644 --- a/src/monitor_utils/database.py +++ b/src/monitor_utils/database.py @@ -398,6 +398,7 @@ def convert_product_to_mongo(product: Product) -> dict: "promoPercentage": metadata["promo_percentage"], "promoDollarOff": metadata["promo_dollar_off"], "promoMinSpend": metadata["promo_min_spend"], + "asin": metadata["asin"], "startDate": timing["start_date"], "endDate": timing["end_date"], "priceBeforeDrop": pricing["price_before_drop"], diff --git a/src/monitor_utils/models.py b/src/monitor_utils/models.py index 9c67563..3536445 100644 --- a/src/monitor_utils/models.py +++ b/src/monitor_utils/models.py @@ -124,6 +124,7 @@ class ProductMetadata: promo_dollar_off: int | None = None promo_min_spend: int | None = None new_listing: bool | None = None + asin: str | None = None @dataclass diff --git a/tests/test_bootstrap_db.py b/tests/test_bootstrap_db.py new file mode 100644 index 0000000..49c2772 --- /dev/null +++ b/tests/test_bootstrap_db.py @@ -0,0 +1,302 @@ +"""Unit tests for scripts/bootstrap_db.py with a fake Mongo at the leaf. + +The $jsonSchema validator's runtime behavior is server-side and is not +covered here; it is verified against the live cluster when the script runs. +""" + +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import bootstrap_db # noqa: E402 + + +class FakeCollection: + def __init__(self, name): + self.name = name + self.docs = {} + self.indexes = [] + + def find_one(self, query, projection=None): + return self.docs.get(query["_id"]) + + def update_one(self, query, update, upsert=False): + class Result: + upserted_id = None + + result = Result() + if query["_id"] not in self.docs and upsert: + self.docs[query["_id"]] = {"_id": query["_id"], **update["$setOnInsert"]} + result.upserted_id = query["_id"] + + return result + + def create_index(self, keys, **options): + self.indexes.append((keys, options)) + return "_".join(f"{field}_{direction}" for field, direction in keys) + + +class FakeDatabase: + def __init__(self, existing=()): + self.collections = {name: FakeCollection(name) for name in existing} + self.options = {name: {} for name in existing} + self.created = [] + self.commands = [] + + def list_collection_names(self): + return list(self.collections) + + def list_collections(self, filter=None): + names = [filter["name"]] if filter else list(self.collections) + return [ + {"name": n, "options": self.options[n]} + for n in names + if n in self.collections + ] + + def create_collection(self, name, **options): + self.collections[name] = FakeCollection(name) + self.options[name] = options + self.created.append((name, options)) + + def command(self, name, collection, **kwargs): + self.commands.append((name, collection, kwargs)) + self.options[collection].update(kwargs) + + def __getitem__(self, name): + return self.collections.setdefault(name, FakeCollection(name)) + + +@pytest.fixture +def seed_root(tmp_path): + for mode in ("production", "testing"): + (tmp_path / mode).mkdir() + (tmp_path / mode / "woot.json").write_text( + json.dumps({"categories": ["tech"], "channels": {"50-60": "1"}}) + ) + (tmp_path / "production" / "bestbuy.json").write_text( + json.dumps( + { + "categories": {"a": "categoryid$a"}, + "channels": {"60-70": "2"}, + "extra": "ignored", + } + ) + ) + (tmp_path / "control.json").write_text( + json.dumps({"monitors": {"names": ["woot"], "settings": {}}}) + ) + return tmp_path + + +def test_ensure_collection_creates_absent_with_options(): + db = FakeDatabase() + assert bootstrap_db.ensure_collection(db, "configs", validator={"x": 1}) is True + assert db.created == [("configs", {"validator": {"x": 1}})] + + +def test_ensure_collection_skips_existing(): + db = FakeDatabase(existing=["configs"]) + assert bootstrap_db.ensure_collection(db, "configs") is False + assert db.created == [] + + +def test_ensure_collection_dry_run_writes_nothing(): + db = FakeDatabase() + assert bootstrap_db.ensure_collection(db, "configs", dry_run=True) is True + assert db.created == [] + + +def test_ensure_config_validator_applies_collmod_once(): + db = FakeDatabase(existing=["configs"]) + assert bootstrap_db.ensure_config_validator(db, "configs") is True + assert db.commands == [ + ("collMod", "configs", {"validator": bootstrap_db.CONFIG_VALIDATOR}) + ] + assert bootstrap_db.ensure_config_validator(db, "configs") is False + assert len(db.commands) == 1 + + +def test_ensure_config_validator_dry_run_and_missing_collection(): + db = FakeDatabase(existing=["configs"]) + assert bootstrap_db.ensure_config_validator(db, "configs", dry_run=True) is True + assert db.commands == [] + assert bootstrap_db.ensure_config_validator(db, "nope") is False + + +def test_load_seed_configs_rejects_list_channels(tmp_path): + (tmp_path / "woot.json").write_text( + json.dumps({"categories": [], "channels": {"50-60": ["1", "2"]}}) + ) + with pytest.raises(ValueError, match="single id strings"): + bootstrap_db.load_seed_configs(tmp_path) + + +def test_alert_indexes_match_consumer_spec(): + alerts = FakeCollection("alerts") + bootstrap_db.ensure_alert_indexes(alerts) + assert alerts.indexes == [ + ([("claimed", 1), ("priority", -1), ("createdAt", 1)], {}), + ( + [("product_id", 1)], + {"unique": True, "partialFilterExpression": {"claimed": False}}, + ), + ] + + +def test_load_seed_configs_keys_by_stem_and_drops_extra_fields(seed_root): + docs = bootstrap_db.load_seed_configs(seed_root / "production") + assert set(docs) == {"bestbuy", "woot"} + assert docs["bestbuy"] == { + "categories": {"a": "categoryid$a"}, + "channels": {"60-70": "2"}, + } + + +def test_load_seed_configs_rejects_missing_keys(tmp_path): + (tmp_path / "bad.json").write_text(json.dumps({"categories": []})) + with pytest.raises(ValueError, match="channels"): + bootstrap_db.load_seed_configs(tmp_path) + + +def test_load_seed_configs_rejects_wrong_shape_for_monitor(tmp_path): + (tmp_path / "bestbuy.json").write_text( + json.dumps({"categories": ["abcat0"], "channels": {}}) + ) + with pytest.raises(ValueError, match="categories must be dict for bestbuy"): + bootstrap_db.load_seed_configs(tmp_path) + + (tmp_path / "bestbuy.json").unlink() + (tmp_path / "woot.json").write_text( + json.dumps({"categories": {"tech": "tech"}, "channels": {}}) + ) + with pytest.raises(ValueError, match="categories must be list for woot"): + bootstrap_db.load_seed_configs(tmp_path) + + +def test_load_seed_configs_empty_dir(tmp_path): + assert bootstrap_db.load_seed_configs(tmp_path) == {} + + +def test_seed_configs_inserts_absent_and_preserves_existing(): + configs = FakeCollection("configs") + configs.docs["woot"] = {"_id": "woot", "categories": ["edited"], "channels": {}} + docs = { + "woot": {"categories": ["tech"], "channels": {}}, + "hp": {"categories": [], "channels": {}}, + } + assert bootstrap_db.seed_configs(configs, docs) == ["hp"] + assert configs.docs["woot"]["categories"] == ["edited"] + assert configs.docs["hp"] == {"_id": "hp", "categories": [], "channels": {}} + + +def test_seed_configs_dry_run_reports_without_writing(): + configs = FakeCollection("configs") + docs = {"hp": {"categories": [], "channels": {}}} + assert bootstrap_db.seed_configs(configs, docs, dry_run=True) == ["hp"] + assert configs.docs == {} + + +def test_load_control_seed_keys_by_id(seed_root): + docs = bootstrap_db.load_control_seed(seed_root / "control.json") + assert docs == {"monitors": {"names": ["woot"], "settings": {}}} + + +def test_load_control_seed_rejects_non_object_doc(tmp_path): + path = tmp_path / "control.json" + path.write_text(json.dumps({"monitors": ["woot"]})) + with pytest.raises(ValueError, match="monitors"): + bootstrap_db.load_control_seed(path) + + +def test_seed_control_inserts_absent_and_preserves_existing(): + control = FakeCollection("control") + control.docs["monitors"] = {"_id": "monitors", "names": ["edited"]} + docs = {"monitors": {"names": ["woot"]}, "other": {"x": 1}} + assert bootstrap_db.seed_control(control, docs) == ["other"] + assert control.docs["monitors"]["names"] == ["edited"] + assert control.docs["other"] == {"_id": "other", "x": 1} + + +def test_seed_control_dry_run_reports_without_writing(): + control = FakeCollection("control") + assert bootstrap_db.seed_control(control, {"monitors": {}}, dry_run=True) == [ + "monitors" + ] + assert control.docs == {} + + +def test_bootstrap_builds_both_modes(seed_root): + db = FakeDatabase() + summary = bootstrap_db.bootstrap(db, seed_root=seed_root) + + created = dict(db.created) + for suffix in ("", "_testing"): + assert created[f"configs{suffix}"] == { + "validator": bootstrap_db.CONFIG_VALIDATOR + } + assert f"alerts{suffix}" in created + assert f"control{suffix}" in created + for monitor in bootstrap_db.MONITORS: + assert f"{monitor}{suffix}" in created + assert len(db[f"alerts{suffix}"].indexes) == 2 + + assert summary["seeded"]["configs"] == ["bestbuy", "woot"] + assert summary["seeded"]["configs_testing"] == ["woot"] + assert summary["seeded"]["control"] == ["monitors"] + assert summary["seeded"]["control_testing"] == ["monitors"] + assert db["control"].docs["monitors"]["names"] == ["woot"] + assert len(summary["created"]) == 2 * (3 + len(bootstrap_db.MONITORS)) + + +def test_bootstrap_rerun_is_noop(seed_root): + db = FakeDatabase() + bootstrap_db.bootstrap(db, seed_root=seed_root) + db.created.clear() + summary = bootstrap_db.bootstrap(db, seed_root=seed_root) + assert db.created == [] + assert db.commands == [] + assert summary["created"] == [] + assert summary["validators"] == [] + assert summary["seeded"] == { + "configs": [], + "configs_testing": [], + "control": [], + "control_testing": [], + } + + +def test_shipped_control_seed_matches_registry_shape(): + """monitors doc: names list and per-name {min_percentage, ping_threshold}""" + + docs = bootstrap_db.load_control_seed(bootstrap_db.CONTROL_SEED) + monitors = docs["monitors"] + assert set(monitors["settings"]) == set(monitors["names"]) + for name, settings in monitors["settings"].items(): + assert set(settings) == {"min_percentage", "ping_threshold"}, name + assert isinstance(settings["min_percentage"], int), name + assert 40 <= settings["min_percentage"] <= 100, name + assert settings["ping_threshold"] is None or isinstance( + settings["ping_threshold"], int + ), name + + +def test_shipped_seed_files_load_and_match_validator_shape(): + """every checked-in seed doc has the shape the validator and setup_config want""" + + for mode in ("production", "testing"): + docs = bootstrap_db.load_seed_configs(bootstrap_db.SEED_ROOT / mode) + assert docs, mode + for name, doc in docs.items(): + categories, channels = doc["categories"], doc["channels"] + assert isinstance(categories, bootstrap_db.CATEGORY_SHAPES[name]), name + if isinstance(categories, dict): + assert all(isinstance(v, str) for v in categories.values()), name + else: + assert all(isinstance(v, str) for v in categories), name + for value in channels.values(): + assert isinstance(value, str) and value.isdigit(), (name, value)