fixed where f strings were still used for logging
This commit is contained in:
tpv committed
1 parent
c13e0d3c77
commit
dcc4f58292
1 file changed
+53
-20
+53
-20
@@ -41,7 +41,10 @@ def init_monitor(
|
||||
logging_level=LOGGING_LEVEL,
|
||||
) -> tuple[list[str], AsyncIOMotorCollection, AioProxies]:
|
||||
setup_logging(
|
||||
name="logs/latest.log", history_name="apple", level=logging_level, output="json"
|
||||
name="logs/latest.log",
|
||||
history_name=monitor_name,
|
||||
level=logging_level,
|
||||
output="json",
|
||||
)
|
||||
set_logger_levels()
|
||||
category_names, _, collection = setup_config(
|
||||
@@ -111,7 +114,10 @@ def build_product_details(
|
||||
current_price = product["currentPrice"]
|
||||
original_price = product["originalPrice"]
|
||||
if not original_price:
|
||||
LOGGER.warning(f"{product_id} - Missing or zero original price, skipping")
|
||||
LOGGER.warning(
|
||||
"missing or zero original price, skipping",
|
||||
extra={"product_id": product_id},
|
||||
)
|
||||
return
|
||||
|
||||
percentage = int(((original_price - current_price) / original_price) * 100)
|
||||
@@ -179,14 +185,14 @@ async def execute_monitor_logic(
|
||||
else get_base_session(headers=headers, cookies=cookies, proxies=proxies)
|
||||
)
|
||||
async with session_ctx as session:
|
||||
LOGGER.info("Starting tasks...")
|
||||
LOGGER.info("starting tasks")
|
||||
if pre_task_func:
|
||||
try:
|
||||
await pre_task_func(session=session, **kwargs)
|
||||
except EXPECTED_TASK_ERRORS as error:
|
||||
LOGGER.warning("Failed to get products", exc_info=error)
|
||||
LOGGER.warning("failed to get products", exc_info=error)
|
||||
except Exception:
|
||||
LOGGER.exception("Unexpected error")
|
||||
LOGGER.exception("unexpected error")
|
||||
raise
|
||||
|
||||
if isinstance(category_identifiers, dict):
|
||||
@@ -206,19 +212,19 @@ async def execute_monitor_logic(
|
||||
for category_identifier, result in zip(category_keys, results):
|
||||
if isinstance(result, EXPECTED_TASK_ERRORS):
|
||||
LOGGER.warning(
|
||||
f"{category_identifier} - Failed to get products",
|
||||
"failed to get products",
|
||||
extra={"category": category_identifier},
|
||||
exc_info=result,
|
||||
)
|
||||
elif isinstance(result, Exception):
|
||||
LOGGER.error(
|
||||
f"{category_identifier} - Unexpected error",
|
||||
"unexpected error",
|
||||
extra={"category": category_identifier},
|
||||
exc_info=result,
|
||||
)
|
||||
raise result
|
||||
|
||||
LOGGER.info(
|
||||
"Sleeping 0.30 seconds before next tasks"
|
||||
) # Might add option to increase/decrease this delay
|
||||
LOGGER.info("sleeping before next tasks", extra={"delay": 0.30})
|
||||
await asyncio.sleep(0.30)
|
||||
|
||||
|
||||
@@ -234,7 +240,7 @@ async def execute_pagination(
|
||||
pagination_mode: str = "offset",
|
||||
**kwargs,
|
||||
):
|
||||
LOGGER.info(f"{category_identifier} - executing pagination")
|
||||
LOGGER.info("executing pagination", extra={"category": category_identifier})
|
||||
total_pages = (int(total_products) + products_per_page - 1) // products_per_page
|
||||
semaphore = asyncio.Semaphore(max_concurrent_tasks)
|
||||
stop_event = asyncio.Event()
|
||||
@@ -270,12 +276,16 @@ async def execute_pagination(
|
||||
|
||||
if failed_pages and not stop_event.is_set():
|
||||
LOGGER.error(
|
||||
f"{category_identifier} - failed to get products after retries: "
|
||||
f"{pagination_label}s {failed_pages}"
|
||||
"failed to get products after retries",
|
||||
extra={
|
||||
"category": category_identifier,
|
||||
"label": pagination_label,
|
||||
"failed": failed_pages,
|
||||
},
|
||||
)
|
||||
|
||||
if stop_event.is_set():
|
||||
LOGGER.info(f"{category_identifier} - pagination stopped early")
|
||||
LOGGER.info("pagination stopped early", extra={"category": category_identifier})
|
||||
|
||||
|
||||
async def fetch_all_pages(
|
||||
@@ -299,24 +309,41 @@ def handle_page_results(
|
||||
retry_attempt: int | None = None,
|
||||
) -> list[int]:
|
||||
failed_pages = []
|
||||
retry_suffix = f" Retry {retry_attempt} " if retry_attempt else " "
|
||||
|
||||
for page_number, result in zip(page_numbers, results):
|
||||
if isinstance(result, EXPECTED_TASK_ERRORS):
|
||||
LOGGER.warning(
|
||||
f"{category_identifier} - {pagination_label} {page_number} -{retry_suffix} failed",
|
||||
"failed",
|
||||
extra={
|
||||
"category": category_identifier,
|
||||
"label": pagination_label,
|
||||
"page": page_number,
|
||||
"retry": retry_attempt,
|
||||
},
|
||||
exc_info=result,
|
||||
)
|
||||
failed_pages.append(page_number)
|
||||
elif isinstance(result, Exception):
|
||||
LOGGER.error(
|
||||
f"{category_identifier} - {pagination_label} {page_number} -{retry_suffix} unexpected error",
|
||||
"unexpected error",
|
||||
extra={
|
||||
"category": category_identifier,
|
||||
"label": pagination_label,
|
||||
"page": page_number,
|
||||
"retry": retry_attempt,
|
||||
},
|
||||
exc_info=result,
|
||||
)
|
||||
raise result
|
||||
elif retry_attempt:
|
||||
LOGGER.info(
|
||||
f"{category_identifier} - {pagination_label} {page_number} - retry {retry_attempt} succeeded"
|
||||
"retry succeeded",
|
||||
extra={
|
||||
"category": category_identifier,
|
||||
"label": pagination_label,
|
||||
"page": page_number,
|
||||
"retry": retry_attempt,
|
||||
},
|
||||
)
|
||||
|
||||
return failed_pages
|
||||
@@ -333,8 +360,14 @@ async def retry_failed_pages(
|
||||
while failed_pages and retry_attempt <= max_retries:
|
||||
await asyncio.sleep(2**retry_attempt)
|
||||
LOGGER.info(
|
||||
f"{category_identifier} - retrying {len(failed_pages)} failed {pagination_label}s "
|
||||
f"(attempt {retry_attempt}/{max_retries})"
|
||||
"retrying failed pages",
|
||||
extra={
|
||||
"category": category_identifier,
|
||||
"label": pagination_label,
|
||||
"count": len(failed_pages),
|
||||
"attempt": retry_attempt,
|
||||
"max_retries": max_retries,
|
||||
},
|
||||
)
|
||||
tasks = [fetch_func(page_number) for page_number in failed_pages]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
Reference in new issue
Block a user