fixed where f strings were still used for logging
This commit is contained in:
1 parent
c13e0d3c77
commit
dcc4f58292
1 file changed
+53
-20
+53
-20
@@ -41,7 +41,10 @@ def init_monitor(
|
|||||||
logging_level=LOGGING_LEVEL,
|
logging_level=LOGGING_LEVEL,
|
||||||
) -> tuple[list[str], AsyncIOMotorCollection, AioProxies]:
|
) -> tuple[list[str], AsyncIOMotorCollection, AioProxies]:
|
||||||
setup_logging(
|
setup_logging(
|
||||||
name="logs/latest.log", history_name="apple", level=logging_level, output="json"
|
name="logs/latest.log",
|
||||||
|
history_name=monitor_name,
|
||||||
|
level=logging_level,
|
||||||
|
output="json",
|
||||||
)
|
)
|
||||||
set_logger_levels()
|
set_logger_levels()
|
||||||
category_names, _, collection = setup_config(
|
category_names, _, collection = setup_config(
|
||||||
@@ -111,7 +114,10 @@ def build_product_details(
|
|||||||
current_price = product["currentPrice"]
|
current_price = product["currentPrice"]
|
||||||
original_price = product["originalPrice"]
|
original_price = product["originalPrice"]
|
||||||
if not original_price:
|
if not original_price:
|
||||||
LOGGER.warning(f"{product_id} - Missing or zero original price, skipping")
|
LOGGER.warning(
|
||||||
|
"missing or zero original price, skipping",
|
||||||
|
extra={"product_id": product_id},
|
||||||
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
percentage = int(((original_price - current_price) / original_price) * 100)
|
percentage = int(((original_price - current_price) / original_price) * 100)
|
||||||
@@ -179,14 +185,14 @@ async def execute_monitor_logic(
|
|||||||
else get_base_session(headers=headers, cookies=cookies, proxies=proxies)
|
else get_base_session(headers=headers, cookies=cookies, proxies=proxies)
|
||||||
)
|
)
|
||||||
async with session_ctx as session:
|
async with session_ctx as session:
|
||||||
LOGGER.info("Starting tasks...")
|
LOGGER.info("starting tasks")
|
||||||
if pre_task_func:
|
if pre_task_func:
|
||||||
try:
|
try:
|
||||||
await pre_task_func(session=session, **kwargs)
|
await pre_task_func(session=session, **kwargs)
|
||||||
except EXPECTED_TASK_ERRORS as error:
|
except EXPECTED_TASK_ERRORS as error:
|
||||||
LOGGER.warning("Failed to get products", exc_info=error)
|
LOGGER.warning("failed to get products", exc_info=error)
|
||||||
except Exception:
|
except Exception:
|
||||||
LOGGER.exception("Unexpected error")
|
LOGGER.exception("unexpected error")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
if isinstance(category_identifiers, dict):
|
if isinstance(category_identifiers, dict):
|
||||||
@@ -206,19 +212,19 @@ async def execute_monitor_logic(
|
|||||||
for category_identifier, result in zip(category_keys, results):
|
for category_identifier, result in zip(category_keys, results):
|
||||||
if isinstance(result, EXPECTED_TASK_ERRORS):
|
if isinstance(result, EXPECTED_TASK_ERRORS):
|
||||||
LOGGER.warning(
|
LOGGER.warning(
|
||||||
f"{category_identifier} - Failed to get products",
|
"failed to get products",
|
||||||
|
extra={"category": category_identifier},
|
||||||
exc_info=result,
|
exc_info=result,
|
||||||
)
|
)
|
||||||
elif isinstance(result, Exception):
|
elif isinstance(result, Exception):
|
||||||
LOGGER.error(
|
LOGGER.error(
|
||||||
f"{category_identifier} - Unexpected error",
|
"unexpected error",
|
||||||
|
extra={"category": category_identifier},
|
||||||
exc_info=result,
|
exc_info=result,
|
||||||
)
|
)
|
||||||
raise result
|
raise result
|
||||||
|
|
||||||
LOGGER.info(
|
LOGGER.info("sleeping before next tasks", extra={"delay": 0.30})
|
||||||
"Sleeping 0.30 seconds before next tasks"
|
|
||||||
) # Might add option to increase/decrease this delay
|
|
||||||
await asyncio.sleep(0.30)
|
await asyncio.sleep(0.30)
|
||||||
|
|
||||||
|
|
||||||
@@ -234,7 +240,7 @@ async def execute_pagination(
|
|||||||
pagination_mode: str = "offset",
|
pagination_mode: str = "offset",
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
LOGGER.info(f"{category_identifier} - executing pagination")
|
LOGGER.info("executing pagination", extra={"category": category_identifier})
|
||||||
total_pages = (int(total_products) + products_per_page - 1) // products_per_page
|
total_pages = (int(total_products) + products_per_page - 1) // products_per_page
|
||||||
semaphore = asyncio.Semaphore(max_concurrent_tasks)
|
semaphore = asyncio.Semaphore(max_concurrent_tasks)
|
||||||
stop_event = asyncio.Event()
|
stop_event = asyncio.Event()
|
||||||
@@ -270,12 +276,16 @@ async def execute_pagination(
|
|||||||
|
|
||||||
if failed_pages and not stop_event.is_set():
|
if failed_pages and not stop_event.is_set():
|
||||||
LOGGER.error(
|
LOGGER.error(
|
||||||
f"{category_identifier} - failed to get products after retries: "
|
"failed to get products after retries",
|
||||||
f"{pagination_label}s {failed_pages}"
|
extra={
|
||||||
|
"category": category_identifier,
|
||||||
|
"label": pagination_label,
|
||||||
|
"failed": failed_pages,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
if stop_event.is_set():
|
if stop_event.is_set():
|
||||||
LOGGER.info(f"{category_identifier} - pagination stopped early")
|
LOGGER.info("pagination stopped early", extra={"category": category_identifier})
|
||||||
|
|
||||||
|
|
||||||
async def fetch_all_pages(
|
async def fetch_all_pages(
|
||||||
@@ -299,24 +309,41 @@ def handle_page_results(
|
|||||||
retry_attempt: int | None = None,
|
retry_attempt: int | None = None,
|
||||||
) -> list[int]:
|
) -> list[int]:
|
||||||
failed_pages = []
|
failed_pages = []
|
||||||
retry_suffix = f" Retry {retry_attempt} " if retry_attempt else " "
|
|
||||||
|
|
||||||
for page_number, result in zip(page_numbers, results):
|
for page_number, result in zip(page_numbers, results):
|
||||||
if isinstance(result, EXPECTED_TASK_ERRORS):
|
if isinstance(result, EXPECTED_TASK_ERRORS):
|
||||||
LOGGER.warning(
|
LOGGER.warning(
|
||||||
f"{category_identifier} - {pagination_label} {page_number} -{retry_suffix} failed",
|
"failed",
|
||||||
|
extra={
|
||||||
|
"category": category_identifier,
|
||||||
|
"label": pagination_label,
|
||||||
|
"page": page_number,
|
||||||
|
"retry": retry_attempt,
|
||||||
|
},
|
||||||
exc_info=result,
|
exc_info=result,
|
||||||
)
|
)
|
||||||
failed_pages.append(page_number)
|
failed_pages.append(page_number)
|
||||||
elif isinstance(result, Exception):
|
elif isinstance(result, Exception):
|
||||||
LOGGER.error(
|
LOGGER.error(
|
||||||
f"{category_identifier} - {pagination_label} {page_number} -{retry_suffix} unexpected error",
|
"unexpected error",
|
||||||
|
extra={
|
||||||
|
"category": category_identifier,
|
||||||
|
"label": pagination_label,
|
||||||
|
"page": page_number,
|
||||||
|
"retry": retry_attempt,
|
||||||
|
},
|
||||||
exc_info=result,
|
exc_info=result,
|
||||||
)
|
)
|
||||||
raise result
|
raise result
|
||||||
elif retry_attempt:
|
elif retry_attempt:
|
||||||
LOGGER.info(
|
LOGGER.info(
|
||||||
f"{category_identifier} - {pagination_label} {page_number} - retry {retry_attempt} succeeded"
|
"retry succeeded",
|
||||||
|
extra={
|
||||||
|
"category": category_identifier,
|
||||||
|
"label": pagination_label,
|
||||||
|
"page": page_number,
|
||||||
|
"retry": retry_attempt,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
return failed_pages
|
return failed_pages
|
||||||
@@ -333,8 +360,14 @@ async def retry_failed_pages(
|
|||||||
while failed_pages and retry_attempt <= max_retries:
|
while failed_pages and retry_attempt <= max_retries:
|
||||||
await asyncio.sleep(2**retry_attempt)
|
await asyncio.sleep(2**retry_attempt)
|
||||||
LOGGER.info(
|
LOGGER.info(
|
||||||
f"{category_identifier} - retrying {len(failed_pages)} failed {pagination_label}s "
|
"retrying failed pages",
|
||||||
f"(attempt {retry_attempt}/{max_retries})"
|
extra={
|
||||||
|
"category": category_identifier,
|
||||||
|
"label": pagination_label,
|
||||||
|
"count": len(failed_pages),
|
||||||
|
"attempt": retry_attempt,
|
||||||
|
"max_retries": max_retries,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
tasks = [fetch_func(page_number) for page_number in failed_pages]
|
tasks = [fetch_func(page_number) for page_number in failed_pages]
|
||||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||||
|
|||||||
Reference in new issue
Block a user