added running pre task same time as category tasks
This commit is contained in:
1 parent
00d3c12ee0
commit
ae8b204a2c
3 files changed
+25
-18
No files matched your search
+15
-17
@@ -191,15 +191,15 @@ async def execute_monitor_logic(
|
||||
proxies: AioProxies,
|
||||
headers: dict | None = None,
|
||||
cookies: dict | None = None,
|
||||
pre_task_func: Callable[..., Awaitable[None]] | None = None,
|
||||
concurrent_pre_task: Callable[..., Awaitable[None]] | None = None,
|
||||
use_tls: bool = False,
|
||||
**kwargs,
|
||||
):
|
||||
"""Run the monitor's category tasks in a loop with a fresh session per cycle.
|
||||
|
||||
Each cycle opens a TLS or base session, optionally runs pre_task_func,
|
||||
dispatches task_func per category concurrently, logs expected failures,
|
||||
raises unexpected ones, then sleeps briefly before the next cycle.
|
||||
Each cycle opens a TLS or base session, dispatches task_func per category
|
||||
concurrently, logs expected failures, raises unexpected ones, then sleeps
|
||||
briefly before the next cycle.
|
||||
|
||||
Args:
|
||||
category_identifiers: List of category ids, or a mapping of category
|
||||
@@ -210,10 +210,13 @@ async def execute_monitor_logic(
|
||||
proxies: Proxies applied to the session.
|
||||
headers: Headers applied to the session.
|
||||
cookies: Cookies applied to the base session; ignored for TLS.
|
||||
pre_task_func: Optional awaitable run once per cycle before category
|
||||
tasks, receiving session and **kwargs.
|
||||
concurrent_pre_task: Optional awaitable run once per cycle, gathered
|
||||
alongside the category tasks and receiving session and **kwargs.
|
||||
Category tasks must not depend on its side effects (session
|
||||
headers, cookies, fetched state) since no ordering is guaranteed.
|
||||
use_tls: Use the TLS fingerprint session instead of the base session.
|
||||
**kwargs: Forwarded verbatim to pre_task_func and every task_func call.
|
||||
**kwargs: Forwarded verbatim to concurrent_pre_task and every
|
||||
task_func call.
|
||||
"""
|
||||
|
||||
while True:
|
||||
@@ -227,15 +230,6 @@ async def execute_monitor_logic(
|
||||
)
|
||||
async with session_ctx as session:
|
||||
LOGGER.info("starting tasks")
|
||||
if pre_task_func:
|
||||
try:
|
||||
await pre_task_func(session=session, **kwargs)
|
||||
except EXPECTED_TASK_ERRORS as exc:
|
||||
LOGGER.warning("failed to get products", exc_info=exc)
|
||||
except Exception:
|
||||
LOGGER.exception("unexpected error")
|
||||
raise
|
||||
|
||||
if isinstance(category_identifiers, dict):
|
||||
tasks = [
|
||||
task_func(category_id, session, query_category_id=query, **kwargs)
|
||||
@@ -247,7 +241,11 @@ async def execute_monitor_logic(
|
||||
task_func(category_identifier, session, **kwargs)
|
||||
for category_identifier in category_identifiers
|
||||
]
|
||||
category_keys = category_identifiers
|
||||
category_keys = list(category_identifiers)
|
||||
|
||||
if concurrent_pre_task:
|
||||
tasks.append(concurrent_pre_task(session=session, **kwargs))
|
||||
category_keys.append("pre_task")
|
||||
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
for category_identifier, result in zip(category_keys, results):
|
||||
|
||||
Reference in new issue
Block a user