added bounded close session for curl_cffi/past_prices to pricing model
This commit is contained in:
1 parent
c29032a765
commit
8a1c1893b7
4 files changed
+191
-3
No files matched your search
@@ -6,6 +6,8 @@ request_with_retries, adding per-request proxy rotation and old-style JSON/text
|
||||
return values so monitor call sites change as little as possible.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
|
||||
from aioproxies import AioProxies
|
||||
@@ -15,9 +17,53 @@ from monitor_utils.constants import (
|
||||
CLIENT_IDENTIFIERS,
|
||||
MAX_CURL_HANDLES,
|
||||
RETRY_LIMIT,
|
||||
SESSION_CLOSE_TIMEOUT_SECONDS,
|
||||
TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BoundedCloseSession(TLSSession):
|
||||
"""A TLSSession whose close cannot outlive SESSION_CLOSE_TIMEOUT_SECONDS.
|
||||
|
||||
curl_cffi's close awaits the multi handle with no timeout of its own, so a
|
||||
wedged transfer parks whatever is closing the session. A caller-side budget
|
||||
cannot cover this reliably: asyncio.timeout cancels its task once, so a
|
||||
cycle budget that has already fired leaves the close running as a fresh,
|
||||
uncancelled await, and the process parks in the `async with` exit that was
|
||||
meant to reclaim it.
|
||||
|
||||
Bounding it here keeps that guarantee attached to the session itself, so
|
||||
every `async with get_default_session(...)` gets it without the call site
|
||||
having to arrange anything. Abandoning a session leaks its handle until the
|
||||
process restarts, which is strictly better than a wedged monitor.
|
||||
|
||||
Known inconsistency, deliberate: an abandoned session reports itself closed
|
||||
while its curl handle is still open. TLSSession.close sets _closed before
|
||||
awaiting the backend, so _is_closed() returns True on this path, and since
|
||||
aioweb's __del__ only warns when _is_closed() is False, its leak warning is
|
||||
suppressed for exactly the sessions that leaked. The "session close exceeded
|
||||
budget" error logged below is therefore the only signal that it happened —
|
||||
alert on it rather than on the library's warning.
|
||||
|
||||
Correcting the flag would mean overriding _is_closed too, and aioweb_tls is
|
||||
a third-party library: this class deliberately overrides one documented
|
||||
lifecycle method and nothing private, so an upstream release has one narrow
|
||||
way to break it. A stale flag on a session discarded the same instant is the
|
||||
cheaper side of that trade.
|
||||
"""
|
||||
|
||||
async def close(self) -> None:
|
||||
try:
|
||||
async with asyncio.timeout(SESSION_CLOSE_TIMEOUT_SECONDS):
|
||||
await super().close()
|
||||
except TimeoutError:
|
||||
LOGGER.error(
|
||||
"session close exceeded budget, abandoning session",
|
||||
extra={"timeout": SESSION_CLOSE_TIMEOUT_SECONDS},
|
||||
)
|
||||
|
||||
|
||||
def get_default_session(
|
||||
proxies: AioProxies,
|
||||
@@ -54,11 +100,12 @@ def get_default_session(
|
||||
requests) so the wait is never reachable.
|
||||
|
||||
Returns:
|
||||
A TLSSession over a curl_cffi backend with a random impersonation, with
|
||||
the proxy pool attached as proxy_pool.
|
||||
A BoundedCloseSession (a TLSSession whose close is bounded) over a
|
||||
curl_cffi backend with a random impersonation, with the proxy pool
|
||||
attached as proxy_pool.
|
||||
"""
|
||||
impersonate = random.choice(CLIENT_IDENTIFIERS)
|
||||
session = TLSSession(
|
||||
session = BoundedCloseSession(
|
||||
backend=CurlCffi(impersonate=impersonate),
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
|
||||
Reference in new issue
Block a user