fix: list JSON body + preserve real last response; retry via commons.aretry (v0.1.1)
- #7: request_with_retries routed only dicts to json=, so a valid JSON list body was form-encoded via data=. add _route_body so dict OR list -> json=. - #6: when every attempt returned a retryable status, the loop discarded the real response and returned a synthetic FailureResponse (status 0). now the real last 4xx/5xx Response is returned on exhaustion (only a pure-exception failure yields FailureResponse). - migrate the retry/backoff loop onto commons.aretry (>=0.2.0); backoff schedule unchanged (1,2,... = backoff_base**n), jitter off to match prior behavior. verified by execution: list->json routing, exhausted 503 returns real 503 + body with correct backoff, success/404 immediate, exception->falsy FailureResponse. Signed-off-by: disqualifier <dev@disqualifier.me>
This commit is contained in:
parent
205a7d5e21
commit
330656c6cf
@ -4,13 +4,17 @@ build-backend = "hatchling.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "aioweb"
|
name = "aioweb"
|
||||||
version = "0.1.0"
|
version = "0.1.1"
|
||||||
description = "Async HTTP session wrapper over aiohttp — proxies, header overwrites, retries, previews. Config-free, installable."
|
description = "Async HTTP session wrapper over aiohttp — proxies, header overwrites, retries, previews. Config-free, installable."
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aiohttp>=3.9",
|
"aiohttp>=3.9",
|
||||||
"yarl>=1.9",
|
"yarl>=1.9",
|
||||||
|
"commons @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.2.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[tool.hatch.metadata]
|
||||||
|
allow-direct-references = true
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
[tool.hatch.build.targets.wheel]
|
||||||
packages = ["src/aioweb"]
|
packages = ["src/aioweb"]
|
||||||
|
|||||||
@ -17,16 +17,41 @@ sessions must be closed explicitly (async with, or await s.close()); there is no
|
|||||||
__del__ auto-close (that pattern is unsafe for async resources).
|
__del__ auto-close (that pattern is unsafe for async resources).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import logging
|
import logging
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from yarl import URL
|
from yarl import URL
|
||||||
|
from commons import aretry
|
||||||
|
|
||||||
from .preview import RequestPreview
|
from .preview import RequestPreview
|
||||||
from .responses import Response, FailureResponse
|
from .responses import Response, FailureResponse
|
||||||
|
|
||||||
|
|
||||||
|
def _route_body(data):
|
||||||
|
"""split a body into (data=, json=) kwargs
|
||||||
|
|
||||||
|
dict OR list bodies are valid JSON and route to json=; everything else
|
||||||
|
(str/bytes/form) routes to data=. previously only dicts went to json=, so a
|
||||||
|
JSON list was wrongly form-encoded.
|
||||||
|
"""
|
||||||
|
if isinstance(data, (dict, list)):
|
||||||
|
return None, data
|
||||||
|
return data, None
|
||||||
|
|
||||||
|
|
||||||
|
class _RetryStatus(Exception):
|
||||||
|
"""internal signal: a retryable HTTP status; carries the real Response
|
||||||
|
|
||||||
|
raised inside an attempt so commons.aretry drives the backoff + cap; the caller
|
||||||
|
catches the final one to return the REAL last response, not a synthetic failure.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, response):
|
||||||
|
super().__init__(f"retryable status {response.status_code}")
|
||||||
|
self.response = response
|
||||||
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
DEFAULT_ATTEMPTS = 3
|
DEFAULT_ATTEMPTS = 3
|
||||||
@ -297,42 +322,42 @@ class ExtendedSession:
|
|||||||
(backoff_base ** attempt).
|
(backoff_base ** attempt).
|
||||||
"""
|
"""
|
||||||
attempts = attempts or DEFAULT_ATTEMPTS
|
attempts = attempts or DEFAULT_ATTEMPTS
|
||||||
last_error = None
|
body_data, body_json = _route_body(data)
|
||||||
|
|
||||||
if debug:
|
if debug:
|
||||||
preview = self.preview(
|
preview = self.preview(
|
||||||
method=method, url=url, params=params,
|
method=method, url=url, params=params,
|
||||||
data=None if isinstance(data, dict) else data,
|
data=body_data, json=body_json,
|
||||||
json=data if isinstance(data, dict) else None,
|
|
||||||
headers=headers, proxies=proxies, timeout=timeout,
|
headers=headers, proxies=proxies, timeout=timeout,
|
||||||
).as_curl()
|
).as_curl()
|
||||||
log.info("[aioweb.debug]\n%s\nproxies: %s inject: %s", preview, self.proxies, self.inject)
|
log.info("[aioweb.debug]\n%s\nproxies: %s inject: %s", preview, self.proxies, self.inject)
|
||||||
|
|
||||||
for attempt in range(attempts):
|
async def attempt():
|
||||||
try:
|
|
||||||
response = await self.request(
|
response = await self.request(
|
||||||
method=method, url=url, params=params,
|
method=method, url=url, params=params,
|
||||||
data=None if isinstance(data, dict) else data,
|
data=body_data, json=body_json,
|
||||||
json=data if isinstance(data, dict) else None,
|
|
||||||
headers=headers, proxies=proxies, timeout=timeout, debug=debug,
|
headers=headers, proxies=proxies, timeout=timeout, debug=debug,
|
||||||
)
|
)
|
||||||
if response.status_code in retry_statuses:
|
if response.status_code in retry_statuses:
|
||||||
last_error = f"retryable status {response.status_code}"
|
log.warning("retryable status %s for %s", response.status_code, url)
|
||||||
log.warning("attempt %d: %s for %s", attempt + 1, last_error, url)
|
raise _RetryStatus(response)
|
||||||
else:
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
try:
|
||||||
|
return await aretry(
|
||||||
|
attempt, attempts=attempts, backoff=1.0, factor=backoff_base,
|
||||||
|
jitter=False, on=(Exception,),
|
||||||
|
)
|
||||||
|
except _RetryStatus as exhausted:
|
||||||
|
log.error("all %d attempts failed for %s (last status %s)",
|
||||||
|
attempts, url, exhausted.response.status_code)
|
||||||
|
return exhausted.response
|
||||||
except aiohttp.ClientError as error:
|
except aiohttp.ClientError as error:
|
||||||
last_error = f"client error: {error}"
|
log.error("all %d attempts failed for %s (client error: %s)", attempts, url, error)
|
||||||
log.warning("attempt %d: %s, retrying", attempt + 1, last_error)
|
return FailureResponse(reason=f"client error: {error}", url=url)
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
last_error = f"unexpected error: {error}"
|
log.error("all %d attempts failed for %s (unexpected: %s)", attempts, url, error)
|
||||||
log.exception("attempt %d: %s, retrying", attempt + 1, last_error)
|
return FailureResponse(reason=f"unexpected error: {error}", url=url)
|
||||||
|
|
||||||
if attempt < attempts - 1:
|
|
||||||
await asyncio.sleep(backoff_base ** attempt)
|
|
||||||
|
|
||||||
log.error("all %d attempts failed for %s", attempts, url)
|
|
||||||
return FailureResponse(reason=last_error, url=url)
|
|
||||||
|
|
||||||
# -------------------------------------------------------------------------
|
# -------------------------------------------------------------------------
|
||||||
# lifecycle
|
# lifecycle
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user