Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1dc27ebc1a | |||
| f8476fe8d4 | |||
| 12cf07919f | |||
| 4be69f3c95 | |||
| 5d444eaf16 | |||
| 449f790571 | |||
| a5b91bed0d | |||
| 83a156fd31 | |||
| de6911fb05 | |||
| c6e3dd1b54 | |||
| 0939917172 |
2
.gitignore
vendored
2
.gitignore
vendored
@ -1,5 +1,5 @@
|
|||||||
# claude
|
# claude
|
||||||
CLAUDE.md
|
.claude/
|
||||||
|
|
||||||
# python
|
# python
|
||||||
__pycache__/
|
__pycache__/
|
||||||
|
|||||||
40
README.md
40
README.md
@ -5,20 +5,23 @@ Small sync helpers shared across projects. Base is stdlib only — **no dependen
|
|||||||
- `timing` — unix-timestamp deltas + timezone-aware datetime conversions
|
- `timing` — unix-timestamp deltas + timezone-aware datetime conversions
|
||||||
- `paths` — nested dict/list access by dotted path
|
- `paths` — nested dict/list access by dotted path
|
||||||
- `masking` — display masking for cards / cvv / tokens
|
- `masking` — display masking for cards / cvv / tokens
|
||||||
|
- `retry` — exponential-backoff retry, sync (`retry`) and async (`aretry`)
|
||||||
- `addr` — ip/address tooling: pure stdlib ip utils in base, async geo lookups
|
- `addr` — ip/address tooling: pure stdlib ip utils in base, async geo lookups
|
||||||
behind the `commons[addr]` extra
|
behind the `commons[addr]` extra
|
||||||
|
|
||||||
## Install
|
## Install
|
||||||
|
|
||||||
```
|
```
|
||||||
commons @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.1.0
|
commons @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.2.3
|
||||||
# async address/geo lookups (fetch_ip / ip_location / fetch_location) need the extra:
|
# async address/geo lookups (fetch_ip / ip_location / fetch_location) need the extra:
|
||||||
commons[addr] @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.1.0
|
commons[addr] @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.2.3
|
||||||
```
|
```
|
||||||
|
|
||||||
The base install pulls **nothing** (stdlib). Only `commons[addr]` adds `aiohttp`, and
|
The base install pulls **nothing** (stdlib). Only `commons[addr]` adds `aiohttp`, and
|
||||||
only for the geo lookups — the pure `commons.addr.ip` utilities ship in base.
|
only for the geo lookups — the pure `commons.addr.ip` utilities ship in base.
|
||||||
|
|
||||||
|
Drop the `@v0.2.3` suffix from the line above to install the latest unpinned.
|
||||||
|
|
||||||
## timing
|
## timing
|
||||||
|
|
||||||
Unix ints stay the storable value; datetimes are produced on demand in whatever
|
Unix ints stay the storable value; datetimes are produced on demand in whatever
|
||||||
@ -112,6 +115,39 @@ phantom("abcdef1234567890") # "abcdef...7890"
|
|||||||
provider("4111111111111111") # "VISA" (VISA/MC/AMEX/UPAY/DISC/JCB/DNRS/UNKW)
|
provider("4111111111111111") # "VISA" (VISA/MC/AMEX/UPAY/DISC/JCB/DNRS/UNKW)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## retry
|
||||||
|
|
||||||
|
Exponential-backoff retry, sync (`retry`) and async (`aretry`). Each works as a **call
|
||||||
|
form** or a **decorator**, with the same kwargs. After the attempts are exhausted the
|
||||||
|
**last exception is re-raised** — it never swallows or returns a default.
|
||||||
|
|
||||||
|
```python
|
||||||
|
from commons import retry, aretry
|
||||||
|
|
||||||
|
# call form
|
||||||
|
rows = retry(lambda: read_db(), attempts=5, on=(IOError,))
|
||||||
|
data = await aretry(lambda: fetch(url), attempts=3, backoff=0.5, on=(TimeoutError,))
|
||||||
|
|
||||||
|
# decorator form (same kwargs)
|
||||||
|
@aretry(attempts=4, backoff=0.5, factor=2.0, on=(ConnectionError,))
|
||||||
|
async def pull():
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
Knobs: `attempts` (total tries), `backoff` / `factor` / `max_backoff` (delay is
|
||||||
|
`min(backoff * factor**n, max_backoff)`), `jitter` (full jitter, on by default),
|
||||||
|
`on=` (tuple of retryable exception types), and `give_up=lambda exc: ...` to stop early
|
||||||
|
on a non-retryable error (e.g. a 400 vs a 429):
|
||||||
|
|
||||||
|
```python
|
||||||
|
# retry 429/5xx but give up immediately on a 4xx
|
||||||
|
await aretry(send, attempts=4, on=(HTTPError,),
|
||||||
|
give_up=lambda e: 400 <= e.status < 500 and e.status != 429)
|
||||||
|
```
|
||||||
|
|
||||||
|
Each retry is logged (emit-only). `sleep=` and `rand=` are injectable for deterministic
|
||||||
|
tests (no real waits).
|
||||||
|
|
||||||
## addr
|
## addr
|
||||||
|
|
||||||
IP/address tooling, exposed as a submodule. The pure `ip` utilities ship in the base
|
IP/address tooling, exposed as a submodule. The pure `ip` utilities ship in the base
|
||||||
|
|||||||
@ -4,8 +4,8 @@ build-backend = "hatchling.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "commons"
|
name = "commons"
|
||||||
version = "0.1.0"
|
version = "0.2.3"
|
||||||
description = "small stdlib-only sync helpers: time/timezone deltas, dotted-path dict access, display masking, and ip/address tooling"
|
description = "small stdlib-based sync helpers: time/timezone deltas, dotted-path dict access, display masking, ip/address tooling, and retry/backoff"
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
dependencies = []
|
dependencies = []
|
||||||
|
|
||||||
|
|||||||
@ -6,6 +6,8 @@ paths: nested dict/list access by dotted path (deep_get / deep_set).
|
|||||||
masking: display masking for cards / cvv / tokens (cosmetic, not a security control).
|
masking: display masking for cards / cvv / tokens (cosmetic, not a security control).
|
||||||
addr: ip/address tooling — pure stdlib ip utils (`commons.addr.ip`) in base,
|
addr: ip/address tooling — pure stdlib ip utils (`commons.addr.ip`) in base,
|
||||||
async geo lookups (`commons.addr.geo`) behind the `commons[addr]` extra.
|
async geo lookups (`commons.addr.geo`) behind the `commons[addr]` extra.
|
||||||
|
retry: exponential-backoff retry, sync (`retry`) and async (`aretry`), call or
|
||||||
|
decorator form; re-raises the last exception (fail loud), never swallows.
|
||||||
|
|
||||||
base is stdlib only, no dependencies (the addr geo lookups add aiohttp via the
|
base is stdlib only, no dependencies (the addr geo lookups add aiohttp via the
|
||||||
extra). to toggle the timing test mode for bare calls, set it on the module —
|
extra). to toggle the timing test mode for bare calls, set it on the module —
|
||||||
@ -16,6 +18,7 @@ addr`); its ip helpers live under `commons.addr.ip` to keep top-level uncluttere
|
|||||||
from . import addr, masking, paths, timing
|
from . import addr, masking, paths, timing
|
||||||
from .masking import credit, cvv, phantom, provider
|
from .masking import credit, cvv, phantom, provider
|
||||||
from .paths import deep_get, deep_set
|
from .paths import deep_get, deep_set
|
||||||
|
from .retry import aretry, retry
|
||||||
from .timing import (
|
from .timing import (
|
||||||
UTC,
|
UTC,
|
||||||
Clock,
|
Clock,
|
||||||
@ -56,6 +59,8 @@ __all__ = [
|
|||||||
"cvv",
|
"cvv",
|
||||||
"phantom",
|
"phantom",
|
||||||
"provider",
|
"provider",
|
||||||
|
"retry",
|
||||||
|
"aretry",
|
||||||
]
|
]
|
||||||
|
|
||||||
__version__ = "0.1.0"
|
__version__ = "0.2.3"
|
||||||
|
|||||||
@ -12,6 +12,7 @@ security: the only secret is geo.ipify's `api_key`, which is a REQUIRED keyword
|
|||||||
`ip_location` — the caller injects it. nothing is hardcoded here.
|
`ip_location` — the caller injects it. nothing is hardcoded here.
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
|
import unicodedata
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
|
|
||||||
@ -51,12 +52,26 @@ def _reverse_url(lat: float, lon: float) -> str:
|
|||||||
|
|
||||||
def _parse_ipify(data: dict) -> Optional[str]:
|
def _parse_ipify(data: dict) -> Optional[str]:
|
||||||
"""pull the ip string out of an ipify response"""
|
"""pull the ip string out of an ipify response"""
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return None
|
||||||
ip = data.get("ip")
|
ip = data.get("ip")
|
||||||
return ip or None
|
return ip or None
|
||||||
|
|
||||||
|
|
||||||
|
def _state_slug(state) -> str:
|
||||||
|
"""lowercase ascii-folded state slug with underscores (e.g. 'New York' -> 'new_york')
|
||||||
|
|
||||||
|
coerces to str first so a malformed non-string `state` doesn't raise TypeError out
|
||||||
|
of unicodedata.normalize and break the 'None on any parse failure' contract.
|
||||||
|
"""
|
||||||
|
folded = unicodedata.normalize("NFKD", str(state)).encode("ascii", "ignore").decode("ascii")
|
||||||
|
return folded.lower().replace(" ", "_")
|
||||||
|
|
||||||
|
|
||||||
def _parse_reverse(data: dict) -> Optional[dict]:
|
def _parse_reverse(data: dict) -> Optional[dict]:
|
||||||
"""parse a nominatim reverse response into {country: iso2 lower, state: slug|None}"""
|
"""parse a nominatim reverse response into {country: iso2 lower, state: slug|None}"""
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return None
|
||||||
address = data.get("address")
|
address = data.get("address")
|
||||||
if not isinstance(address, dict):
|
if not isinstance(address, dict):
|
||||||
return None
|
return None
|
||||||
@ -66,7 +81,7 @@ def _parse_reverse(data: dict) -> Optional[dict]:
|
|||||||
state = address.get("state")
|
state = address.get("state")
|
||||||
return {
|
return {
|
||||||
"country": str(country).lower(),
|
"country": str(country).lower(),
|
||||||
"state": state.lower().replace(" ", "-") if state else None,
|
"state": _state_slug(state) if state else None,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -77,10 +92,11 @@ async def _get_json(
|
|||||||
if not _HAVE_AIOHTTP:
|
if not _HAVE_AIOHTTP:
|
||||||
raise RuntimeError(_MISSING)
|
raise RuntimeError(_MISSING)
|
||||||
owns = session is None
|
owns = session is None
|
||||||
|
request_timeout = aiohttp.ClientTimeout(total=timeout)
|
||||||
if owns:
|
if owns:
|
||||||
session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout))
|
session = aiohttp.ClientSession(timeout=request_timeout)
|
||||||
try:
|
try:
|
||||||
async with session.get(url, headers=headers) as resp:
|
async with session.get(url, headers=headers, timeout=request_timeout) as resp:
|
||||||
if resp.status != 200:
|
if resp.status != 200:
|
||||||
log.warning("address lookup %s -> %s", url, resp.status)
|
log.warning("address lookup %s -> %s", url, resp.status)
|
||||||
return None
|
return None
|
||||||
|
|||||||
165
src/commons/retry.py
Normal file
165
src/commons/retry.py
Normal file
@ -0,0 +1,165 @@
|
|||||||
|
"""retry with exponential backoff — sync and async, one backoff engine.
|
||||||
|
|
||||||
|
de-duplicates retry logic that was written divergently across several libs (HTTP
|
||||||
|
429/5xx caps, proxy burn/rotate caps, IMAP reconnects). both a call form and a
|
||||||
|
decorator form share one implementation per flavor; the backoff schedule is a pure
|
||||||
|
generator so it tests without real sleeps.
|
||||||
|
|
||||||
|
from commons import retry, aretry
|
||||||
|
|
||||||
|
# call form
|
||||||
|
rows = retry(lambda: db_read(), attempts=5, on=(IOError,))
|
||||||
|
data = await aretry(lambda: fetch(url), attempts=3, on=(TimeoutError,))
|
||||||
|
|
||||||
|
# decorator form (same kwargs)
|
||||||
|
@aretry(attempts=4, backoff=0.5, on=(ConnectionError,))
|
||||||
|
async def pull():
|
||||||
|
...
|
||||||
|
|
||||||
|
after `attempts` are exhausted the LAST exception is re-raised (fail loud, never
|
||||||
|
swallowed). `on` narrows which exceptions retry; `give_up(exc) -> bool` stops early
|
||||||
|
on a non-retryable error (e.g. a 400 vs a 429). each retry is logged (emit-only),
|
||||||
|
never printed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import functools
|
||||||
|
import logging
|
||||||
|
import random
|
||||||
|
import time
|
||||||
|
from typing import Callable, Iterable, Optional, Tuple, Type
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
ExcTypes = Tuple[Type[BaseException], ...]
|
||||||
|
|
||||||
|
|
||||||
|
def _delays(attempts: int, backoff: float, factor: float, max_backoff: float):
|
||||||
|
"""yield the wait before each retry: min(backoff * factor**n, max_backoff)
|
||||||
|
|
||||||
|
yields `attempts - 1` delays (one before each retry after the first try). the
|
||||||
|
raw, un-jittered schedule — jitter is applied at call time so the schedule stays
|
||||||
|
pure and testable.
|
||||||
|
"""
|
||||||
|
for n in range(max(0, attempts - 1)):
|
||||||
|
yield min(backoff * (factor ** n), max_backoff)
|
||||||
|
|
||||||
|
|
||||||
|
def _jittered(delay: float, jitter: bool, rand: Callable[[], float]) -> float:
|
||||||
|
"""apply full jitter to a delay when enabled: uniform(0, delay)"""
|
||||||
|
if not jitter or delay <= 0:
|
||||||
|
return delay
|
||||||
|
return rand() * delay
|
||||||
|
|
||||||
|
|
||||||
|
def _as_types(on: Iterable[Type[BaseException]]) -> ExcTypes:
|
||||||
|
"""coerce the `on` argument into a tuple of exception types"""
|
||||||
|
if isinstance(on, type):
|
||||||
|
return (on,)
|
||||||
|
return tuple(on)
|
||||||
|
|
||||||
|
|
||||||
|
def retry(
|
||||||
|
fn: Optional[Callable] = None,
|
||||||
|
*,
|
||||||
|
attempts: int = 3,
|
||||||
|
backoff: float = 0.5,
|
||||||
|
factor: float = 2.0,
|
||||||
|
max_backoff: float = 30.0,
|
||||||
|
jitter: bool = True,
|
||||||
|
on: Iterable[Type[BaseException]] = (Exception,),
|
||||||
|
give_up: Optional[Callable[[BaseException], bool]] = None,
|
||||||
|
sleep: Callable[[float], None] = time.sleep,
|
||||||
|
rand: Callable[[], float] = random.random,
|
||||||
|
):
|
||||||
|
"""retry a sync callable with exponential backoff; call form or decorator
|
||||||
|
|
||||||
|
`retry(fn, ...)` runs immediately; `@retry(...)` wraps a function. retries on the
|
||||||
|
`on` exceptions, stops early if `give_up(exc)` is true, re-raises the last
|
||||||
|
exception once `attempts` are exhausted. `sleep`/`rand` are injectable for tests.
|
||||||
|
`attempts` is floored at 1 so the callable always runs at least once.
|
||||||
|
"""
|
||||||
|
types = _as_types(on)
|
||||||
|
attempts = max(1, attempts)
|
||||||
|
|
||||||
|
def run(target: Callable, args, kwargs):
|
||||||
|
delays = list(_delays(attempts, backoff, factor, max_backoff))
|
||||||
|
last_index = attempts - 1
|
||||||
|
for index in range(attempts):
|
||||||
|
try:
|
||||||
|
return target(*args, **kwargs)
|
||||||
|
except types as exc:
|
||||||
|
if give_up is not None and give_up(exc):
|
||||||
|
raise
|
||||||
|
if index == last_index:
|
||||||
|
raise
|
||||||
|
wait = _jittered(delays[index], jitter, rand)
|
||||||
|
log.warning(
|
||||||
|
"retry %d/%d after %s: %s",
|
||||||
|
index + 1, attempts, type(exc).__name__, exc,
|
||||||
|
)
|
||||||
|
if wait > 0:
|
||||||
|
sleep(wait)
|
||||||
|
|
||||||
|
def decorator(target: Callable) -> Callable:
|
||||||
|
@functools.wraps(target)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
return run(target, args, kwargs)
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
if fn is not None:
|
||||||
|
return run(fn, (), {})
|
||||||
|
return decorator
|
||||||
|
|
||||||
|
|
||||||
|
def aretry(
|
||||||
|
fn: Optional[Callable] = None,
|
||||||
|
*,
|
||||||
|
attempts: int = 3,
|
||||||
|
backoff: float = 0.5,
|
||||||
|
factor: float = 2.0,
|
||||||
|
max_backoff: float = 30.0,
|
||||||
|
jitter: bool = True,
|
||||||
|
on: Iterable[Type[BaseException]] = (Exception,),
|
||||||
|
give_up: Optional[Callable[[BaseException], bool]] = None,
|
||||||
|
sleep: Callable[[float], "asyncio.Future"] = asyncio.sleep,
|
||||||
|
rand: Callable[[], float] = random.random,
|
||||||
|
):
|
||||||
|
"""retry an async callable with exponential backoff; call form or decorator
|
||||||
|
|
||||||
|
async twin of `retry`. `await aretry(coro_fn, ...)` runs immediately;
|
||||||
|
`@aretry(...)` wraps a coroutine function. same semantics: retry on `on`, stop on
|
||||||
|
`give_up`, re-raise the last exception after `attempts`. `sleep`/`rand` injectable.
|
||||||
|
`attempts` is floored at 1 so the callable always runs at least once.
|
||||||
|
"""
|
||||||
|
types = _as_types(on)
|
||||||
|
attempts = max(1, attempts)
|
||||||
|
|
||||||
|
async def run(target: Callable, args, kwargs):
|
||||||
|
delays = list(_delays(attempts, backoff, factor, max_backoff))
|
||||||
|
last_index = attempts - 1
|
||||||
|
for index in range(attempts):
|
||||||
|
try:
|
||||||
|
return await target(*args, **kwargs)
|
||||||
|
except types as exc:
|
||||||
|
if give_up is not None and give_up(exc):
|
||||||
|
raise
|
||||||
|
if index == last_index:
|
||||||
|
raise
|
||||||
|
wait = _jittered(delays[index], jitter, rand)
|
||||||
|
log.warning(
|
||||||
|
"retry %d/%d after %s: %s",
|
||||||
|
index + 1, attempts, type(exc).__name__, exc,
|
||||||
|
)
|
||||||
|
if wait > 0:
|
||||||
|
await sleep(wait)
|
||||||
|
|
||||||
|
def decorator(target: Callable) -> Callable:
|
||||||
|
@functools.wraps(target)
|
||||||
|
async def wrapper(*args, **kwargs):
|
||||||
|
return await run(target, args, kwargs)
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
if fn is not None:
|
||||||
|
return run(fn, (), {})
|
||||||
|
return decorator
|
||||||
Loading…
Reference in New Issue
Block a user