Compare commits
10
Commits
v0.3.2
...
5a8ca8327d
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5a8ca8327d | ||
|
|
a6bd95bda7 | ||
|
|
a1702eede8 | ||
|
|
0030daeb7b | ||
|
|
e6e655335e | ||
|
|
7fa5916eda | ||
|
|
718c8a79b0 | ||
|
|
33ade498da | ||
|
|
e86184986f | ||
|
|
18646f313c |
@@ -12,15 +12,15 @@ Small sync helpers shared across projects. Base is stdlib only — **no dependen
|
||||
## Install
|
||||
|
||||
```
|
||||
commons @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.3.2
|
||||
commons @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.3.5
|
||||
# async address/geo lookups (fetch_ip / ip_location / fetch_location) need the extra:
|
||||
commons[addr] @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.3.2
|
||||
commons[addr] @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.3.5
|
||||
```
|
||||
|
||||
The base install pulls **nothing** (stdlib). Only `commons[addr]` adds `aiohttp`, and
|
||||
only for the geo lookups — the pure `commons.addr.ip` utilities ship in base.
|
||||
|
||||
Drop the `@v0.3.2` suffix from the line above to install the latest unpinned.
|
||||
Drop the `@v0.3.5` suffix from the line above to install the latest unpinned.
|
||||
|
||||
## timing
|
||||
|
||||
@@ -89,13 +89,17 @@ timing.FAST_MODE = True # in test setup
|
||||
|
||||
## paths
|
||||
|
||||
Two pairs of verbs: **single-path** (scalar in/out) and **wildcard** (bulk, always a list).
|
||||
|
||||
### single-path — `deep_get` / `deep_set`
|
||||
|
||||
```python
|
||||
from commons import deep_get, deep_set
|
||||
|
||||
data = {"in": {"this": {"old": {"notation": 42}}}, "items": [{"id": "a"}, {"id": "b"}]}
|
||||
|
||||
deep_get(data, "in.this.old.notation") # 42
|
||||
deep_get(data, "items.1.id") # "b" (numeric segment indexes a list)
|
||||
deep_get(data, "items.1.id") # "b" (numeric segment indexes a list/tuple)
|
||||
deep_get(data, "in.nope.here", "DEF") # "DEF" (missing -> default, no raise)
|
||||
|
||||
deep_set({}, "a.b.c", 9) # {"a": {"b": {"c": 9}}}
|
||||
@@ -105,10 +109,34 @@ deep_set(data, "items.9.id", "X") # raises IndexError (out of range, no
|
||||
# silent mis-store)
|
||||
```
|
||||
|
||||
`deep_set` mirrors `deep_get`'s list indexing: a numeric segment over an existing
|
||||
list/tuple updates that element in place rather than replacing the list with a dict.
|
||||
An out-of-range numeric segment raises `IndexError` instead of silently corrupting
|
||||
the structure.
|
||||
`deep_get` steps into both lists and tuples on a numeric segment. `deep_set` updates a
|
||||
**list** element in place; setting into (or through) a **tuple** raises `TypeError` —
|
||||
tuples are immutable, so "update in place" is impossible; flatten to a list first. An
|
||||
out-of-range numeric segment raises `IndexError` instead of silently corrupting the
|
||||
structure.
|
||||
|
||||
### wildcard — `deep_gets` / `deep_sets`
|
||||
|
||||
A `*` segment iterates every element at that level (dict values or list/tuple items);
|
||||
multiple `*` fan out cartesian. These are **separate verbs** with a fixed list/bulk
|
||||
return type — a `*` passed to `deep_get`/`deep_set` raises `ValueError` (use the plural
|
||||
verb).
|
||||
|
||||
```python
|
||||
from commons import deep_gets, deep_sets
|
||||
|
||||
data = {"users": [{"username": "al"}, {"username": "bo"}, {"username": "cy"}]}
|
||||
|
||||
deep_gets(data, "users.*.username") # ["al", "bo", "cy"] (ALWAYS a list, in order)
|
||||
deep_gets(data, "users.*.nope") # [] (no match -> empty list, no raise)
|
||||
|
||||
deep_sets(data, "users.*.username", "X") # set every match to "X"
|
||||
deep_sets(data, "users.*.username", str.upper) # callable fn(current)->new per match
|
||||
```
|
||||
|
||||
`deep_sets`'s second arg is either a plain value (write it to every match) or a callable
|
||||
`fn(current) -> new` (compute each). It writes only matches that already exist (it does
|
||||
not create missing keys). Setting through/into a tuple raises `TypeError`.
|
||||
|
||||
## masking
|
||||
|
||||
@@ -116,7 +144,7 @@ Display helpers only — they format a value for showing; they are **not a secur
|
||||
control** (the underlying value is unchanged and still needs proper handling).
|
||||
|
||||
```python
|
||||
from commons import credit, cvv, phantom, provider, mask_url, mask_proxy
|
||||
from commons import credit, cvv, phantom, provider, mask_url, mask_proxy, stable_id
|
||||
|
||||
credit("4111 1111 1111 1234") # "•••• •••• •••• 1234"
|
||||
cvv("123") # "•••"
|
||||
@@ -125,18 +153,35 @@ phantom("1234567890") # "••••••••••" (len <= 10
|
||||
provider("4111111111111111") # "VISA" (VISA/MC/AMEX/UPAY/DISC/JCB/DNRS/UNKW)
|
||||
provider("4111²111111111234") # "VISA" (unicode digit lookalikes ignored, never raises)
|
||||
|
||||
# redact credentials in a connection string before logging it
|
||||
mask_url("https://u:pw@api.x/v2?apiKey=SECRET&ip=8.8.8.8")
|
||||
# -> "https://api.x/v2?apiKey=***&ip=8.8.8.8" (userinfo dropped, secret query masked)
|
||||
mask_url("redis://:pw@127.0.0.1:6379/0") # -> "redis://127.0.0.1:6379/0"
|
||||
mask_proxy("1.2.3.4:8080:user:supersecret") # -> "1.2.3.4:8080:user:****"
|
||||
mask_proxy("1.2.3.4:8080") # -> "1.2.3.4:8080" (no auth, untouched)
|
||||
# strip query + fragment off a url before logging it
|
||||
mask_url("https://u:pw@api.x/v2?apiKey=SECRET&ip=8.8.8.8") # -> "https://u:pw@api.x/v2"
|
||||
mask_url("https://api.x/v2") # -> "https://api.x/v2" (unchanged)
|
||||
|
||||
# partial, user-reportable proxy id: creds stripped, ipv4 middle octets masked
|
||||
mask_proxy("172.58.32.1:8080") # -> "172.***.***.1:8080"
|
||||
mask_proxy("user:supersecret@172.58.32.1:8080") # -> "172.***.***.1:8080" (creds gone)
|
||||
mask_proxy("proxy.host.net:8080") # -> "proxy.host.net:8080" (hostname untouched)
|
||||
|
||||
# deterministic, regenerable id from ordered parts
|
||||
stable_id("profit_lounge", "user_123") # -> "d1f4…" (same 16 hex chars every call)
|
||||
```
|
||||
|
||||
`mask_url` strips `user:pass@` userinfo and replaces the values of sensitive query
|
||||
params (`apiKey`, `token`, `password`, `secret`, …; override via `keys=`) with `***`.
|
||||
`mask_proxy` bullets the password of a `host:port:user:password` spec. Non-URL /
|
||||
non-conforming input is returned unchanged.
|
||||
`mask_url` drops the query and fragment, keeping scheme/host/path. It does **not** parse
|
||||
or hunt for sensitive params — a secret in a URL is the caller's bug, not this function's
|
||||
to detect — so it structurally cannot leak a query param and cannot over-mask a legit one.
|
||||
A URL with no query/fragment is returned unchanged. (Note: `user:pass@` userinfo is part of
|
||||
the authority and is **not** stripped; keep credentials out of the URL you log.)
|
||||
|
||||
`mask_proxy` yields a partial identifier a user can safely report for debugging. Any
|
||||
`user:pass@` userinfo (or a `scheme://` prefix) is dropped entirely, then an IPv4 host
|
||||
`A.B.C.D` is masked to `A.***.***.D` with the port kept in full; a non-IPv4 host (hostname
|
||||
or IPv6) passes through unchanged with only its credentials stripped. A spec with no
|
||||
parseable host raises `ValueError`.
|
||||
|
||||
`stable_id` derives a fixed-length hex id from ordered string parts via SHA-256 — same
|
||||
inputs always produce the same id, no state, so it's regenerable anywhere. Parts join with
|
||||
a `\x00` separator so `("ab","c")` and `("a","bc")` differ; an empty or non-str part, or a
|
||||
non-positive `length`, raises `ValueError`.
|
||||
|
||||
## retry
|
||||
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "commons"
|
||||
version = "0.3.2"
|
||||
version = "1.0.0"
|
||||
description = "small stdlib-based sync helpers: time/timezone deltas, dotted-path dict access, display masking, ip/address tooling, and retry/backoff"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = []
|
||||
|
||||
+13
-20
@@ -1,26 +1,13 @@
|
||||
"""commons — small sync helpers shared across projects.
|
||||
|
||||
timing: unix-timestamp deltas + timezone-aware datetime conversions, bare
|
||||
functions and a configurable Clock.
|
||||
paths: nested dict/list access by dotted path (deep_get / deep_set); deep_set
|
||||
mirrors deep_get's list indexing, so a get/set round-trip on the same
|
||||
path never destroys a list.
|
||||
masking: display masking for cards / cvv / tokens, plus url/proxy credential
|
||||
redaction for logging (cosmetic, not a security control).
|
||||
addr: ip/address tooling — pure stdlib ip utils (`commons.addr.ip`) in base,
|
||||
async geo lookups (`commons.addr.geo`) behind the `commons[addr]` extra.
|
||||
retry: exponential-backoff retry, sync (`retry`) and async (`aretry`), call or
|
||||
decorator form; re-raises the last exception (fail loud), never swallows.
|
||||
"""commons - small sync helpers shared across projects: timing, paths, masking, addr, retry.
|
||||
|
||||
base is stdlib only, no dependencies (the addr geo lookups add aiohttp via the
|
||||
extra). to toggle the timing test mode for bare calls, set it on the module —
|
||||
`from commons import timing; timing.FAST_MODE = True` — or use `Clock(fast=True)`
|
||||
for instance-scoped control. addr is exposed as a submodule (`from commons import
|
||||
addr`); its ip helpers live under `commons.addr.ip` to keep top-level uncluttered.
|
||||
`commons[addr]` extra). see each submodule's docstring for its api.
|
||||
"""
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
|
||||
from . import addr, masking, paths, timing
|
||||
from .masking import credit, cvv, mask_proxy, mask_url, phantom, provider
|
||||
from .paths import deep_get, deep_set
|
||||
from .masking import credit, cvv, mask_proxy, mask_url, phantom, provider, stable_id
|
||||
from .paths import deep_get, deep_gets, deep_set, deep_sets
|
||||
from .retry import aretry, retry
|
||||
from .timing import (
|
||||
UTC,
|
||||
@@ -58,14 +45,20 @@ __all__ = [
|
||||
"Clock",
|
||||
"deep_get",
|
||||
"deep_set",
|
||||
"deep_gets",
|
||||
"deep_sets",
|
||||
"credit",
|
||||
"cvv",
|
||||
"phantom",
|
||||
"provider",
|
||||
"mask_url",
|
||||
"mask_proxy",
|
||||
"stable_id",
|
||||
"retry",
|
||||
"aretry",
|
||||
]
|
||||
|
||||
__version__ = "0.3.2"
|
||||
try:
|
||||
__version__ = version("commons")
|
||||
except PackageNotFoundError:
|
||||
__version__ = "0.0.0+unknown"
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
"""addr — ip/address tooling for commons.
|
||||
"""addr - ip/address tooling for commons.
|
||||
|
||||
two concerns:
|
||||
- `ip` — pure stdlib ipaddress utilities (validation, membership, cidr shape,
|
||||
conversions). ships in the base install, no dependencies.
|
||||
- `geo` — async network lookups (public ip, ip->geo, reverse geocode). needs
|
||||
aiohttp, gated behind the `commons[addr]` extra; importing this package is fine
|
||||
without it, but calling a geo function raises until it is installed.
|
||||
|
||||
the geo.ipify api_key is always injected by the caller — nothing is hardcoded.
|
||||
`ip`: pure stdlib ipaddress utilities, ships in the base install, no dependencies.
|
||||
`geo`: async network lookups (aiohttp), gated behind the `commons[addr]` extra;
|
||||
importing this package is fine without it, but calling a geo function raises until
|
||||
it is installed. the geo.ipify api_key is always injected by the caller, never
|
||||
hardcoded.
|
||||
"""
|
||||
from . import geo, ip
|
||||
from .geo import fetch_ip, fetch_location, ip_location
|
||||
|
||||
+7
-15
@@ -1,15 +1,10 @@
|
||||
"""async ip/geo network lookups (aiohttp, gated behind the [addr] extra).
|
||||
|
||||
importing this module without aiohttp is fine; only calling a lookup without it
|
||||
raises a clear RuntimeError. each call may reuse a caller-supplied aiohttp
|
||||
ClientSession (`session=`) or create and close one internally.
|
||||
|
||||
url-building and json->dict parsing are pulled into pure helpers (`_*_url`,
|
||||
`_parse_*`) so they unit-test without network. every lookup returns None on any
|
||||
request/parse failure (logged, never raised, never printed).
|
||||
|
||||
security: the only secret is geo.ipify's `api_key`, which is a REQUIRED keyword on
|
||||
`ip_location` — the caller injects it. nothing is hardcoded here.
|
||||
raises a clear RuntimeError. every lookup returns None on any request/parse
|
||||
failure (logged, never raised, never printed). the only secret is geo.ipify's
|
||||
`api_key`, a REQUIRED keyword on `ip_location` - the caller injects it, nothing
|
||||
is hardcoded here.
|
||||
"""
|
||||
import logging
|
||||
import unicodedata
|
||||
@@ -59,11 +54,8 @@ def _parse_ipify(data: dict) -> Optional[str]:
|
||||
|
||||
|
||||
def _state_slug(state) -> str:
|
||||
"""lowercase ascii-folded state slug with underscores (e.g. 'New York' -> 'new_york')
|
||||
|
||||
coerces to str first so a malformed non-string `state` doesn't raise TypeError out
|
||||
of unicodedata.normalize and break the 'None on any parse failure' contract.
|
||||
"""
|
||||
"""lowercase ascii-folded state slug with underscores (e.g. 'New York' -> 'new_york');
|
||||
coerces to str first so a non-string `state` can't TypeError out of the parse contract"""
|
||||
folded = unicodedata.normalize("NFKD", str(state)).encode("ascii", "ignore").decode("ascii")
|
||||
return folded.lower().replace(" ", "_")
|
||||
|
||||
@@ -121,7 +113,7 @@ async def ip_location(
|
||||
"""look up geo data for an ip via geo.ipify; api_key is required and injected
|
||||
|
||||
returns the raw geo.ipify json dict, or None on failure. raises ValueError if
|
||||
no api_key is supplied — there is no default and nothing hardcoded.
|
||||
no api_key is supplied - there is no default and nothing hardcoded.
|
||||
"""
|
||||
if not api_key:
|
||||
raise ValueError("ip_location requires an api_key (inject it; never hardcode)")
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
"""pure ip/address utilities over stdlib ipaddress (no network, no deps).
|
||||
|
||||
every function accepts strings. validation/membership helpers
|
||||
(`is_valid`/`version`/`in_network`/`in_any`) never raise on bad input — they return
|
||||
a falsy value. functions that require a valid address or cidr (`to_int`/`set_bits`/
|
||||
the network-shape helpers) let `ValueError` propagate so misuse is visible.
|
||||
|
||||
cidr parsing uses `ip_network(cidr, strict=False)` throughout, so host bits set in
|
||||
the network string are tolerated (e.g. "10.0.0.5/24").
|
||||
validation/membership helpers (`is_valid`/`version`/`in_network`/`in_any`) never
|
||||
raise on bad input - they return a falsy value. functions that require a valid
|
||||
address or cidr (`to_int`/`set_bits`/the network-shape helpers) let `ValueError`
|
||||
propagate so misuse is visible. cidr parsing uses `ip_network(cidr, strict=False)`
|
||||
throughout, so host bits set in the network string are tolerated (e.g. "10.0.0.5/24").
|
||||
"""
|
||||
import ipaddress
|
||||
from typing import List, Optional
|
||||
@@ -117,7 +115,7 @@ def hosts(cidr: str, *, limit: Optional[int] = None) -> List[str]:
|
||||
gen = ipaddress.ip_network(cidr, strict=False).hosts()
|
||||
out: List[str] = []
|
||||
for host in gen:
|
||||
out.append(str(host))
|
||||
if limit is not None and len(out) >= limit:
|
||||
break
|
||||
out.append(str(host))
|
||||
return out
|
||||
|
||||
+73
-42
@@ -1,19 +1,18 @@
|
||||
"""display masking for sensitive-looking values.
|
||||
"""display masking for sensitive-looking values (e.g. "•••• •••• •••• 1234").
|
||||
|
||||
these are DISPLAY helpers only — they format a value for showing in a UI or log
|
||||
(e.g. "•••• •••• •••• 1234"). they are not a security control: the underlying
|
||||
value is unchanged and still needs proper handling (encryption at rest, etc.).
|
||||
DISPLAY only, not a security control: the underlying value is unchanged and still
|
||||
needs proper handling (encryption at rest, etc.).
|
||||
|
||||
card helpers: `credit`, `cvv`, `provider`. token: `phantom`. url/proxy redaction
|
||||
for logging connection strings: `mask_url`, `mask_proxy`.
|
||||
- ``credit``/``cvv``/``phantom``/``provider`` mask card-shaped values for display.
|
||||
- ``mask_url`` strips the query and fragment off a url (keeping scheme/host/path),
|
||||
so a url can be logged without leaking whatever a caller stuck in its params - it
|
||||
does NOT hunt for tokens; a secret in a url is the caller's bug, not this to detect.
|
||||
- ``mask_proxy`` gives a partial, user-reportable proxy identifier: credentials are
|
||||
stripped and an ipv4 host's middle octets are masked (first/last octet + port kept).
|
||||
- ``stable_id`` derives a deterministic, regenerable id from ordered string parts.
|
||||
"""
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
SENSITIVE_QUERY_KEYS = frozenset(
|
||||
{"apikey", "api_key", "key", "token", "access_token", "refresh_token",
|
||||
"auth", "password", "passwd", "pwd", "pass", "secret", "client_secret",
|
||||
"sig", "signature", "session"}
|
||||
)
|
||||
import hashlib
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
|
||||
def _digits(value: str) -> str:
|
||||
@@ -35,7 +34,7 @@ def cvv(value: str) -> str:
|
||||
def phantom(value: str) -> str:
|
||||
"""show a long token as first-six...last-four (e.g. a hash or id)
|
||||
|
||||
values of length <= 10 fully mask instead — first6+last4 would otherwise
|
||||
values of length <= 10 fully mask instead - first6+last4 would otherwise
|
||||
reveal (or double-reveal) every character while still looking masked.
|
||||
"""
|
||||
if len(value) <= 10:
|
||||
@@ -46,7 +45,7 @@ def phantom(value: str) -> str:
|
||||
def provider(card_number: str) -> str:
|
||||
"""short card brand from the number's prefix, or UNKW if undetermined
|
||||
|
||||
a BIN-prefix heuristic for display/labeling — not authoritative validation.
|
||||
a BIN-prefix heuristic for display/labeling - not authoritative validation.
|
||||
tolerates spaces/dashes and short or non-numeric input (returns UNKW).
|
||||
"""
|
||||
n = _digits(card_number)
|
||||
@@ -69,41 +68,73 @@ def provider(card_number: str) -> str:
|
||||
return "UNKW"
|
||||
|
||||
|
||||
def mask_url(url: str, *, keys: "frozenset[str] | None" = None) -> str:
|
||||
"""redact credentials in a url for logging: drop userinfo, ``***`` sensitive query values
|
||||
def mask_url(url: str) -> str:
|
||||
"""strip query and fragment from a url, keeping scheme, host and path
|
||||
|
||||
param-name matching is case-insensitive; ``keys`` overrides the default sensitive-name
|
||||
set. non-url or unparseable input is returned unchanged.
|
||||
does NOT parse or hunt for sensitive params - it simply drops everything after the
|
||||
path, so it structurally cannot leak a query param and cannot over-mask a legit one.
|
||||
a url with no query/fragment is returned unchanged; a url urlsplit cannot parse
|
||||
(a malformed ipv6 literal) is also returned unchanged rather than crashing the
|
||||
display/log call this feeds.
|
||||
"""
|
||||
sensitive = SENSITIVE_QUERY_KEYS if keys is None else frozenset(k.lower() for k in keys)
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
except ValueError:
|
||||
return url
|
||||
if not parts.scheme and not parts.netloc:
|
||||
return url
|
||||
netloc = parts.netloc
|
||||
if "@" in netloc:
|
||||
netloc = netloc.rsplit("@", 1)[1]
|
||||
query = parts.query
|
||||
if query:
|
||||
pairs = parse_qsl(query, keep_blank_values=True)
|
||||
query = urlencode(
|
||||
[(k, "***" if k.lower() in sensitive else v) for k, v in pairs],
|
||||
quote_via=lambda s, *_: s,
|
||||
)
|
||||
return urlunsplit((parts.scheme, netloc, parts.path, query, parts.fragment))
|
||||
return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
|
||||
|
||||
|
||||
def _is_ipv4(host: str) -> bool:
|
||||
"""true if host is a dotted-quad ipv4 literal (four 0-255 octets)"""
|
||||
octets = host.split(".")
|
||||
if len(octets) != 4:
|
||||
return False
|
||||
for octet in octets:
|
||||
if not (octet.isdigit() and octet.isascii()):
|
||||
return False
|
||||
if not 0 <= int(octet) <= 255:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def mask_proxy(spec: str) -> str:
|
||||
"""redact the password of a ``host:port:user:password`` proxy string to ``:****``
|
||||
"""mask the middle octets of a proxy ip, keeping first/last octet and full port; strips any credentials
|
||||
|
||||
a plain ``host:port`` (no auth) and any other non-conforming input pass through unchanged.
|
||||
drops a ``scheme://`` prefix and any ``user:pass@`` userinfo entirely (a credential is
|
||||
removed, never partial-masked). an ipv4 host ``A.B.C.D`` becomes ``A.***.***.D`` with the
|
||||
port kept as-is (``172.58.32.1:8080`` -> ``172.***.***.1:8080``); a non-ipv4 host (hostname,
|
||||
ipv6) passes through unchanged with only its credentials stripped. raises ValueError on a
|
||||
spec with no parseable host.
|
||||
"""
|
||||
parts = spec.split(":")
|
||||
if len(parts) == 2:
|
||||
return spec
|
||||
if len(parts) == 4:
|
||||
host, port, user, _ = parts
|
||||
return f"{host}:{port}:{user}:****"
|
||||
return spec
|
||||
rest = spec
|
||||
if "://" in rest:
|
||||
rest = rest.split("://", 1)[1]
|
||||
if "@" in rest:
|
||||
rest = rest.rsplit("@", 1)[1]
|
||||
host, sep, port = rest.partition(":")
|
||||
if not host:
|
||||
raise ValueError(f"mask_proxy: no host in proxy spec {spec!r}")
|
||||
if _is_ipv4(host):
|
||||
octets = host.split(".")
|
||||
host = f"{octets[0]}.***.***.{octets[3]}"
|
||||
return f"{host}:{port}" if sep else host
|
||||
|
||||
|
||||
def stable_id(*parts: str, length: int = 16) -> str:
|
||||
"""deterministic id from ordered parts; same inputs always produce the same id
|
||||
|
||||
joins the parts with a ``\\x00`` separator (so ``("ab","c")`` != ``("a","bc")``) and
|
||||
returns the leading ``length`` hex chars of their sha256. raises ValueError on no parts,
|
||||
an empty part, a non-str part, or a non-positive length - fail loud rather than emit a
|
||||
weak or constant id.
|
||||
"""
|
||||
if length <= 0:
|
||||
raise ValueError(f"stable_id: length must be positive, got {length}")
|
||||
if not parts:
|
||||
raise ValueError("stable_id: needs at least one part")
|
||||
for part in parts:
|
||||
if not isinstance(part, str):
|
||||
raise ValueError(f"stable_id: parts must be str, got {type(part).__name__}")
|
||||
if part == "":
|
||||
raise ValueError("stable_id: parts must be non-empty")
|
||||
return hashlib.sha256("\x00".join(parts).encode()).hexdigest()[:length]
|
||||
|
||||
+126
-15
@@ -1,15 +1,17 @@
|
||||
"""nested access by dotted path.
|
||||
"""nested dict/list access by dotted path
|
||||
|
||||
`deep_get(data, "in.this.old.notation")` walks dicts (and lists, when a segment is
|
||||
a number) and returns a default instead of raising on a missing/!wrong path.
|
||||
`deep_set` writes a nested value, creating intermediate dicts and indexing into
|
||||
existing lists on numeric segments (mirroring deep_get), so a get/set round-trip
|
||||
on the same path never corrupts a list. an out-of-range numeric segment raises
|
||||
IndexError rather than silently mis-storing.
|
||||
- deep_get/deep_set: single path, scalar in/out. numeric segment indexes a list;
|
||||
setting into a tuple raises (immutable).
|
||||
- deep_gets/deep_sets: `*` segments iterate every element at a level (bulk get -> list,
|
||||
bulk set -> value or fn(current)->new); multiple `*` fan out cartesian.
|
||||
|
||||
a `*` in the scalar verbs raises ValueError - use the plural verbs so the return type
|
||||
is never ambiguous.
|
||||
"""
|
||||
from typing import Any
|
||||
from typing import Any, Callable, List, Union
|
||||
|
||||
_MISSING = object()
|
||||
_WILDCARD = "*"
|
||||
|
||||
|
||||
def deep_get(data: Any, path: str, default: Any = None, *, sep: str = ".") -> Any:
|
||||
@@ -17,10 +19,14 @@ def deep_get(data: Any, path: str, default: Any = None, *, sep: str = ".") -> An
|
||||
|
||||
dict keys are matched by name; a numeric segment indexes a list/tuple
|
||||
(e.g. "items.0.id"). any missing key, out-of-range index, or non-container
|
||||
along the way yields `default` rather than raising.
|
||||
along the way yields `default` rather than raising. a `*` segment raises
|
||||
ValueError - use deep_gets for wildcard paths.
|
||||
"""
|
||||
segments = path.split(sep)
|
||||
if _WILDCARD in segments:
|
||||
raise ValueError(f"deep_get: wildcard '*' in path {path!r}; use deep_gets for wildcard paths")
|
||||
cur = data
|
||||
for seg in path.split(sep):
|
||||
for seg in segments:
|
||||
if isinstance(cur, dict):
|
||||
cur = cur.get(seg, _MISSING)
|
||||
if cur is _MISSING:
|
||||
@@ -38,29 +44,35 @@ def deep_get(data: Any, path: str, default: Any = None, *, sep: str = ".") -> An
|
||||
def deep_set(data: dict, path: str, value: Any, *, sep: str = ".") -> dict:
|
||||
"""set a nested value by dotted path, creating intermediate dicts; returns data for chaining
|
||||
|
||||
mirrors deep_get's list indexing (a numeric segment updates an existing list/tuple
|
||||
element rather than corrupting it); unlike deep_get, an out-of-range index raises
|
||||
IndexError instead of silently mis-storing, since there's no safe default to fall back to.
|
||||
a numeric segment updates a LIST element in place (out-of-range raises IndexError);
|
||||
setting through or into a tuple raises TypeError (immutable). a `*` raises ValueError -
|
||||
use deep_sets for wildcard paths.
|
||||
"""
|
||||
segments = path.split(sep)
|
||||
if _WILDCARD in segments:
|
||||
raise ValueError(f"deep_set: wildcard '*' in path {path!r}; use deep_sets for wildcard paths")
|
||||
cur = data
|
||||
for seg in segments[:-1]:
|
||||
if isinstance(cur, tuple):
|
||||
raise TypeError(f"deep_set: cannot set into an immutable tuple at {path!r}")
|
||||
if isinstance(cur, list):
|
||||
idx = int(seg)
|
||||
if not -len(cur) <= idx < len(cur):
|
||||
raise IndexError(f"deep_set: index {seg!r} out of range for list of length {len(cur)}")
|
||||
nxt = cur[idx]
|
||||
if not isinstance(nxt, (dict, list)):
|
||||
if not isinstance(nxt, (dict, list, tuple)):
|
||||
nxt = {}
|
||||
cur[idx] = nxt
|
||||
cur = nxt
|
||||
else:
|
||||
nxt = cur.get(seg)
|
||||
if not isinstance(nxt, (dict, list)):
|
||||
if not isinstance(nxt, (dict, list, tuple)):
|
||||
nxt = {}
|
||||
cur[seg] = nxt
|
||||
cur = nxt
|
||||
last = segments[-1]
|
||||
if isinstance(cur, tuple):
|
||||
raise TypeError(f"deep_set: cannot set into an immutable tuple at {path!r}")
|
||||
if isinstance(cur, list):
|
||||
idx = int(last)
|
||||
if not -len(cur) <= idx < len(cur):
|
||||
@@ -69,3 +81,102 @@ def deep_set(data: dict, path: str, value: Any, *, sep: str = ".") -> dict:
|
||||
else:
|
||||
cur[last] = value
|
||||
return data
|
||||
|
||||
|
||||
def _iter_children(node: Any):
|
||||
"""yield (key, child) pairs for a `*` step: dict items, or list/tuple index/item pairs"""
|
||||
if isinstance(node, dict):
|
||||
yield from node.items()
|
||||
elif isinstance(node, (list, tuple)):
|
||||
yield from enumerate(node)
|
||||
|
||||
|
||||
def deep_gets(data: Any, path: str, *, sep: str = ".") -> List[Any]:
|
||||
"""get every value matching a dotted path with `*` wildcards, as a list (may be empty)
|
||||
|
||||
a `*` iterates every element at that level; multiple `*` fan out cartesian. a branch
|
||||
that dead-ends (missing key, out-of-range index, non-container) is skipped, not raised.
|
||||
a path with no `*` returns a one- or zero-element list.
|
||||
"""
|
||||
segments = path.split(sep)
|
||||
frontier = [data]
|
||||
for seg in segments:
|
||||
nxt: List[Any] = []
|
||||
if seg == _WILDCARD:
|
||||
for node in frontier:
|
||||
for _, child in _iter_children(node):
|
||||
nxt.append(child)
|
||||
else:
|
||||
for node in frontier:
|
||||
if isinstance(node, dict):
|
||||
got = node.get(seg, _MISSING)
|
||||
if got is not _MISSING:
|
||||
nxt.append(got)
|
||||
elif isinstance(node, (list, tuple)):
|
||||
try:
|
||||
nxt.append(node[int(seg)])
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
frontier = nxt
|
||||
return frontier
|
||||
|
||||
|
||||
def deep_sets(data: Any, path: str, value: "Union[Any, Callable[[Any], Any]]", *, sep: str = ".") -> Any:
|
||||
"""set every value matching a dotted path with `*` wildcards; returns data for chaining
|
||||
|
||||
`value` is a plain value written to every match, or a callable `fn(current) -> new`
|
||||
(a bare callable is always invoked - to store one, wrap it `lambda _c, f=fn: f`). a `*`
|
||||
iterates every element at that level; multiple `*` fan out cartesian. best-effort: any
|
||||
dead-end (missing key, out-of-range index) is skipped, not raised. setting through or
|
||||
into a tuple raises TypeError (immutable).
|
||||
"""
|
||||
compute: Callable[[Any], Any] = value if callable(value) else (lambda _cur, _v=value: _v)
|
||||
segments = path.split(sep)
|
||||
parents = segments[:-1]
|
||||
last = segments[-1]
|
||||
|
||||
frontier = [data]
|
||||
for seg in parents:
|
||||
nxt: List[Any] = []
|
||||
if seg == _WILDCARD:
|
||||
for node in frontier:
|
||||
if isinstance(node, tuple):
|
||||
raise TypeError(f"deep_sets: cannot set through an immutable tuple at {path!r}")
|
||||
for _, child in _iter_children(node):
|
||||
nxt.append(child)
|
||||
else:
|
||||
for node in frontier:
|
||||
if isinstance(node, tuple):
|
||||
raise TypeError(f"deep_sets: cannot set through an immutable tuple at {path!r}")
|
||||
if isinstance(node, dict):
|
||||
got = node.get(seg, _MISSING)
|
||||
if got is not _MISSING:
|
||||
nxt.append(got)
|
||||
elif isinstance(node, list):
|
||||
try:
|
||||
nxt.append(node[int(seg)])
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
frontier = nxt
|
||||
|
||||
for node in frontier:
|
||||
if isinstance(node, tuple):
|
||||
raise TypeError(f"deep_sets: cannot set into an immutable tuple at {path!r}")
|
||||
if last == _WILDCARD:
|
||||
if isinstance(node, dict):
|
||||
for key in list(node.keys()):
|
||||
node[key] = compute(node[key])
|
||||
elif isinstance(node, list):
|
||||
for idx in range(len(node)):
|
||||
node[idx] = compute(node[idx])
|
||||
elif isinstance(node, dict):
|
||||
if node.get(last, _MISSING) is not _MISSING:
|
||||
node[last] = compute(node[last])
|
||||
elif isinstance(node, list):
|
||||
try:
|
||||
idx = int(last)
|
||||
except ValueError:
|
||||
continue
|
||||
if -len(node) <= idx < len(node):
|
||||
node[idx] = compute(node[idx])
|
||||
return data
|
||||
|
||||
+3
-21
@@ -1,25 +1,7 @@
|
||||
"""retry with exponential backoff — sync and async, one backoff engine.
|
||||
|
||||
de-duplicates retry logic that was written divergently across several libs (HTTP
|
||||
429/5xx caps, proxy burn/rotate caps, IMAP reconnects). both a call form and a
|
||||
decorator form share one implementation per flavor; the backoff schedule is a pure
|
||||
generator so it tests without real sleeps.
|
||||
|
||||
from commons import retry, aretry
|
||||
|
||||
# call form
|
||||
rows = retry(lambda: db_read(), attempts=5, on=(IOError,))
|
||||
data = await aretry(lambda: fetch(url), attempts=3, on=(TimeoutError,))
|
||||
|
||||
# decorator form (same kwargs)
|
||||
@aretry(attempts=4, backoff=0.5, on=(ConnectionError,))
|
||||
async def pull():
|
||||
...
|
||||
"""retry with exponential backoff, sync (`retry`) and async (`aretry`), call or decorator form.
|
||||
|
||||
after `attempts` are exhausted the LAST exception is re-raised (fail loud, never
|
||||
swallowed). `on` narrows which exceptions retry; `give_up(exc) -> bool` stops early
|
||||
on a non-retryable error (e.g. a 400 vs a 429). each retry is logged (emit-only),
|
||||
never printed.
|
||||
swallowed). see README for usage examples.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -38,7 +20,7 @@ def _delays(attempts: int, backoff: float, factor: float, max_backoff: float):
|
||||
"""yield the wait before each retry: min(backoff * factor**n, max_backoff)
|
||||
|
||||
yields `attempts - 1` delays (one before each retry after the first try). the
|
||||
raw, un-jittered schedule — jitter is applied at call time so the schedule stays
|
||||
raw, un-jittered schedule - jitter is applied at call time so the schedule stays
|
||||
pure and testable.
|
||||
"""
|
||||
for n in range(max(0, attempts - 1)):
|
||||
|
||||
+6
-12
@@ -1,22 +1,16 @@
|
||||
"""time helpers built on unix timestamps with timezone-aware datetime support.
|
||||
|
||||
one engine, two ergonomics:
|
||||
- bare module functions (now/add/ahead/ago/is_expired/to_dt/...) operate on unix
|
||||
ints for quick, stateless use. all delta math routes through `_delta_seconds`.
|
||||
- `Clock` holds a timezone + fast flag and delegates to the same functions, so a
|
||||
configured clock and the bare functions never diverge.
|
||||
|
||||
unix ints stay the storable value; datetimes are produced on demand in whatever
|
||||
timezone you ask for (stdlib zoneinfo, no pytz).
|
||||
one engine, two ergonomics: bare module functions for stateless unix-int math
|
||||
(all delta math routes through `_delta_seconds`, the single source of truth), and
|
||||
`Clock`, which holds a timezone + fast flag and delegates to the same functions.
|
||||
"""
|
||||
import time as _time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Union
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
# default fast flag for bare module calls. when True, every unit (day/hour/minute)
|
||||
# counts as one second so time-based flows run fast in tests. set
|
||||
# `commons.timing.FAST_MODE` from test setup; off by default. not a config import.
|
||||
# default fast flag for bare calls (test-only: collapses each unit to 1s); set
|
||||
# `commons.timing.FAST_MODE` from test setup, off by default, not a config import.
|
||||
FAST_MODE = False
|
||||
|
||||
UTC = timezone.utc
|
||||
@@ -42,7 +36,7 @@ def now() -> int:
|
||||
def _delta_seconds(days: int, hours: int, minutes: int, seconds: int, fast: bool) -> int:
|
||||
"""seconds for a unit combination; collapses to 1s/unit when fast is True
|
||||
|
||||
single source of truth for unit math — every delta helper routes through here.
|
||||
single source of truth for unit math - every delta helper routes through here.
|
||||
"""
|
||||
if fast:
|
||||
return days + hours + minutes + seconds
|
||||
|
||||
Reference in New Issue
Block a user