Compare commits
10
Commits
4f835ff003
...
v1.0.0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bbfcc4818b | ||
|
|
a6bd95bda7 | ||
|
|
a1702eede8 | ||
|
|
0030daeb7b | ||
|
|
e6e655335e | ||
|
|
7fa5916eda | ||
|
|
718c8a79b0 | ||
|
|
33ade498da | ||
|
|
e86184986f | ||
|
|
18646f313c |
@@ -12,15 +12,15 @@ Small sync helpers shared across projects. Base is stdlib only — **no dependen
|
|||||||
## Install
|
## Install
|
||||||
|
|
||||||
```
|
```
|
||||||
commons @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.3.2
|
commons @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v1.0.0
|
||||||
# async address/geo lookups (fetch_ip / ip_location / fetch_location) need the extra:
|
# async address/geo lookups (fetch_ip / ip_location / fetch_location) need the extra:
|
||||||
commons[addr] @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.3.2
|
commons[addr] @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v1.0.0
|
||||||
```
|
```
|
||||||
|
|
||||||
The base install pulls **nothing** (stdlib). Only `commons[addr]` adds `aiohttp`, and
|
The base install pulls **nothing** (stdlib). Only `commons[addr]` adds `aiohttp`, and
|
||||||
only for the geo lookups — the pure `commons.addr.ip` utilities ship in base.
|
only for the geo lookups — the pure `commons.addr.ip` utilities ship in base.
|
||||||
|
|
||||||
Drop the `@v0.3.2` suffix from the line above to install the latest unpinned.
|
Drop the `@v1.0.0` suffix from the line above to install the latest unpinned.
|
||||||
|
|
||||||
## timing
|
## timing
|
||||||
|
|
||||||
@@ -89,13 +89,17 @@ timing.FAST_MODE = True # in test setup
|
|||||||
|
|
||||||
## paths
|
## paths
|
||||||
|
|
||||||
|
Two pairs of verbs: **single-path** (scalar in/out) and **wildcard** (bulk, always a list).
|
||||||
|
|
||||||
|
### single-path — `deep_get` / `deep_set`
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from commons import deep_get, deep_set
|
from commons import deep_get, deep_set
|
||||||
|
|
||||||
data = {"in": {"this": {"old": {"notation": 42}}}, "items": [{"id": "a"}, {"id": "b"}]}
|
data = {"in": {"this": {"old": {"notation": 42}}}, "items": [{"id": "a"}, {"id": "b"}]}
|
||||||
|
|
||||||
deep_get(data, "in.this.old.notation") # 42
|
deep_get(data, "in.this.old.notation") # 42
|
||||||
deep_get(data, "items.1.id") # "b" (numeric segment indexes a list)
|
deep_get(data, "items.1.id") # "b" (numeric segment indexes a list/tuple)
|
||||||
deep_get(data, "in.nope.here", "DEF") # "DEF" (missing -> default, no raise)
|
deep_get(data, "in.nope.here", "DEF") # "DEF" (missing -> default, no raise)
|
||||||
|
|
||||||
deep_set({}, "a.b.c", 9) # {"a": {"b": {"c": 9}}}
|
deep_set({}, "a.b.c", 9) # {"a": {"b": {"c": 9}}}
|
||||||
@@ -105,10 +109,34 @@ deep_set(data, "items.9.id", "X") # raises IndexError (out of range, no
|
|||||||
# silent mis-store)
|
# silent mis-store)
|
||||||
```
|
```
|
||||||
|
|
||||||
`deep_set` mirrors `deep_get`'s list indexing: a numeric segment over an existing
|
`deep_get` steps into both lists and tuples on a numeric segment. `deep_set` updates a
|
||||||
list/tuple updates that element in place rather than replacing the list with a dict.
|
**list** element in place; setting into (or through) a **tuple** raises `TypeError` —
|
||||||
An out-of-range numeric segment raises `IndexError` instead of silently corrupting
|
tuples are immutable, so "update in place" is impossible; flatten to a list first. An
|
||||||
the structure.
|
out-of-range numeric segment raises `IndexError` instead of silently corrupting the
|
||||||
|
structure.
|
||||||
|
|
||||||
|
### wildcard — `deep_gets` / `deep_sets`
|
||||||
|
|
||||||
|
A `*` segment iterates every element at that level (dict values or list/tuple items);
|
||||||
|
multiple `*` fan out cartesian. These are **separate verbs** with a fixed list/bulk
|
||||||
|
return type — a `*` passed to `deep_get`/`deep_set` raises `ValueError` (use the plural
|
||||||
|
verb).
|
||||||
|
|
||||||
|
```python
|
||||||
|
from commons import deep_gets, deep_sets
|
||||||
|
|
||||||
|
data = {"users": [{"username": "al"}, {"username": "bo"}, {"username": "cy"}]}
|
||||||
|
|
||||||
|
deep_gets(data, "users.*.username") # ["al", "bo", "cy"] (ALWAYS a list, in order)
|
||||||
|
deep_gets(data, "users.*.nope") # [] (no match -> empty list, no raise)
|
||||||
|
|
||||||
|
deep_sets(data, "users.*.username", "X") # set every match to "X"
|
||||||
|
deep_sets(data, "users.*.username", str.upper) # callable fn(current)->new per match
|
||||||
|
```
|
||||||
|
|
||||||
|
`deep_sets`'s second arg is either a plain value (write it to every match) or a callable
|
||||||
|
`fn(current) -> new` (compute each). It writes only matches that already exist (it does
|
||||||
|
not create missing keys). Setting through/into a tuple raises `TypeError`.
|
||||||
|
|
||||||
## masking
|
## masking
|
||||||
|
|
||||||
@@ -116,7 +144,7 @@ Display helpers only — they format a value for showing; they are **not a secur
|
|||||||
control** (the underlying value is unchanged and still needs proper handling).
|
control** (the underlying value is unchanged and still needs proper handling).
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from commons import credit, cvv, phantom, provider, mask_url, mask_proxy
|
from commons import credit, cvv, phantom, provider, mask_url, mask_proxy, stable_id
|
||||||
|
|
||||||
credit("4111 1111 1111 1234") # "•••• •••• •••• 1234"
|
credit("4111 1111 1111 1234") # "•••• •••• •••• 1234"
|
||||||
cvv("123") # "•••"
|
cvv("123") # "•••"
|
||||||
@@ -125,18 +153,35 @@ phantom("1234567890") # "••••••••••" (len <= 10
|
|||||||
provider("4111111111111111") # "VISA" (VISA/MC/AMEX/UPAY/DISC/JCB/DNRS/UNKW)
|
provider("4111111111111111") # "VISA" (VISA/MC/AMEX/UPAY/DISC/JCB/DNRS/UNKW)
|
||||||
provider("4111²111111111234") # "VISA" (unicode digit lookalikes ignored, never raises)
|
provider("4111²111111111234") # "VISA" (unicode digit lookalikes ignored, never raises)
|
||||||
|
|
||||||
# redact credentials in a connection string before logging it
|
# strip query + fragment off a url before logging it
|
||||||
mask_url("https://u:pw@api.x/v2?apiKey=SECRET&ip=8.8.8.8")
|
mask_url("https://u:pw@api.x/v2?apiKey=SECRET&ip=8.8.8.8") # -> "https://u:pw@api.x/v2"
|
||||||
# -> "https://api.x/v2?apiKey=***&ip=8.8.8.8" (userinfo dropped, secret query masked)
|
mask_url("https://api.x/v2") # -> "https://api.x/v2" (unchanged)
|
||||||
mask_url("redis://:pw@127.0.0.1:6379/0") # -> "redis://127.0.0.1:6379/0"
|
|
||||||
mask_proxy("1.2.3.4:8080:user:supersecret") # -> "1.2.3.4:8080:user:****"
|
# partial, user-reportable proxy id: creds stripped, ipv4 middle octets masked
|
||||||
mask_proxy("1.2.3.4:8080") # -> "1.2.3.4:8080" (no auth, untouched)
|
mask_proxy("172.58.32.1:8080") # -> "172.***.***.1:8080"
|
||||||
|
mask_proxy("user:supersecret@172.58.32.1:8080") # -> "172.***.***.1:8080" (creds gone)
|
||||||
|
mask_proxy("proxy.host.net:8080") # -> "proxy.host.net:8080" (hostname untouched)
|
||||||
|
|
||||||
|
# deterministic, regenerable id from ordered parts
|
||||||
|
stable_id("profit_lounge", "user_123") # -> "d1f4…" (same 16 hex chars every call)
|
||||||
```
|
```
|
||||||
|
|
||||||
`mask_url` strips `user:pass@` userinfo and replaces the values of sensitive query
|
`mask_url` drops the query and fragment, keeping scheme/host/path. It does **not** parse
|
||||||
params (`apiKey`, `token`, `password`, `secret`, …; override via `keys=`) with `***`.
|
or hunt for sensitive params — a secret in a URL is the caller's bug, not this function's
|
||||||
`mask_proxy` bullets the password of a `host:port:user:password` spec. Non-URL /
|
to detect — so it structurally cannot leak a query param and cannot over-mask a legit one.
|
||||||
non-conforming input is returned unchanged.
|
A URL with no query/fragment is returned unchanged. (Note: `user:pass@` userinfo is part of
|
||||||
|
the authority and is **not** stripped; keep credentials out of the URL you log.)
|
||||||
|
|
||||||
|
`mask_proxy` yields a partial identifier a user can safely report for debugging. Any
|
||||||
|
`user:pass@` userinfo (or a `scheme://` prefix) is dropped entirely, then an IPv4 host
|
||||||
|
`A.B.C.D` is masked to `A.***.***.D` with the port kept in full; a non-IPv4 host (hostname
|
||||||
|
or IPv6) passes through unchanged with only its credentials stripped. A spec with no
|
||||||
|
parseable host raises `ValueError`.
|
||||||
|
|
||||||
|
`stable_id` derives a fixed-length hex id from ordered string parts via SHA-256 — same
|
||||||
|
inputs always produce the same id, no state, so it's regenerable anywhere. Parts join with
|
||||||
|
a `\x00` separator so `("ab","c")` and `("a","bc")` differ; an empty or non-str part, or a
|
||||||
|
non-positive `length`, raises `ValueError`.
|
||||||
|
|
||||||
## retry
|
## retry
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "commons"
|
name = "commons"
|
||||||
version = "0.3.2"
|
version = "1.0.0"
|
||||||
description = "small stdlib-based sync helpers: time/timezone deltas, dotted-path dict access, display masking, ip/address tooling, and retry/backoff"
|
description = "small stdlib-based sync helpers: time/timezone deltas, dotted-path dict access, display masking, ip/address tooling, and retry/backoff"
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
dependencies = []
|
dependencies = []
|
||||||
|
|||||||
+13
-20
@@ -1,26 +1,13 @@
|
|||||||
"""commons — small sync helpers shared across projects.
|
"""commons - small sync helpers shared across projects: timing, paths, masking, addr, retry.
|
||||||
|
|
||||||
timing: unix-timestamp deltas + timezone-aware datetime conversions, bare
|
|
||||||
functions and a configurable Clock.
|
|
||||||
paths: nested dict/list access by dotted path (deep_get / deep_set); deep_set
|
|
||||||
mirrors deep_get's list indexing, so a get/set round-trip on the same
|
|
||||||
path never destroys a list.
|
|
||||||
masking: display masking for cards / cvv / tokens, plus url/proxy credential
|
|
||||||
redaction for logging (cosmetic, not a security control).
|
|
||||||
addr: ip/address tooling — pure stdlib ip utils (`commons.addr.ip`) in base,
|
|
||||||
async geo lookups (`commons.addr.geo`) behind the `commons[addr]` extra.
|
|
||||||
retry: exponential-backoff retry, sync (`retry`) and async (`aretry`), call or
|
|
||||||
decorator form; re-raises the last exception (fail loud), never swallows.
|
|
||||||
|
|
||||||
base is stdlib only, no dependencies (the addr geo lookups add aiohttp via the
|
base is stdlib only, no dependencies (the addr geo lookups add aiohttp via the
|
||||||
extra). to toggle the timing test mode for bare calls, set it on the module —
|
`commons[addr]` extra). see each submodule's docstring for its api.
|
||||||
`from commons import timing; timing.FAST_MODE = True` — or use `Clock(fast=True)`
|
|
||||||
for instance-scoped control. addr is exposed as a submodule (`from commons import
|
|
||||||
addr`); its ip helpers live under `commons.addr.ip` to keep top-level uncluttered.
|
|
||||||
"""
|
"""
|
||||||
|
from importlib.metadata import PackageNotFoundError, version
|
||||||
|
|
||||||
from . import addr, masking, paths, timing
|
from . import addr, masking, paths, timing
|
||||||
from .masking import credit, cvv, mask_proxy, mask_url, phantom, provider
|
from .masking import credit, cvv, mask_proxy, mask_url, phantom, provider, stable_id
|
||||||
from .paths import deep_get, deep_set
|
from .paths import deep_get, deep_gets, deep_set, deep_sets
|
||||||
from .retry import aretry, retry
|
from .retry import aretry, retry
|
||||||
from .timing import (
|
from .timing import (
|
||||||
UTC,
|
UTC,
|
||||||
@@ -58,14 +45,20 @@ __all__ = [
|
|||||||
"Clock",
|
"Clock",
|
||||||
"deep_get",
|
"deep_get",
|
||||||
"deep_set",
|
"deep_set",
|
||||||
|
"deep_gets",
|
||||||
|
"deep_sets",
|
||||||
"credit",
|
"credit",
|
||||||
"cvv",
|
"cvv",
|
||||||
"phantom",
|
"phantom",
|
||||||
"provider",
|
"provider",
|
||||||
"mask_url",
|
"mask_url",
|
||||||
"mask_proxy",
|
"mask_proxy",
|
||||||
|
"stable_id",
|
||||||
"retry",
|
"retry",
|
||||||
"aretry",
|
"aretry",
|
||||||
]
|
]
|
||||||
|
|
||||||
__version__ = "0.3.2"
|
try:
|
||||||
|
__version__ = version("commons")
|
||||||
|
except PackageNotFoundError:
|
||||||
|
__version__ = "0.0.0+unknown"
|
||||||
|
|||||||
@@ -1,13 +1,10 @@
|
|||||||
"""addr — ip/address tooling for commons.
|
"""addr - ip/address tooling for commons.
|
||||||
|
|
||||||
two concerns:
|
`ip`: pure stdlib ipaddress utilities, ships in the base install, no dependencies.
|
||||||
- `ip` — pure stdlib ipaddress utilities (validation, membership, cidr shape,
|
`geo`: async network lookups (aiohttp), gated behind the `commons[addr]` extra;
|
||||||
conversions). ships in the base install, no dependencies.
|
importing this package is fine without it, but calling a geo function raises until
|
||||||
- `geo` — async network lookups (public ip, ip->geo, reverse geocode). needs
|
it is installed. the geo.ipify api_key is always injected by the caller, never
|
||||||
aiohttp, gated behind the `commons[addr]` extra; importing this package is fine
|
hardcoded.
|
||||||
without it, but calling a geo function raises until it is installed.
|
|
||||||
|
|
||||||
the geo.ipify api_key is always injected by the caller — nothing is hardcoded.
|
|
||||||
"""
|
"""
|
||||||
from . import geo, ip
|
from . import geo, ip
|
||||||
from .geo import fetch_ip, fetch_location, ip_location
|
from .geo import fetch_ip, fetch_location, ip_location
|
||||||
|
|||||||
+7
-15
@@ -1,15 +1,10 @@
|
|||||||
"""async ip/geo network lookups (aiohttp, gated behind the [addr] extra).
|
"""async ip/geo network lookups (aiohttp, gated behind the [addr] extra).
|
||||||
|
|
||||||
importing this module without aiohttp is fine; only calling a lookup without it
|
importing this module without aiohttp is fine; only calling a lookup without it
|
||||||
raises a clear RuntimeError. each call may reuse a caller-supplied aiohttp
|
raises a clear RuntimeError. every lookup returns None on any request/parse
|
||||||
ClientSession (`session=`) or create and close one internally.
|
failure (logged, never raised, never printed). the only secret is geo.ipify's
|
||||||
|
`api_key`, a REQUIRED keyword on `ip_location` - the caller injects it, nothing
|
||||||
url-building and json->dict parsing are pulled into pure helpers (`_*_url`,
|
is hardcoded here.
|
||||||
`_parse_*`) so they unit-test without network. every lookup returns None on any
|
|
||||||
request/parse failure (logged, never raised, never printed).
|
|
||||||
|
|
||||||
security: the only secret is geo.ipify's `api_key`, which is a REQUIRED keyword on
|
|
||||||
`ip_location` — the caller injects it. nothing is hardcoded here.
|
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import unicodedata
|
import unicodedata
|
||||||
@@ -59,11 +54,8 @@ def _parse_ipify(data: dict) -> Optional[str]:
|
|||||||
|
|
||||||
|
|
||||||
def _state_slug(state) -> str:
|
def _state_slug(state) -> str:
|
||||||
"""lowercase ascii-folded state slug with underscores (e.g. 'New York' -> 'new_york')
|
"""lowercase ascii-folded state slug with underscores (e.g. 'New York' -> 'new_york');
|
||||||
|
coerces to str first so a non-string `state` can't TypeError out of the parse contract"""
|
||||||
coerces to str first so a malformed non-string `state` doesn't raise TypeError out
|
|
||||||
of unicodedata.normalize and break the 'None on any parse failure' contract.
|
|
||||||
"""
|
|
||||||
folded = unicodedata.normalize("NFKD", str(state)).encode("ascii", "ignore").decode("ascii")
|
folded = unicodedata.normalize("NFKD", str(state)).encode("ascii", "ignore").decode("ascii")
|
||||||
return folded.lower().replace(" ", "_")
|
return folded.lower().replace(" ", "_")
|
||||||
|
|
||||||
@@ -121,7 +113,7 @@ async def ip_location(
|
|||||||
"""look up geo data for an ip via geo.ipify; api_key is required and injected
|
"""look up geo data for an ip via geo.ipify; api_key is required and injected
|
||||||
|
|
||||||
returns the raw geo.ipify json dict, or None on failure. raises ValueError if
|
returns the raw geo.ipify json dict, or None on failure. raises ValueError if
|
||||||
no api_key is supplied — there is no default and nothing hardcoded.
|
no api_key is supplied - there is no default and nothing hardcoded.
|
||||||
"""
|
"""
|
||||||
if not api_key:
|
if not api_key:
|
||||||
raise ValueError("ip_location requires an api_key (inject it; never hardcode)")
|
raise ValueError("ip_location requires an api_key (inject it; never hardcode)")
|
||||||
|
|||||||
@@ -1,12 +1,10 @@
|
|||||||
"""pure ip/address utilities over stdlib ipaddress (no network, no deps).
|
"""pure ip/address utilities over stdlib ipaddress (no network, no deps).
|
||||||
|
|
||||||
every function accepts strings. validation/membership helpers
|
validation/membership helpers (`is_valid`/`version`/`in_network`/`in_any`) never
|
||||||
(`is_valid`/`version`/`in_network`/`in_any`) never raise on bad input — they return
|
raise on bad input - they return a falsy value. functions that require a valid
|
||||||
a falsy value. functions that require a valid address or cidr (`to_int`/`set_bits`/
|
address or cidr (`to_int`/`set_bits`/the network-shape helpers) let `ValueError`
|
||||||
the network-shape helpers) let `ValueError` propagate so misuse is visible.
|
propagate so misuse is visible. cidr parsing uses `ip_network(cidr, strict=False)`
|
||||||
|
throughout, so host bits set in the network string are tolerated (e.g. "10.0.0.5/24").
|
||||||
cidr parsing uses `ip_network(cidr, strict=False)` throughout, so host bits set in
|
|
||||||
the network string are tolerated (e.g. "10.0.0.5/24").
|
|
||||||
"""
|
"""
|
||||||
import ipaddress
|
import ipaddress
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
@@ -117,7 +115,7 @@ def hosts(cidr: str, *, limit: Optional[int] = None) -> List[str]:
|
|||||||
gen = ipaddress.ip_network(cidr, strict=False).hosts()
|
gen = ipaddress.ip_network(cidr, strict=False).hosts()
|
||||||
out: List[str] = []
|
out: List[str] = []
|
||||||
for host in gen:
|
for host in gen:
|
||||||
out.append(str(host))
|
|
||||||
if limit is not None and len(out) >= limit:
|
if limit is not None and len(out) >= limit:
|
||||||
break
|
break
|
||||||
|
out.append(str(host))
|
||||||
return out
|
return out
|
||||||
|
|||||||
+73
-42
@@ -1,19 +1,18 @@
|
|||||||
"""display masking for sensitive-looking values.
|
"""display masking for sensitive-looking values (e.g. "•••• •••• •••• 1234").
|
||||||
|
|
||||||
these are DISPLAY helpers only — they format a value for showing in a UI or log
|
DISPLAY only, not a security control: the underlying value is unchanged and still
|
||||||
(e.g. "•••• •••• •••• 1234"). they are not a security control: the underlying
|
needs proper handling (encryption at rest, etc.).
|
||||||
value is unchanged and still needs proper handling (encryption at rest, etc.).
|
|
||||||
|
|
||||||
card helpers: `credit`, `cvv`, `provider`. token: `phantom`. url/proxy redaction
|
- ``credit``/``cvv``/``phantom``/``provider`` mask card-shaped values for display.
|
||||||
for logging connection strings: `mask_url`, `mask_proxy`.
|
- ``mask_url`` strips the query and fragment off a url (keeping scheme/host/path),
|
||||||
|
so a url can be logged without leaking whatever a caller stuck in its params - it
|
||||||
|
does NOT hunt for tokens; a secret in a url is the caller's bug, not this to detect.
|
||||||
|
- ``mask_proxy`` gives a partial, user-reportable proxy identifier: credentials are
|
||||||
|
stripped and an ipv4 host's middle octets are masked (first/last octet + port kept).
|
||||||
|
- ``stable_id`` derives a deterministic, regenerable id from ordered string parts.
|
||||||
"""
|
"""
|
||||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
import hashlib
|
||||||
|
from urllib.parse import urlsplit, urlunsplit
|
||||||
SENSITIVE_QUERY_KEYS = frozenset(
|
|
||||||
{"apikey", "api_key", "key", "token", "access_token", "refresh_token",
|
|
||||||
"auth", "password", "passwd", "pwd", "pass", "secret", "client_secret",
|
|
||||||
"sig", "signature", "session"}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _digits(value: str) -> str:
|
def _digits(value: str) -> str:
|
||||||
@@ -35,7 +34,7 @@ def cvv(value: str) -> str:
|
|||||||
def phantom(value: str) -> str:
|
def phantom(value: str) -> str:
|
||||||
"""show a long token as first-six...last-four (e.g. a hash or id)
|
"""show a long token as first-six...last-four (e.g. a hash or id)
|
||||||
|
|
||||||
values of length <= 10 fully mask instead — first6+last4 would otherwise
|
values of length <= 10 fully mask instead - first6+last4 would otherwise
|
||||||
reveal (or double-reveal) every character while still looking masked.
|
reveal (or double-reveal) every character while still looking masked.
|
||||||
"""
|
"""
|
||||||
if len(value) <= 10:
|
if len(value) <= 10:
|
||||||
@@ -46,7 +45,7 @@ def phantom(value: str) -> str:
|
|||||||
def provider(card_number: str) -> str:
|
def provider(card_number: str) -> str:
|
||||||
"""short card brand from the number's prefix, or UNKW if undetermined
|
"""short card brand from the number's prefix, or UNKW if undetermined
|
||||||
|
|
||||||
a BIN-prefix heuristic for display/labeling — not authoritative validation.
|
a BIN-prefix heuristic for display/labeling - not authoritative validation.
|
||||||
tolerates spaces/dashes and short or non-numeric input (returns UNKW).
|
tolerates spaces/dashes and short or non-numeric input (returns UNKW).
|
||||||
"""
|
"""
|
||||||
n = _digits(card_number)
|
n = _digits(card_number)
|
||||||
@@ -69,41 +68,73 @@ def provider(card_number: str) -> str:
|
|||||||
return "UNKW"
|
return "UNKW"
|
||||||
|
|
||||||
|
|
||||||
def mask_url(url: str, *, keys: "frozenset[str] | None" = None) -> str:
|
def mask_url(url: str) -> str:
|
||||||
"""redact credentials in a url for logging: drop userinfo, ``***`` sensitive query values
|
"""strip query and fragment from a url, keeping scheme, host and path
|
||||||
|
|
||||||
param-name matching is case-insensitive; ``keys`` overrides the default sensitive-name
|
does NOT parse or hunt for sensitive params - it simply drops everything after the
|
||||||
set. non-url or unparseable input is returned unchanged.
|
path, so it structurally cannot leak a query param and cannot over-mask a legit one.
|
||||||
|
a url with no query/fragment is returned unchanged; a url urlsplit cannot parse
|
||||||
|
(a malformed ipv6 literal) is also returned unchanged rather than crashing the
|
||||||
|
display/log call this feeds.
|
||||||
"""
|
"""
|
||||||
sensitive = SENSITIVE_QUERY_KEYS if keys is None else frozenset(k.lower() for k in keys)
|
|
||||||
try:
|
try:
|
||||||
parts = urlsplit(url)
|
parts = urlsplit(url)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
return url
|
return url
|
||||||
if not parts.scheme and not parts.netloc:
|
return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
|
||||||
return url
|
|
||||||
netloc = parts.netloc
|
|
||||||
if "@" in netloc:
|
def _is_ipv4(host: str) -> bool:
|
||||||
netloc = netloc.rsplit("@", 1)[1]
|
"""true if host is a dotted-quad ipv4 literal (four 0-255 octets)"""
|
||||||
query = parts.query
|
octets = host.split(".")
|
||||||
if query:
|
if len(octets) != 4:
|
||||||
pairs = parse_qsl(query, keep_blank_values=True)
|
return False
|
||||||
query = urlencode(
|
for octet in octets:
|
||||||
[(k, "***" if k.lower() in sensitive else v) for k, v in pairs],
|
if not (octet.isdigit() and octet.isascii()):
|
||||||
quote_via=lambda s, *_: s,
|
return False
|
||||||
)
|
if not 0 <= int(octet) <= 255:
|
||||||
return urlunsplit((parts.scheme, netloc, parts.path, query, parts.fragment))
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def mask_proxy(spec: str) -> str:
|
def mask_proxy(spec: str) -> str:
|
||||||
"""redact the password of a ``host:port:user:password`` proxy string to ``:****``
|
"""mask the middle octets of a proxy ip, keeping first/last octet and full port; strips any credentials
|
||||||
|
|
||||||
a plain ``host:port`` (no auth) and any other non-conforming input pass through unchanged.
|
drops a ``scheme://`` prefix and any ``user:pass@`` userinfo entirely (a credential is
|
||||||
|
removed, never partial-masked). an ipv4 host ``A.B.C.D`` becomes ``A.***.***.D`` with the
|
||||||
|
port kept as-is (``172.58.32.1:8080`` -> ``172.***.***.1:8080``); a non-ipv4 host (hostname,
|
||||||
|
ipv6) passes through unchanged with only its credentials stripped. raises ValueError on a
|
||||||
|
spec with no parseable host.
|
||||||
"""
|
"""
|
||||||
parts = spec.split(":")
|
rest = spec
|
||||||
if len(parts) == 2:
|
if "://" in rest:
|
||||||
return spec
|
rest = rest.split("://", 1)[1]
|
||||||
if len(parts) == 4:
|
if "@" in rest:
|
||||||
host, port, user, _ = parts
|
rest = rest.rsplit("@", 1)[1]
|
||||||
return f"{host}:{port}:{user}:****"
|
host, sep, port = rest.partition(":")
|
||||||
return spec
|
if not host:
|
||||||
|
raise ValueError(f"mask_proxy: no host in proxy spec {spec!r}")
|
||||||
|
if _is_ipv4(host):
|
||||||
|
octets = host.split(".")
|
||||||
|
host = f"{octets[0]}.***.***.{octets[3]}"
|
||||||
|
return f"{host}:{port}" if sep else host
|
||||||
|
|
||||||
|
|
||||||
|
def stable_id(*parts: str, length: int = 16) -> str:
|
||||||
|
"""deterministic id from ordered parts; same inputs always produce the same id
|
||||||
|
|
||||||
|
joins the parts with a ``\\x00`` separator (so ``("ab","c")`` != ``("a","bc")``) and
|
||||||
|
returns the leading ``length`` hex chars of their sha256. raises ValueError on no parts,
|
||||||
|
an empty part, a non-str part, or a non-positive length - fail loud rather than emit a
|
||||||
|
weak or constant id.
|
||||||
|
"""
|
||||||
|
if length <= 0:
|
||||||
|
raise ValueError(f"stable_id: length must be positive, got {length}")
|
||||||
|
if not parts:
|
||||||
|
raise ValueError("stable_id: needs at least one part")
|
||||||
|
for part in parts:
|
||||||
|
if not isinstance(part, str):
|
||||||
|
raise ValueError(f"stable_id: parts must be str, got {type(part).__name__}")
|
||||||
|
if part == "":
|
||||||
|
raise ValueError("stable_id: parts must be non-empty")
|
||||||
|
return hashlib.sha256("\x00".join(parts).encode()).hexdigest()[:length]
|
||||||
|
|||||||
+126
-15
@@ -1,15 +1,17 @@
|
|||||||
"""nested access by dotted path.
|
"""nested dict/list access by dotted path
|
||||||
|
|
||||||
`deep_get(data, "in.this.old.notation")` walks dicts (and lists, when a segment is
|
- deep_get/deep_set: single path, scalar in/out. numeric segment indexes a list;
|
||||||
a number) and returns a default instead of raising on a missing/!wrong path.
|
setting into a tuple raises (immutable).
|
||||||
`deep_set` writes a nested value, creating intermediate dicts and indexing into
|
- deep_gets/deep_sets: `*` segments iterate every element at a level (bulk get -> list,
|
||||||
existing lists on numeric segments (mirroring deep_get), so a get/set round-trip
|
bulk set -> value or fn(current)->new); multiple `*` fan out cartesian.
|
||||||
on the same path never corrupts a list. an out-of-range numeric segment raises
|
|
||||||
IndexError rather than silently mis-storing.
|
a `*` in the scalar verbs raises ValueError - use the plural verbs so the return type
|
||||||
|
is never ambiguous.
|
||||||
"""
|
"""
|
||||||
from typing import Any
|
from typing import Any, Callable, List, Union
|
||||||
|
|
||||||
_MISSING = object()
|
_MISSING = object()
|
||||||
|
_WILDCARD = "*"
|
||||||
|
|
||||||
|
|
||||||
def deep_get(data: Any, path: str, default: Any = None, *, sep: str = ".") -> Any:
|
def deep_get(data: Any, path: str, default: Any = None, *, sep: str = ".") -> Any:
|
||||||
@@ -17,10 +19,14 @@ def deep_get(data: Any, path: str, default: Any = None, *, sep: str = ".") -> An
|
|||||||
|
|
||||||
dict keys are matched by name; a numeric segment indexes a list/tuple
|
dict keys are matched by name; a numeric segment indexes a list/tuple
|
||||||
(e.g. "items.0.id"). any missing key, out-of-range index, or non-container
|
(e.g. "items.0.id"). any missing key, out-of-range index, or non-container
|
||||||
along the way yields `default` rather than raising.
|
along the way yields `default` rather than raising. a `*` segment raises
|
||||||
|
ValueError - use deep_gets for wildcard paths.
|
||||||
"""
|
"""
|
||||||
|
segments = path.split(sep)
|
||||||
|
if _WILDCARD in segments:
|
||||||
|
raise ValueError(f"deep_get: wildcard '*' in path {path!r}; use deep_gets for wildcard paths")
|
||||||
cur = data
|
cur = data
|
||||||
for seg in path.split(sep):
|
for seg in segments:
|
||||||
if isinstance(cur, dict):
|
if isinstance(cur, dict):
|
||||||
cur = cur.get(seg, _MISSING)
|
cur = cur.get(seg, _MISSING)
|
||||||
if cur is _MISSING:
|
if cur is _MISSING:
|
||||||
@@ -38,29 +44,35 @@ def deep_get(data: Any, path: str, default: Any = None, *, sep: str = ".") -> An
|
|||||||
def deep_set(data: dict, path: str, value: Any, *, sep: str = ".") -> dict:
|
def deep_set(data: dict, path: str, value: Any, *, sep: str = ".") -> dict:
|
||||||
"""set a nested value by dotted path, creating intermediate dicts; returns data for chaining
|
"""set a nested value by dotted path, creating intermediate dicts; returns data for chaining
|
||||||
|
|
||||||
mirrors deep_get's list indexing (a numeric segment updates an existing list/tuple
|
a numeric segment updates a LIST element in place (out-of-range raises IndexError);
|
||||||
element rather than corrupting it); unlike deep_get, an out-of-range index raises
|
setting through or into a tuple raises TypeError (immutable). a `*` raises ValueError -
|
||||||
IndexError instead of silently mis-storing, since there's no safe default to fall back to.
|
use deep_sets for wildcard paths.
|
||||||
"""
|
"""
|
||||||
segments = path.split(sep)
|
segments = path.split(sep)
|
||||||
|
if _WILDCARD in segments:
|
||||||
|
raise ValueError(f"deep_set: wildcard '*' in path {path!r}; use deep_sets for wildcard paths")
|
||||||
cur = data
|
cur = data
|
||||||
for seg in segments[:-1]:
|
for seg in segments[:-1]:
|
||||||
|
if isinstance(cur, tuple):
|
||||||
|
raise TypeError(f"deep_set: cannot set into an immutable tuple at {path!r}")
|
||||||
if isinstance(cur, list):
|
if isinstance(cur, list):
|
||||||
idx = int(seg)
|
idx = int(seg)
|
||||||
if not -len(cur) <= idx < len(cur):
|
if not -len(cur) <= idx < len(cur):
|
||||||
raise IndexError(f"deep_set: index {seg!r} out of range for list of length {len(cur)}")
|
raise IndexError(f"deep_set: index {seg!r} out of range for list of length {len(cur)}")
|
||||||
nxt = cur[idx]
|
nxt = cur[idx]
|
||||||
if not isinstance(nxt, (dict, list)):
|
if not isinstance(nxt, (dict, list, tuple)):
|
||||||
nxt = {}
|
nxt = {}
|
||||||
cur[idx] = nxt
|
cur[idx] = nxt
|
||||||
cur = nxt
|
cur = nxt
|
||||||
else:
|
else:
|
||||||
nxt = cur.get(seg)
|
nxt = cur.get(seg)
|
||||||
if not isinstance(nxt, (dict, list)):
|
if not isinstance(nxt, (dict, list, tuple)):
|
||||||
nxt = {}
|
nxt = {}
|
||||||
cur[seg] = nxt
|
cur[seg] = nxt
|
||||||
cur = nxt
|
cur = nxt
|
||||||
last = segments[-1]
|
last = segments[-1]
|
||||||
|
if isinstance(cur, tuple):
|
||||||
|
raise TypeError(f"deep_set: cannot set into an immutable tuple at {path!r}")
|
||||||
if isinstance(cur, list):
|
if isinstance(cur, list):
|
||||||
idx = int(last)
|
idx = int(last)
|
||||||
if not -len(cur) <= idx < len(cur):
|
if not -len(cur) <= idx < len(cur):
|
||||||
@@ -69,3 +81,102 @@ def deep_set(data: dict, path: str, value: Any, *, sep: str = ".") -> dict:
|
|||||||
else:
|
else:
|
||||||
cur[last] = value
|
cur[last] = value
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_children(node: Any):
|
||||||
|
"""yield (key, child) pairs for a `*` step: dict items, or list/tuple index/item pairs"""
|
||||||
|
if isinstance(node, dict):
|
||||||
|
yield from node.items()
|
||||||
|
elif isinstance(node, (list, tuple)):
|
||||||
|
yield from enumerate(node)
|
||||||
|
|
||||||
|
|
||||||
|
def deep_gets(data: Any, path: str, *, sep: str = ".") -> List[Any]:
|
||||||
|
"""get every value matching a dotted path with `*` wildcards, as a list (may be empty)
|
||||||
|
|
||||||
|
a `*` iterates every element at that level; multiple `*` fan out cartesian. a branch
|
||||||
|
that dead-ends (missing key, out-of-range index, non-container) is skipped, not raised.
|
||||||
|
a path with no `*` returns a one- or zero-element list.
|
||||||
|
"""
|
||||||
|
segments = path.split(sep)
|
||||||
|
frontier = [data]
|
||||||
|
for seg in segments:
|
||||||
|
nxt: List[Any] = []
|
||||||
|
if seg == _WILDCARD:
|
||||||
|
for node in frontier:
|
||||||
|
for _, child in _iter_children(node):
|
||||||
|
nxt.append(child)
|
||||||
|
else:
|
||||||
|
for node in frontier:
|
||||||
|
if isinstance(node, dict):
|
||||||
|
got = node.get(seg, _MISSING)
|
||||||
|
if got is not _MISSING:
|
||||||
|
nxt.append(got)
|
||||||
|
elif isinstance(node, (list, tuple)):
|
||||||
|
try:
|
||||||
|
nxt.append(node[int(seg)])
|
||||||
|
except (ValueError, IndexError):
|
||||||
|
pass
|
||||||
|
frontier = nxt
|
||||||
|
return frontier
|
||||||
|
|
||||||
|
|
||||||
|
def deep_sets(data: Any, path: str, value: "Union[Any, Callable[[Any], Any]]", *, sep: str = ".") -> Any:
|
||||||
|
"""set every value matching a dotted path with `*` wildcards; returns data for chaining
|
||||||
|
|
||||||
|
`value` is a plain value written to every match, or a callable `fn(current) -> new`
|
||||||
|
(a bare callable is always invoked - to store one, wrap it `lambda _c, f=fn: f`). a `*`
|
||||||
|
iterates every element at that level; multiple `*` fan out cartesian. best-effort: any
|
||||||
|
dead-end (missing key, out-of-range index) is skipped, not raised. setting through or
|
||||||
|
into a tuple raises TypeError (immutable).
|
||||||
|
"""
|
||||||
|
compute: Callable[[Any], Any] = value if callable(value) else (lambda _cur, _v=value: _v)
|
||||||
|
segments = path.split(sep)
|
||||||
|
parents = segments[:-1]
|
||||||
|
last = segments[-1]
|
||||||
|
|
||||||
|
frontier = [data]
|
||||||
|
for seg in parents:
|
||||||
|
nxt: List[Any] = []
|
||||||
|
if seg == _WILDCARD:
|
||||||
|
for node in frontier:
|
||||||
|
if isinstance(node, tuple):
|
||||||
|
raise TypeError(f"deep_sets: cannot set through an immutable tuple at {path!r}")
|
||||||
|
for _, child in _iter_children(node):
|
||||||
|
nxt.append(child)
|
||||||
|
else:
|
||||||
|
for node in frontier:
|
||||||
|
if isinstance(node, tuple):
|
||||||
|
raise TypeError(f"deep_sets: cannot set through an immutable tuple at {path!r}")
|
||||||
|
if isinstance(node, dict):
|
||||||
|
got = node.get(seg, _MISSING)
|
||||||
|
if got is not _MISSING:
|
||||||
|
nxt.append(got)
|
||||||
|
elif isinstance(node, list):
|
||||||
|
try:
|
||||||
|
nxt.append(node[int(seg)])
|
||||||
|
except (ValueError, IndexError):
|
||||||
|
pass
|
||||||
|
frontier = nxt
|
||||||
|
|
||||||
|
for node in frontier:
|
||||||
|
if isinstance(node, tuple):
|
||||||
|
raise TypeError(f"deep_sets: cannot set into an immutable tuple at {path!r}")
|
||||||
|
if last == _WILDCARD:
|
||||||
|
if isinstance(node, dict):
|
||||||
|
for key in list(node.keys()):
|
||||||
|
node[key] = compute(node[key])
|
||||||
|
elif isinstance(node, list):
|
||||||
|
for idx in range(len(node)):
|
||||||
|
node[idx] = compute(node[idx])
|
||||||
|
elif isinstance(node, dict):
|
||||||
|
if node.get(last, _MISSING) is not _MISSING:
|
||||||
|
node[last] = compute(node[last])
|
||||||
|
elif isinstance(node, list):
|
||||||
|
try:
|
||||||
|
idx = int(last)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
if -len(node) <= idx < len(node):
|
||||||
|
node[idx] = compute(node[idx])
|
||||||
|
return data
|
||||||
|
|||||||
+3
-21
@@ -1,25 +1,7 @@
|
|||||||
"""retry with exponential backoff — sync and async, one backoff engine.
|
"""retry with exponential backoff, sync (`retry`) and async (`aretry`), call or decorator form.
|
||||||
|
|
||||||
de-duplicates retry logic that was written divergently across several libs (HTTP
|
|
||||||
429/5xx caps, proxy burn/rotate caps, IMAP reconnects). both a call form and a
|
|
||||||
decorator form share one implementation per flavor; the backoff schedule is a pure
|
|
||||||
generator so it tests without real sleeps.
|
|
||||||
|
|
||||||
from commons import retry, aretry
|
|
||||||
|
|
||||||
# call form
|
|
||||||
rows = retry(lambda: db_read(), attempts=5, on=(IOError,))
|
|
||||||
data = await aretry(lambda: fetch(url), attempts=3, on=(TimeoutError,))
|
|
||||||
|
|
||||||
# decorator form (same kwargs)
|
|
||||||
@aretry(attempts=4, backoff=0.5, on=(ConnectionError,))
|
|
||||||
async def pull():
|
|
||||||
...
|
|
||||||
|
|
||||||
after `attempts` are exhausted the LAST exception is re-raised (fail loud, never
|
after `attempts` are exhausted the LAST exception is re-raised (fail loud, never
|
||||||
swallowed). `on` narrows which exceptions retry; `give_up(exc) -> bool` stops early
|
swallowed). see README for usage examples.
|
||||||
on a non-retryable error (e.g. a 400 vs a 429). each retry is logged (emit-only),
|
|
||||||
never printed.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
@@ -38,7 +20,7 @@ def _delays(attempts: int, backoff: float, factor: float, max_backoff: float):
|
|||||||
"""yield the wait before each retry: min(backoff * factor**n, max_backoff)
|
"""yield the wait before each retry: min(backoff * factor**n, max_backoff)
|
||||||
|
|
||||||
yields `attempts - 1` delays (one before each retry after the first try). the
|
yields `attempts - 1` delays (one before each retry after the first try). the
|
||||||
raw, un-jittered schedule — jitter is applied at call time so the schedule stays
|
raw, un-jittered schedule - jitter is applied at call time so the schedule stays
|
||||||
pure and testable.
|
pure and testable.
|
||||||
"""
|
"""
|
||||||
for n in range(max(0, attempts - 1)):
|
for n in range(max(0, attempts - 1)):
|
||||||
|
|||||||
+6
-12
@@ -1,22 +1,16 @@
|
|||||||
"""time helpers built on unix timestamps with timezone-aware datetime support.
|
"""time helpers built on unix timestamps with timezone-aware datetime support.
|
||||||
|
|
||||||
one engine, two ergonomics:
|
one engine, two ergonomics: bare module functions for stateless unix-int math
|
||||||
- bare module functions (now/add/ahead/ago/is_expired/to_dt/...) operate on unix
|
(all delta math routes through `_delta_seconds`, the single source of truth), and
|
||||||
ints for quick, stateless use. all delta math routes through `_delta_seconds`.
|
`Clock`, which holds a timezone + fast flag and delegates to the same functions.
|
||||||
- `Clock` holds a timezone + fast flag and delegates to the same functions, so a
|
|
||||||
configured clock and the bare functions never diverge.
|
|
||||||
|
|
||||||
unix ints stay the storable value; datetimes are produced on demand in whatever
|
|
||||||
timezone you ask for (stdlib zoneinfo, no pytz).
|
|
||||||
"""
|
"""
|
||||||
import time as _time
|
import time as _time
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from typing import Optional, Union
|
from typing import Optional, Union
|
||||||
from zoneinfo import ZoneInfo
|
from zoneinfo import ZoneInfo
|
||||||
|
|
||||||
# default fast flag for bare module calls. when True, every unit (day/hour/minute)
|
# default fast flag for bare calls (test-only: collapses each unit to 1s); set
|
||||||
# counts as one second so time-based flows run fast in tests. set
|
# `commons.timing.FAST_MODE` from test setup, off by default, not a config import.
|
||||||
# `commons.timing.FAST_MODE` from test setup; off by default. not a config import.
|
|
||||||
FAST_MODE = False
|
FAST_MODE = False
|
||||||
|
|
||||||
UTC = timezone.utc
|
UTC = timezone.utc
|
||||||
@@ -42,7 +36,7 @@ def now() -> int:
|
|||||||
def _delta_seconds(days: int, hours: int, minutes: int, seconds: int, fast: bool) -> int:
|
def _delta_seconds(days: int, hours: int, minutes: int, seconds: int, fast: bool) -> int:
|
||||||
"""seconds for a unit combination; collapses to 1s/unit when fast is True
|
"""seconds for a unit combination; collapses to 1s/unit when fast is True
|
||||||
|
|
||||||
single source of truth for unit math — every delta helper routes through here.
|
single source of truth for unit math - every delta helper routes through here.
|
||||||
"""
|
"""
|
||||||
if fast:
|
if fast:
|
||||||
return days + hours + minutes + seconds
|
return days + hours + minutes + seconds
|
||||||
|
|||||||
Reference in New Issue
Block a user