diff --git a/README.md b/README.md index 5b7a26e..ab71114 100644 --- a/README.md +++ b/README.md @@ -12,15 +12,15 @@ Small sync helpers shared across projects. Base is stdlib only — **no dependen ## Install ``` -commons @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.2.3 +commons @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.3.1 # async address/geo lookups (fetch_ip / ip_location / fetch_location) need the extra: -commons[addr] @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.2.3 +commons[addr] @ git+ssh://git@git.rethinkstudios.io/rethink-public/commons.git@v0.3.1 ``` The base install pulls **nothing** (stdlib). Only `commons[addr]` adds `aiohttp`, and only for the geo lookups — the pure `commons.addr.ip` utilities ship in base. -Drop the `@v0.2.3` suffix from the line above to install the latest unpinned. +Drop the `@v0.3.1` suffix from the line above to install the latest unpinned. ## timing @@ -99,22 +99,43 @@ deep_get(data, "items.1.id") # "b" (numeric segment indexes a list) deep_get(data, "in.nope.here", "DEF") # "DEF" (missing -> default, no raise) deep_set({}, "a.b.c", 9) # {"a": {"b": {"c": 9}}} +deep_set(data, "items.1.id", "B") # updates the list element in place +deep_get(data, "items.1.id") # "B" (get/set share the same grammar) +deep_set(data, "items.9.id", "X") # raises IndexError (out of range, no + # silent mis-store) ``` +`deep_set` mirrors `deep_get`'s list indexing: a numeric segment over an existing +list/tuple updates that element in place rather than replacing the list with a dict. +An out-of-range numeric segment raises `IndexError` instead of silently corrupting +the structure. + ## masking Display helpers only — they format a value for showing; they are **not a security control** (the underlying value is unchanged and still needs proper handling). ```python -from commons import credit, cvv, phantom, provider +from commons import credit, cvv, phantom, provider, mask_url, mask_proxy credit("4111 1111 1111 1234") # "•••• •••• •••• 1234" cvv("123") # "•••" phantom("abcdef1234567890") # "abcdef...7890" provider("4111111111111111") # "VISA" (VISA/MC/AMEX/UPAY/DISC/JCB/DNRS/UNKW) + +# redact credentials in a connection string before logging it +mask_url("https://u:pw@api.x/v2?apiKey=SECRET&ip=8.8.8.8") +# -> "https://api.x/v2?apiKey=***&ip=8.8.8.8" (userinfo dropped, secret query masked) +mask_url("redis://:pw@127.0.0.1:6379/0") # -> "redis://127.0.0.1:6379/0" +mask_proxy("1.2.3.4:8080:user:supersecret") # -> "1.2.3.4:8080:user:****" +mask_proxy("1.2.3.4:8080") # -> "1.2.3.4:8080" (no auth, untouched) ``` +`mask_url` strips `user:pass@` userinfo and replaces the values of sensitive query +params (`apiKey`, `token`, `password`, `secret`, …; override via `keys=`) with `***`. +`mask_proxy` bullets the password of a `host:port:user:password` spec. Non-URL / +non-conforming input is returned unchanged. + ## retry Exponential-backoff retry, sync (`retry`) and async (`aretry`). Each works as a **call diff --git a/pyproject.toml b/pyproject.toml index a9ecfd6..fee0352 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "commons" -version = "0.2.3" +version = "0.3.1" description = "small stdlib-based sync helpers: time/timezone deltas, dotted-path dict access, display masking, ip/address tooling, and retry/backoff" requires-python = ">=3.10" dependencies = [] diff --git a/src/commons/__init__.py b/src/commons/__init__.py index d4c8e05..7749abd 100644 --- a/src/commons/__init__.py +++ b/src/commons/__init__.py @@ -2,8 +2,11 @@ timing: unix-timestamp deltas + timezone-aware datetime conversions, bare functions and a configurable Clock. -paths: nested dict/list access by dotted path (deep_get / deep_set). -masking: display masking for cards / cvv / tokens (cosmetic, not a security control). +paths: nested dict/list access by dotted path (deep_get / deep_set); deep_set + mirrors deep_get's list indexing, so a get/set round-trip on the same + path never destroys a list. +masking: display masking for cards / cvv / tokens, plus url/proxy credential + redaction for logging (cosmetic, not a security control). addr: ip/address tooling — pure stdlib ip utils (`commons.addr.ip`) in base, async geo lookups (`commons.addr.geo`) behind the `commons[addr]` extra. retry: exponential-backoff retry, sync (`retry`) and async (`aretry`), call or @@ -16,7 +19,7 @@ for instance-scoped control. addr is exposed as a submodule (`from commons impor addr`); its ip helpers live under `commons.addr.ip` to keep top-level uncluttered. """ from . import addr, masking, paths, timing -from .masking import credit, cvv, phantom, provider +from .masking import credit, cvv, mask_proxy, mask_url, phantom, provider from .paths import deep_get, deep_set from .retry import aretry, retry from .timing import ( @@ -59,8 +62,10 @@ __all__ = [ "cvv", "phantom", "provider", + "mask_url", + "mask_proxy", "retry", "aretry", ] -__version__ = "0.2.3" +__version__ = "0.3.1" diff --git a/src/commons/masking.py b/src/commons/masking.py index eefd824..1d7f0a1 100644 --- a/src/commons/masking.py +++ b/src/commons/masking.py @@ -3,7 +3,17 @@ these are DISPLAY helpers only — they format a value for showing in a UI or log (e.g. "•••• •••• •••• 1234"). they are not a security control: the underlying value is unchanged and still needs proper handling (encryption at rest, etc.). + +card helpers: `credit`, `cvv`, `provider`. token: `phantom`. url/proxy redaction +for logging connection strings: `mask_url`, `mask_proxy`. """ +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +SENSITIVE_QUERY_KEYS = frozenset( + {"apikey", "api_key", "key", "token", "access_token", "refresh_token", + "auth", "password", "passwd", "pwd", "pass", "secret", "client_secret", + "sig", "signature", "session"} +) def _digits(value: str) -> str: @@ -51,3 +61,48 @@ def provider(card_number: str) -> str: if n.startswith(("30", "36", "38", "39")): return "DNRS" return "UNKW" + + +def mask_url(url: str, *, keys: "frozenset[str] | None" = None) -> str: + """redact credentials in a url for logging + + strips userinfo (``user:pass@host`` -> ``host``) and replaces the values of + sensitive query parameters (apiKey, token, password, secret, ...) with + ``***``, keeping the parameter name so the shape stays readable. matching is + case-insensitive on the parameter name; pass ``keys`` to override the default + sensitive-name set. non-url or unparseable input is returned unchanged. + """ + sensitive = SENSITIVE_QUERY_KEYS if keys is None else frozenset(k.lower() for k in keys) + try: + parts = urlsplit(url) + except ValueError: + return url + if not parts.scheme and not parts.netloc: + return url + netloc = parts.netloc + if "@" in netloc: + netloc = netloc.rsplit("@", 1)[1] + query = parts.query + if query: + pairs = parse_qsl(query, keep_blank_values=True) + query = urlencode( + [(k, "***" if k.lower() in sensitive else v) for k, v in pairs], + quote_via=lambda s, *_: s, + ) + return urlunsplit((parts.scheme, netloc, parts.path, query, parts.fragment)) + + +def mask_proxy(spec: str) -> str: + """redact the password of a ``host:port:user:password`` proxy string + + renders ``host:port`` when there is no auth, or ``host:port:user:****`` when + a user/password is present — never the password itself. leaves a plain + ``host:port`` untouched and returns non-conforming input unchanged. + """ + parts = spec.split(":") + if len(parts) == 2: + return spec + if len(parts) == 4: + host, port, user, _ = parts + return f"{host}:{port}:{user}:****" + return spec diff --git a/src/commons/paths.py b/src/commons/paths.py index ff0c612..a0abd89 100644 --- a/src/commons/paths.py +++ b/src/commons/paths.py @@ -2,7 +2,10 @@ `deep_get(data, "in.this.old.notation")` walks dicts (and lists, when a segment is a number) and returns a default instead of raising on a missing/!wrong path. -`deep_set` writes a nested value, creating intermediate dicts. +`deep_set` writes a nested value, creating intermediate dicts and indexing into +existing lists on numeric segments (mirroring deep_get), so a get/set round-trip +on the same path never corrupts a list. an out-of-range numeric segment raises +IndexError rather than silently mis-storing. """ from typing import Any @@ -35,16 +38,38 @@ def deep_get(data: Any, path: str, default: Any = None, *, sep: str = ".") -> An def deep_set(data: dict, path: str, value: Any, *, sep: str = ".") -> dict: """set a nested value by dotted path, creating intermediate dicts - returns the same top-level dict for chaining. overwrites any non-dict value - sitting where an intermediate dict is needed. + returns the same top-level dict for chaining. mirrors deep_get: a numeric segment + indexes into an existing list/tuple instead of being treated as a dict key, so a + read-modify-write round-trip using the same path never corrupts a list. overwrites + any non-dict/non-list value sitting where an intermediate container is needed. an + out-of-range list index raises IndexError rather than silently mis-storing (deep_get + returns default on out-of-range; deep_set has no safe default to fall back to, so it + fails loud instead). """ segments = path.split(sep) cur = data for seg in segments[:-1]: - nxt = cur.get(seg) - if not isinstance(nxt, dict): - nxt = {} - cur[seg] = nxt - cur = nxt - cur[segments[-1]] = value + if isinstance(cur, list): + idx = int(seg) + if not -len(cur) <= idx < len(cur): + raise IndexError(f"deep_set: index {seg!r} out of range for list of length {len(cur)}") + nxt = cur[idx] + if not isinstance(nxt, (dict, list)): + nxt = {} + cur[idx] = nxt + cur = nxt + else: + nxt = cur.get(seg) + if not isinstance(nxt, (dict, list)): + nxt = {} + cur[seg] = nxt + cur = nxt + last = segments[-1] + if isinstance(cur, list): + idx = int(last) + if not -len(cur) <= idx < len(cur): + raise IndexError(f"deep_set: index {last!r} out of range for list of length {len(cur)}") + cur[idx] = value + else: + cur[last] = value return data