Compare commits
6
Commits
5c2fdd80f9
...
a2917cce23
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a2917cce23 | ||
|
|
68be8f7101 | ||
|
|
98f7190f18 | ||
|
|
ddf76d8241 | ||
|
|
5e3eb4e704 | ||
|
|
8381b2ecbd |
@@ -12,18 +12,18 @@ you `delete` or `clear` them.
|
|||||||
`requirements.txt`:
|
`requirements.txt`:
|
||||||
|
|
||||||
```
|
```
|
||||||
aiokv @ git+ssh://git@git.rethinkstudios.io/rethink-public/aiokv.git@v0.2.0
|
aiokv @ git+ssh://git@git.rethinkstudios.io/rethink-public/aiokv.git@v0.2.2
|
||||||
```
|
```
|
||||||
|
|
||||||
Direct:
|
Direct:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install "aiokv @ git+ssh://git@git.rethinkstudios.io/rethink-public/aiokv.git@v0.2.0"
|
pip install "aiokv @ git+ssh://git@git.rethinkstudios.io/rethink-public/aiokv.git@v0.2.2"
|
||||||
```
|
```
|
||||||
|
|
||||||
Requires `aiofiles` (pulled transitively).
|
Requires `aiofiles` (pulled transitively).
|
||||||
|
|
||||||
Drop the `@v0.2.0` suffix from the line above to install the latest unpinned.
|
Drop the `@v0.2.2` suffix from the line above to install the latest unpinned.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
@@ -66,11 +66,17 @@ Writes are **atomic**: data is written to a temp file in the same directory and
|
|||||||
written through rather than clobbered. A **process** crash mid-write leaves the previous
|
written through rather than clobbered. A **process** crash mid-write leaves the previous
|
||||||
good file intact, and a reader never observes a partial file. (This is process-crash
|
good file intact, and a reader never observes a partial file. (This is process-crash
|
||||||
safety, not power-loss durability — there's no `fsync`, so an OS/power failure could
|
safety, not power-loss durability — there's no `fsync`, so an OS/power failure could
|
||||||
still lose the last write; fine for reconstructible single-process state.) Orphaned
|
still lose the last write; fine for reconstructible single-process state.) `clear()` is
|
||||||
`.<pid>.tmp` files left by a hard crash of a *different, dead* process are swept on the
|
symlink-safe the same way: it removes the realpath'd target, not the symlink itself, so
|
||||||
next save. A single `asyncio.Lock` guards every read and write, so concurrent operations
|
a symlinked store is genuinely erased rather than just having its symlink unlinked.
|
||||||
on one instance are consistent and no update is lost. All blocking filesystem calls and
|
Orphaned `.<pid>.tmp` files left by a hard crash of a *different, dead* process are swept
|
||||||
JSON (de)serialization run via `asyncio.to_thread`, so nothing stalls the event loop.
|
on the next save; the sweep matches only the exact `<realpath basename>.<pid>.tmp` shape
|
||||||
|
it creates, so a differently-shaped neighbor file is never touched and a store filename
|
||||||
|
containing glob-like characters (e.g. `state[prod].json`) doesn't miss its own orphans or
|
||||||
|
cross-match another store's. A single `asyncio.Lock` guards every read and write, so
|
||||||
|
concurrent operations on one instance are consistent and no update is lost. All blocking
|
||||||
|
filesystem calls and JSON (de)serialization run via `asyncio.to_thread`, so nothing stalls
|
||||||
|
the event loop.
|
||||||
|
|
||||||
## Scope — read this
|
## Scope — read this
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "aiokv"
|
name = "aiokv"
|
||||||
version = "0.2.0"
|
version = "1.0.0"
|
||||||
description = "Async file-backed key-value store for single-process local state — atomic writes, no TTL, config-free, installable."
|
description = "Async file-backed key-value store for single-process local state — atomic writes, no TTL, config-free, installable."
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
|
from importlib.metadata import version, PackageNotFoundError
|
||||||
|
|
||||||
from .aiokv import AioKV, aiocache
|
from .aiokv import AioKV, aiocache
|
||||||
|
|
||||||
__version__ = "0.2.0"
|
try:
|
||||||
|
__version__ = version("aiokv")
|
||||||
|
except PackageNotFoundError:
|
||||||
|
__version__ = "0.0.0+unknown"
|
||||||
|
|
||||||
__all__ = ["AioKV", "aiocache"]
|
__all__ = ["AioKV", "aiocache"]
|
||||||
|
|||||||
+36
-40
@@ -1,41 +1,25 @@
|
|||||||
"""
|
"""
|
||||||
async file-backed key-value store for single-process local state
|
async file-backed key-value store for single-process local state
|
||||||
|
|
||||||
a persist-forever KV store (last-used-command, rate-limit timestamps, seen-ids,
|
a persist-forever KV store backed by a JSON file, not a cache (no TTL/expiry/eviction).
|
||||||
simple bot state) backed by a JSON file. NOT a cache: no TTL, no expiry, no
|
see README for usage and the full API.
|
||||||
eviction — values live until you delete or clear them.
|
|
||||||
|
|
||||||
from aiokv import AioKV
|
from aiokv import AioKV
|
||||||
|
|
||||||
kv = AioKV("state.json")
|
kv = AioKV("state.json")
|
||||||
await kv.set("last_seen", 12345)
|
await kv.set("last_seen", 12345)
|
||||||
await kv.set("ran_cleanup") # value omitted -> stores int(time.time())
|
when = await kv.get("last_seen")
|
||||||
when = await kv.get("ran_cleanup")
|
|
||||||
await kv.delete("last_seen")
|
|
||||||
|
|
||||||
durability: atomic writes (temp file + os.replace, symlink-safe), stale
|
atomic writes are process-crash safe (temp file + os.replace) but NOT power-loss safe
|
||||||
`.<pid>.tmp` files from a hard crash are swept on save. this is process-crash
|
(no fsync). the lock is per-instance only - two AioKV instances or processes on the
|
||||||
safety, NOT power-loss durability — no fsync, so an OS/power failure can still
|
same file will clobber each other; this is single-process local state, not shared
|
||||||
lose the last write. a single asyncio.Lock guards every read and write, so
|
cross-process storage.
|
||||||
concurrent operations on one instance are consistent. JSON (de)serialization
|
|
||||||
runs off-loop via asyncio.to_thread so a large store never stalls other tasks.
|
|
||||||
|
|
||||||
scope: SINGLE-PROCESS, single-instance local state only. the lock is per-instance —
|
|
||||||
two AioKV instances (or two processes) pointing at the same file are NOT safe and
|
|
||||||
will clobber each other. for shared cross-process/cross-bot state, use a database
|
|
||||||
(e.g. mongo), not this.
|
|
||||||
|
|
||||||
config-free: the file path is passed at construction; nothing is read from a global
|
|
||||||
config. keys must be str (ValueError otherwise — JSON object keys are always
|
|
||||||
strings, so a non-str key would silently never round-trip). errors in delete/clear
|
|
||||||
are logged and swallowed (returning False); get/set raise on unexpected i/o so a
|
|
||||||
real failure is visible.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import glob
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
from typing import Any, Dict
|
from typing import Any, Dict
|
||||||
@@ -87,11 +71,15 @@ class AioKV:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
async def clear(self) -> bool:
|
async def clear(self) -> bool:
|
||||||
"""remove the backing file entirely; True on success or if already absent, False on error"""
|
"""remove the backing file entirely; True on success or if already absent, False on error
|
||||||
|
|
||||||
|
symlink-safe: removes the realpath'd target rather than the symlink itself, mirroring
|
||||||
|
_save()'s symlink-safe writes - a symlinked store's real data file is what gets erased"""
|
||||||
try:
|
try:
|
||||||
async with self.lock:
|
async with self.lock:
|
||||||
|
target = await asyncio.to_thread(os.path.realpath, self.file)
|
||||||
try:
|
try:
|
||||||
await asyncio.to_thread(os.remove, self.file)
|
await asyncio.to_thread(os.remove, target)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
pass
|
pass
|
||||||
return True
|
return True
|
||||||
@@ -106,7 +94,7 @@ class AioKV:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _check_key(key: str) -> None:
|
def _check_key(key: str) -> None:
|
||||||
"""raise ValueError if key is not str (JSON object keys are always strings)"""
|
"""raise ValueError if key is not str"""
|
||||||
if not isinstance(key, str):
|
if not isinstance(key, str):
|
||||||
raise ValueError(f"aiokv keys must be str, got {type(key).__name__}")
|
raise ValueError(f"aiokv keys must be str, got {type(key).__name__}")
|
||||||
|
|
||||||
@@ -128,12 +116,12 @@ class AioKV:
|
|||||||
async def _save(self, cache: Dict[str, Any]) -> None:
|
async def _save(self, cache: Dict[str, Any]) -> None:
|
||||||
"""write atomically: temp file in the same dir, then os.replace over the realpath'd
|
"""write atomically: temp file in the same dir, then os.replace over the realpath'd
|
||||||
target (symlink-safe). process-crash safe (a crash mid-write leaves the prior good
|
target (symlink-safe). process-crash safe (a crash mid-write leaves the prior good
|
||||||
file intact), NOT power-loss safe — no fsync, so an OS/power failure can still lose
|
file intact), NOT power-loss safe - no fsync, so an OS/power failure can still lose
|
||||||
the last write (acceptable: reconstructible single-process state, not a db)"""
|
the last write (acceptable: reconstructible single-process state, not a db)"""
|
||||||
target = await asyncio.to_thread(os.path.realpath, self.file)
|
target = await asyncio.to_thread(os.path.realpath, self.file)
|
||||||
directory = os.path.dirname(target) or "."
|
directory = os.path.dirname(target) or "."
|
||||||
await asyncio.to_thread(os.makedirs, directory, exist_ok=True)
|
await asyncio.to_thread(os.makedirs, directory, exist_ok=True)
|
||||||
await self._sweep_stale_tmp(directory)
|
await self._sweep_stale_tmp(directory, os.path.basename(target))
|
||||||
|
|
||||||
payload = await asyncio.to_thread(json.dumps, cache, allow_nan=False)
|
payload = await asyncio.to_thread(json.dumps, cache, allow_nan=False)
|
||||||
tmp = f"{target}.{os.getpid()}.tmp"
|
tmp = f"{target}.{os.getpid()}.tmp"
|
||||||
@@ -149,19 +137,27 @@ class AioKV:
|
|||||||
log.exception("aiokv: failed to clean up temp file %s", tmp)
|
log.exception("aiokv: failed to clean up temp file %s", tmp)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
async def _sweep_stale_tmp(self, directory: str) -> None:
|
async def _sweep_stale_tmp(self, directory: str, base: str) -> None:
|
||||||
"""remove orphaned .<pid>.tmp files left by a hard crash of a different, dead process"""
|
"""remove orphaned .<pid>.tmp files left by a hard crash of a different, dead process
|
||||||
base = os.path.basename(await asyncio.to_thread(os.path.realpath, self.file))
|
|
||||||
pattern = os.path.join(directory, f"{base}.*.tmp")
|
matches only the exact shape _save() creates (<realpath basename>.<pid>.tmp), via a
|
||||||
for path in await asyncio.to_thread(glob.glob, pattern):
|
re.escape'd regex over a directory listing rather than a raw glob - a glob pattern
|
||||||
try:
|
both crosses unrelated dots (matching non-aiokv neighbors like foo.backup.999.tmp) and
|
||||||
pid = int(path.rsplit(".", 2)[-2])
|
mistreats glob metacharacters in the store's own filename (e.g. state[prod].json).
|
||||||
except (ValueError, IndexError):
|
`base` is the realpath'd basename _save() already resolved - passed in rather than
|
||||||
|
re-resolved here so a save does one realpath call, not two."""
|
||||||
|
candidate = re.compile(rf"^{re.escape(base)}\.(\d+)\.tmp$")
|
||||||
|
names = await asyncio.to_thread(os.listdir, directory)
|
||||||
|
for name in names:
|
||||||
|
match = candidate.match(name)
|
||||||
|
if not match:
|
||||||
continue
|
continue
|
||||||
|
pid = int(match.group(1))
|
||||||
if pid == os.getpid():
|
if pid == os.getpid():
|
||||||
continue
|
continue
|
||||||
if await asyncio.to_thread(self._pid_alive, pid):
|
if await asyncio.to_thread(self._pid_alive, pid):
|
||||||
continue
|
continue
|
||||||
|
path = os.path.join(directory, name)
|
||||||
try:
|
try:
|
||||||
await asyncio.to_thread(os.remove, path)
|
await asyncio.to_thread(os.remove, path)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
@@ -174,13 +170,13 @@ class AioKV:
|
|||||||
"""check whether pid refers to a live process, without permission to signal counting as alive"""
|
"""check whether pid refers to a live process, without permission to signal counting as alive"""
|
||||||
try:
|
try:
|
||||||
os.kill(pid, 0)
|
os.kill(pid, 0)
|
||||||
except ProcessLookupError:
|
except (ProcessLookupError, OverflowError):
|
||||||
|
# OverflowError: a pid too large for a C int can't name a live process
|
||||||
return False
|
return False
|
||||||
except PermissionError:
|
except PermissionError:
|
||||||
return True
|
return True
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
# back-compat: this lib was originally named aiocache; legacy call sites using
|
# back-compat: originally named aiocache; prefer AioKV in new code.
|
||||||
# `aiocache(...)` keep working via this alias. prefer AioKV in new code.
|
|
||||||
aiocache = AioKV
|
aiocache = AioKV
|
||||||
|
|||||||
Reference in New Issue
Block a user