fix: clear() and stale-tmp sweep resolve the real symlink target and match only <base>.<pid>.tmp
clear() removed the raw path, so on a symlinked store it unlinked only the
symlink while the real data file kept every value - clear() returned True
but the data resurfaced if the symlink was recreated. It now realpaths the
target first, mirroring _save()'s symlink-safe writes.
_sweep_stale_tmp globbed {base}.*.tmp, which crosses dots and can match an
unrelated neighbor file (deleting it outside the lib's own artifacts), and
interpolated the store's basename unescaped, so glob metacharacters in the
filename (e.g. state[prod].json) either missed the store's own orphans or
cross-matched a different store's. The sweep now lists the directory and
matches a re.escape'd regex anchored to the exact <base>.<pid>.tmp shape
_save() creates.
Signed-off-by: disqualifier <dev@disqualifier.me>
This commit is contained in:
@@ -12,18 +12,18 @@ you `delete` or `clear` them.
|
|||||||
`requirements.txt`:
|
`requirements.txt`:
|
||||||
|
|
||||||
```
|
```
|
||||||
aiokv @ git+ssh://git@git.rethinkstudios.io/rethink-public/aiokv.git@v0.2.1
|
aiokv @ git+ssh://git@git.rethinkstudios.io/rethink-public/aiokv.git@v0.2.2
|
||||||
```
|
```
|
||||||
|
|
||||||
Direct:
|
Direct:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install "aiokv @ git+ssh://git@git.rethinkstudios.io/rethink-public/aiokv.git@v0.2.1"
|
pip install "aiokv @ git+ssh://git@git.rethinkstudios.io/rethink-public/aiokv.git@v0.2.2"
|
||||||
```
|
```
|
||||||
|
|
||||||
Requires `aiofiles` (pulled transitively).
|
Requires `aiofiles` (pulled transitively).
|
||||||
|
|
||||||
Drop the `@v0.2.1` suffix from the line above to install the latest unpinned.
|
Drop the `@v0.2.2` suffix from the line above to install the latest unpinned.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
@@ -66,11 +66,17 @@ Writes are **atomic**: data is written to a temp file in the same directory and
|
|||||||
written through rather than clobbered. A **process** crash mid-write leaves the previous
|
written through rather than clobbered. A **process** crash mid-write leaves the previous
|
||||||
good file intact, and a reader never observes a partial file. (This is process-crash
|
good file intact, and a reader never observes a partial file. (This is process-crash
|
||||||
safety, not power-loss durability — there's no `fsync`, so an OS/power failure could
|
safety, not power-loss durability — there's no `fsync`, so an OS/power failure could
|
||||||
still lose the last write; fine for reconstructible single-process state.) Orphaned
|
still lose the last write; fine for reconstructible single-process state.) `clear()` is
|
||||||
`.<pid>.tmp` files left by a hard crash of a *different, dead* process are swept on the
|
symlink-safe the same way: it removes the realpath'd target, not the symlink itself, so
|
||||||
next save. A single `asyncio.Lock` guards every read and write, so concurrent operations
|
a symlinked store is genuinely erased rather than just having its symlink unlinked.
|
||||||
on one instance are consistent and no update is lost. All blocking filesystem calls and
|
Orphaned `.<pid>.tmp` files left by a hard crash of a *different, dead* process are swept
|
||||||
JSON (de)serialization run via `asyncio.to_thread`, so nothing stalls the event loop.
|
on the next save; the sweep matches only the exact `<realpath basename>.<pid>.tmp` shape
|
||||||
|
it creates, so a differently-shaped neighbor file is never touched and a store filename
|
||||||
|
containing glob-like characters (e.g. `state[prod].json`) doesn't miss its own orphans or
|
||||||
|
cross-match another store's. A single `asyncio.Lock` guards every read and write, so
|
||||||
|
concurrent operations on one instance are consistent and no update is lost. All blocking
|
||||||
|
filesystem calls and JSON (de)serialization run via `asyncio.to_thread`, so nothing stalls
|
||||||
|
the event loop.
|
||||||
|
|
||||||
## Scope — read this
|
## Scope — read this
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "aiokv"
|
name = "aiokv"
|
||||||
version = "0.2.1"
|
version = "0.2.2"
|
||||||
description = "Async file-backed key-value store for single-process local state — atomic writes, no TTL, config-free, installable."
|
description = "Async file-backed key-value store for single-process local state — atomic writes, no TTL, config-free, installable."
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
|||||||
+20
-9
@@ -17,9 +17,9 @@ cross-process storage.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import glob
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
from typing import Any, Dict
|
from typing import Any, Dict
|
||||||
@@ -71,11 +71,15 @@ class AioKV:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
async def clear(self) -> bool:
|
async def clear(self) -> bool:
|
||||||
"""remove the backing file entirely; True on success or if already absent, False on error"""
|
"""remove the backing file entirely; True on success or if already absent, False on error
|
||||||
|
|
||||||
|
symlink-safe: removes the realpath'd target rather than the symlink itself, mirroring
|
||||||
|
_save()'s symlink-safe writes - a symlinked store's real data file is what gets erased"""
|
||||||
try:
|
try:
|
||||||
async with self.lock:
|
async with self.lock:
|
||||||
|
target = await asyncio.to_thread(os.path.realpath, self.file)
|
||||||
try:
|
try:
|
||||||
await asyncio.to_thread(os.remove, self.file)
|
await asyncio.to_thread(os.remove, target)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
pass
|
pass
|
||||||
return True
|
return True
|
||||||
@@ -134,18 +138,25 @@ class AioKV:
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
async def _sweep_stale_tmp(self, directory: str) -> None:
|
async def _sweep_stale_tmp(self, directory: str) -> None:
|
||||||
"""remove orphaned .<pid>.tmp files left by a hard crash of a different, dead process"""
|
"""remove orphaned .<pid>.tmp files left by a hard crash of a different, dead process
|
||||||
|
|
||||||
|
matches only the exact shape _save() creates (<realpath basename>.<pid>.tmp), via a
|
||||||
|
re.escape'd regex over a directory listing rather than a raw glob - a glob pattern
|
||||||
|
both crosses unrelated dots (matching non-aiokv neighbors like foo.backup.999.tmp) and
|
||||||
|
mistreats glob metacharacters in the store's own filename (e.g. state[prod].json)"""
|
||||||
base = os.path.basename(await asyncio.to_thread(os.path.realpath, self.file))
|
base = os.path.basename(await asyncio.to_thread(os.path.realpath, self.file))
|
||||||
pattern = os.path.join(directory, f"{base}.*.tmp")
|
candidate = re.compile(rf"^{re.escape(base)}\.(\d+)\.tmp$")
|
||||||
for path in await asyncio.to_thread(glob.glob, pattern):
|
names = await asyncio.to_thread(os.listdir, directory)
|
||||||
try:
|
for name in names:
|
||||||
pid = int(path.rsplit(".", 2)[-2])
|
match = candidate.match(name)
|
||||||
except (ValueError, IndexError):
|
if not match:
|
||||||
continue
|
continue
|
||||||
|
pid = int(match.group(1))
|
||||||
if pid == os.getpid():
|
if pid == os.getpid():
|
||||||
continue
|
continue
|
||||||
if await asyncio.to_thread(self._pid_alive, pid):
|
if await asyncio.to_thread(self._pid_alive, pid):
|
||||||
continue
|
continue
|
||||||
|
path = os.path.join(directory, name)
|
||||||
try:
|
try:
|
||||||
await asyncio.to_thread(os.remove, path)
|
await asyncio.to_thread(os.remove, path)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
|
|||||||
Reference in New Issue
Block a user