2 Commits
Author SHA1 Message Date
dsql 6a10f3acc0 fix: tiered rotate='size' actually rotates and tiers (logsetup-1, logsetup-4)
tiered size mode was broken two ways: (1) stdlib RotatingFileHandler's numbered .1/.2 shift
can't manage files redirected into log_dir, so every roll overwrote slot 1 -> ~99% of
history silently lost; (2) doRollover is gated on backupCount>0, so backup_count=0 (which
the docstring says is ignored in tiered mode) meant NO rotation + unbounded live file.

fix: tiered size now uses a timestamped per-roll namer (make_size_namer) like daily/on_start
so retier manages the pile, and forces a nonzero internal backupCount so the roll always
fires (retier bounds retention, not backupCount). non-tiered size unchanged.

verified: tiered size rotates + bounds to tier total (was stuck at 1); backup_count=0 rotates
+ live file bounded (was unbounded); legacy size back-compat intact; v0.4.0/v0.4.1 suites
still pass. bump v0.4.1 -> v0.4.2

Signed-off-by: disqualifier <dev@disqualifier.me>
2026-06-30 21:04:58 -04:00
dsql ece9a6b9ca fix: retention dead when name contains a directory (basename the stem)
rolled files land in log_dir under their basename (the namer/rotate_on_start basename
them), but prune()/retier() globbed the un-basenamed name. a name like 'sub/run' matched
nothing, so old .log/.gz files piled up forever (slow disk leak; the live file was fine).
basename the stem at the top of both prune() and retier(). regression-verified with
name='sub/run' (10 retained vs 12/dead before). bump v0.4.0 -> v0.4.1

Signed-off-by: disqualifier <dev@disqualifier.me>
2026-06-30 03:49:35 -04:00
5 changed files with 56 additions and 6 deletions
+2 -2
View File
@@ -13,12 +13,12 @@ and emit; their records flow into the handlers `log_setup` wired.
## Install ## Install
``` ```
log_setup @ git+ssh://git@git.rethinkstudios.io/rethink-public/log_setup.git@v0.4.0 log_setup @ git+ssh://git@git.rethinkstudios.io/rethink-public/log_setup.git@v0.4.2
``` ```
No dependencies — stdlib only. No dependencies — stdlib only.
Drop the `@v0.4.0` suffix from the line above to install the latest unpinned. Drop the `@v0.4.2` suffix from the line above to install the latest unpinned.
## Quick start ## Quick start
+1 -1
View File
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project] [project]
name = "log_setup" name = "log_setup"
version = "0.4.0" version = "0.4.2"
description = "stdlib app-entry-point logging setup: live run.log, rotation, gzip, retention, consistent format" description = "stdlib app-entry-point logging setup: live run.log, rotation, gzip, retention, consistent format"
requires-python = ">=3.10" requires-python = ">=3.10"
dependencies = [] dependencies = []
+1 -1
View File
@@ -19,4 +19,4 @@ from .setup import setup_logging
__all__ = ["setup_logging"] __all__ = ["setup_logging"]
__version__ = "0.4.0" __version__ = "0.4.2"
+45 -1
View File
@@ -62,6 +62,30 @@ def make_namer(log_dir: str, compress: bool) -> Callable[[str], str]:
return namer return namer
def make_size_namer(
stem: str, log_dir: str, clock=time.localtime,
) -> Callable[[str], str]:
"""namer for tiered SIZE mode: a unique timestamped dest per roll, plain (no .gz)
stdlib RotatingFileHandler names rolls `<live>.1`, `<live>.2`, ... and shifts them —
a scheme that breaks once files are redirected into log_dir (the shift can't find
them, so every roll reuses slot 1). tiered retention wants unique per-roll names it
can rank + tier like the daily/on_start paths, so ignore the handler's `.N` suffix
entirely and mint `<stem>.<Y-m-d_H-M-S>.log`, disambiguating a same-second collision
(against both the .log and .log.gz forms) with a counter. always plain — retier
decides compression.
"""
def namer(default_name: str) -> str:
stamp = time.strftime("%Y-%m-%d_%H-%M-%S", clock())
dest = os.path.join(log_dir, f"{stem}.{stamp}.log")
counter = 1
while os.path.exists(dest) or os.path.exists(dest + ".gz"):
dest = os.path.join(log_dir, f"{stem}.{stamp}.{counter}.log")
counter += 1
return dest
return namer
def make_rotator( def make_rotator(
compress: bool, log_dir: Optional[str] = None, compress: bool, log_dir: Optional[str] = None,
prune_stem: Optional[str] = None, backup_count: int = 0, prune_stem: Optional[str] = None, backup_count: int = 0,
@@ -154,7 +178,13 @@ def retier(log_dir: str, stem: str, keep_uncompressed: int, keep_compressed: int
to <name>.gz and the plain source removed), and everything beyond to <name>.gz and the plain source removed), and everything beyond
keep_uncompressed+keep_compressed is deleted. the live <stem>.log is never touched. keep_uncompressed+keep_compressed is deleted. the live <stem>.log is never touched.
fail-soft per file (skip on OSError) so retention never crashes setup. fail-soft per file (skip on OSError) so retention never crashes setup.
`stem` is reduced to its basename: rolled files land in log_dir under the basename
(the namer/rotate_on_start basename them), so a `name` containing a directory (e.g.
"sub/run") must be matched by "run." here or nothing matches and retention silently
never fires (unbounded pileup).
""" """
stem = os.path.basename(stem)
try: try:
names = [ names = [
name for name in os.listdir(log_dir) name for name in os.listdir(log_dir)
@@ -188,9 +218,14 @@ def prune(log_dir: str, stem: str, backup_count: int) -> None:
matches files beginning with `<stem>.` (e.g. run.*), sorted by mtime, deleting the matches files beginning with `<stem>.` (e.g. run.*), sorted by mtime, deleting the
oldest beyond the count. used for on_start, which the handlers don't auto-prune. oldest beyond the count. used for on_start, which the handlers don't auto-prune.
`stem` is reduced to its basename so a `name` containing a directory (e.g. "sub/run")
still matches the basenamed rolled files in log_dir (else nothing matches and old
files pile up forever).
""" """
if backup_count <= 0: if backup_count <= 0:
return return
stem = os.path.basename(stem)
try: try:
entries = [ entries = [
os.path.join(log_dir, name) os.path.join(log_dir, name)
@@ -220,6 +255,7 @@ def attach_rolling(
handler, log_dir: str, compress: bool, handler, log_dir: str, compress: bool,
prune_stem: Optional[str] = None, backup_count: int = 0, prune_stem: Optional[str] = None, backup_count: int = 0,
keep_uncompressed: Optional[int] = None, keep_compressed: Optional[int] = None, keep_uncompressed: Optional[int] = None, keep_compressed: Optional[int] = None,
size_tiered: bool = False,
) -> Tuple[Callable, Callable]: ) -> Tuple[Callable, Callable]:
"""wire the custom namer + rotator onto a rotating handler; return them """wire the custom namer + rotator onto a rotating handler; return them
@@ -227,8 +263,16 @@ def attach_rolling(
(the handler's own retention can't see the redirected rolled files). pass (the handler's own retention can't see the redirected rolled files). pass
`keep_uncompressed`/`keep_compressed` instead to use tiered retention (newest plain, `keep_uncompressed`/`keep_compressed` instead to use tiered retention (newest plain,
next gzipped, rest deleted) — see make_rotator. next gzipped, rest deleted) — see make_rotator.
`size_tiered` uses a timestamped per-roll namer (make_size_namer) instead of the
default one, for a tiered RotatingFileHandler (size mode): stdlib's `.1/.2` numbered
shift can't manage files redirected into log_dir, so each roll gets a unique dated
name that retier ranks/tiers like the daily/on_start paths.
""" """
namer = make_namer(log_dir, compress) if size_tiered:
namer = make_size_namer(os.path.basename(prune_stem or ""), log_dir)
else:
namer = make_namer(log_dir, compress)
rotator = make_rotator( rotator = make_rotator(
compress, log_dir, prune_stem, backup_count, keep_uncompressed, keep_compressed, compress, log_dir, prune_stem, backup_count, keep_uncompressed, keep_compressed,
) )
+7 -1
View File
@@ -128,12 +128,18 @@ def _file_handler(
"""build the configured file handler with custom rolling into log_dir""" """build the configured file handler with custom rolling into log_dir"""
tiered = keep_uncompressed is not None or keep_compressed is not None tiered = keep_uncompressed is not None or keep_compressed is not None
if rotate == "size": if rotate == "size":
# stdlib doRollover is a no-op when backupCount == 0, and its numbered .1/.2 shift
# can't manage files redirected into log_dir. in tiered size mode force a nonzero
# backupCount so the roll always fires (retier bounds retention, not backupCount)
# and use the timestamped size-namer (size_tiered) instead of the .N shift.
size_backup = backup_count if not tiered else max(backup_count, 1)
handler = logging.handlers.RotatingFileHandler( handler = logging.handlers.RotatingFileHandler(
live_path, maxBytes=max_bytes, backupCount=backup_count, encoding="utf-8", live_path, maxBytes=max_bytes, backupCount=size_backup, encoding="utf-8",
) )
attach_rolling( attach_rolling(
handler, log_dir, compress, prune_stem=name, backup_count=backup_count, handler, log_dir, compress, prune_stem=name, backup_count=backup_count,
keep_uncompressed=keep_uncompressed, keep_compressed=keep_compressed, keep_uncompressed=keep_uncompressed, keep_compressed=keep_compressed,
size_tiered=tiered,
) )
elif rotate == "daily": elif rotate == "daily":
handler = logging.handlers.TimedRotatingFileHandler( handler = logging.handlers.TimedRotatingFileHandler(