fix: prune/retier recognize legacy rolls named off the live name; correct gzip docs

Legacy pre-v0.5.0 daily rolls were named off the LIVE file name (run.log.<date>), but
prune/retier only matched the history stem (the cwd basename), so in the default upgrade
path an existing legacy backlog was never recognized and piled up forever. prune/retier/
make_rotator/attach_rolling now thread the live name as an optional live_stem so both shapes
are pruned; a foreign same-stem file is still left alone (all forms stay date-bearing).
README + CLAUDE.md corrected to describe the atomic compress-or-skip behavior (40310b8 removed
the runtime _gz_intact reconciliation the docs still claimed).

Signed-off-by: disqualifier <dev@disqualifier.me>
This commit is contained in:
2026-07-06 19:35:53 -04:00
parent 3f3a797fde
commit cb8acac76f
3 changed files with 39 additions and 26 deletions
+7 -7
View File
@@ -258,13 +258,13 @@ setup_logging(name="run", queue=True)
rotate-mode aware: the zero-retention delete only ever fires for `"size"`, matching the rotate-mode aware: the zero-retention delete only ever fires for `"size"`, matching the
contract in the bullet above — `"daily"`/`"on_start"` with `backup_count=0` were always contract in the bullet above — `"daily"`/`"on_start"` with `backup_count=0` were always
meant to roll without pruning and now do again. meant to roll without pruning and now do again.
- **Gzip writes are crash-safe (v0.5.1+).** `_gzip_file` now writes to a `.tmp` sibling and - **Gzip is atomic compress-or-skip (v0.5.1+).** `_gzip_file` writes to a `.tmp` sibling,
atomically `os.replace`s it onto the final `.gz` path, so a crash/OOM/power-loss mid-write verifies it decompresses cleanly, and only then `os.replace`s it onto the final `.gz` and
can never leave a truncated `.gz` at the path retention logic trusts. Tiered retention's removes the plain source — a crash/OOM/interrupt at any point leaves the plain `.log`
plain/gz dedupe additionally verifies a `.gz` decompresses cleanly before deleting its intact with no `.gz` (or the `.tmp` cleaned up), so this code never produces a truncated
plain twin — a corrupt `.gz` (from before this fix, or an external cause) is never `.gz` at the path retention trusts. Because a corrupt `.gz` can't arise from this path, the
preferred over an intact plain copy; the plain is kept and the `.gz` gets rewritten tiered retention dedupe drops a plain twin unconditionally when its `.gz` exists (no
cleanly on the next retier pass instead of being deleted. runtime `_gz_intact` reconciliation — that was removed once compression became atomic).
- **Setup-time warnings reach the log file (v0.6.0+).** Previously, a warning raised - **Setup-time warnings reach the log file (v0.6.0+).** Previously, a warning raised
during `setup_logging` itself (an invalid `module_levels` entry, a handler failing to during `setup_logging` itself (an invalid `module_levels` entry, a handler failing to
close on re-setup, an unknown `rotate` value) was emitted *before* any handler was close on re-setup, an unknown `rotate` value) was emitted *before* any handler was
+29 -18
View File
@@ -132,7 +132,7 @@ def make_rotator(
compress: bool, log_dir: Optional[str] = None, compress: bool, log_dir: Optional[str] = None,
prune_stem: Optional[str] = None, backup_count: int = 0, prune_stem: Optional[str] = None, backup_count: int = 0,
keep_uncompressed: Optional[int] = None, keep_compressed: Optional[int] = None, keep_uncompressed: Optional[int] = None, keep_compressed: Optional[int] = None,
rotate_mode: Optional[str] = None, rotate_mode: Optional[str] = None, live_stem: Optional[str] = None,
) -> Callable[[str, str], None]: ) -> Callable[[str, str], None]:
"""rotator: move (or gzip) the source live file to the destination rolled path """rotator: move (or gzip) the source live file to the destination rolled path
@@ -162,7 +162,7 @@ def make_rotator(
plain_dest = _free_dest(dest[:-3] if dest.endswith(".gz") else dest) plain_dest = _free_dest(dest[:-3] if dest.endswith(".gz") else dest)
_move(source, plain_dest) _move(source, plain_dest)
if log_dir is not None and prune_stem is not None: if log_dir is not None and prune_stem is not None:
retier(log_dir, prune_stem, keep_uncompressed or 0, keep_compressed or 0) retier(log_dir, prune_stem, keep_uncompressed or 0, keep_compressed or 0, live_stem)
return return
if compress: if compress:
_gzip_file(source, dest) _gzip_file(source, dest)
@@ -175,7 +175,7 @@ def make_rotator(
except OSError: except OSError:
pass pass
elif log_dir is not None and prune_stem is not None: elif log_dir is not None and prune_stem is not None:
prune(log_dir, prune_stem, backup_count) prune(log_dir, prune_stem, backup_count, live_stem)
return rotator return rotator
@@ -225,7 +225,8 @@ def rotate_on_start(
retier(log_dir, stem, keep_uncompressed or 0, keep_compressed or 0) retier(log_dir, stem, keep_uncompressed or 0, keep_compressed or 0)
def retier(log_dir: str, stem: str, keep_uncompressed: int, keep_compressed: int) -> None: def retier(log_dir: str, stem: str, keep_uncompressed: int, keep_compressed: int,
live_stem: Optional[str] = None) -> None:
"""re-tier rolled files for stem: newest plain, next gzipped, rest deleted """re-tier rolled files for stem: newest plain, next gzipped, rest deleted
newest-first by mtime: the first `keep_uncompressed` stay uncompressed, the next newest-first by mtime: the first `keep_uncompressed` stay uncompressed, the next
@@ -248,10 +249,11 @@ def retier(log_dir: str, stem: str, keep_uncompressed: int, keep_compressed: int
the same log_dir) is left alone rather than tiered/gzipped/deleted. the same log_dir) is left alone rather than tiered/gzipped/deleted.
""" """
stem = os.path.basename(stem) stem = os.path.basename(stem)
live_stem = os.path.basename(live_stem) if live_stem else None
try: try:
names = [ names = [
name for name in os.listdir(log_dir) name for name in os.listdir(log_dir)
if name != f"{stem}.log" and _is_rolled_name(stem, name) if name != f"{stem}.log" and _is_rolled_name(stem, name, live_stem)
] ]
except OSError: except OSError:
return return
@@ -297,26 +299,30 @@ def retier(log_dir: str, stem: str, keep_uncompressed: int, keep_compressed: int
pass pass
def _rolled_name_pattern(stem: str) -> "re.Pattern": def _rolled_name_pattern(stem: str, live_stem: Optional[str] = None) -> "re.Pattern":
"""compiled regex matching this lib's own rolled-file shapes for stem """compiled regex matching this lib's own rolled-file shapes for stem
two forms, both date-bearing so a foreign same-stem file (e.g. `proj.audit.log`) is all forms are date-bearing so a foreign same-stem file (e.g. `proj.audit.log`) is
never mistaken for a roll and pruned/retiered/deleted: never mistaken for a roll and pruned/retiered/deleted:
- current uniform namer: `<stem>.<Y-m-d_H-M-S>[.N].log[.gz]` - current uniform namer: `<stem>.<Y-m-d_H-M-S>[.N].log[.gz]`
(make_history_namer/rotate_on_start, see attach_rolling) (make_history_namer/rotate_on_start, see attach_rolling)
- legacy pre-v0.5.0 daily: `<stem>.log.<Y-m-d>[.gz]` (the stdlib TimedRotatingFileHandler - legacy pre-v0.5.0 daily: `<stem>.log.<Y-m-d>[.gz]` (the stdlib TimedRotatingFileHandler
shape) - matched so an upgraded deployment's existing history is still pruned/retired shape) - matched so an upgraded deployment's existing history is still pruned/retired
instead of piling up forever instead of piling up forever. legacy rolls were named off the LIVE file's name, not the
history stem, so when `live_stem` is given (and differs) its legacy shape is matched too
""" """
esc = re.escape(stem) stems = [stem] if live_stem is None or live_stem == stem else [stem, live_stem]
uniform = rf"{esc}\.\d{{4}}-\d{{2}}-\d{{2}}_\d{{2}}-\d{{2}}-\d{{2}}(?:\.\d+)?\.log(?:\.gz)?" forms = []
legacy = rf"{esc}\.log\.\d{{4}}-\d{{2}}-\d{{2}}(?:\.gz)?" for s in stems:
return re.compile(rf"^(?:{uniform}|{legacy})$") esc = re.escape(s)
forms.append(rf"{esc}\.\d{{4}}-\d{{2}}-\d{{2}}_\d{{2}}-\d{{2}}-\d{{2}}(?:\.\d+)?\.log(?:\.gz)?")
forms.append(rf"{esc}\.log\.\d{{4}}-\d{{2}}-\d{{2}}(?:\.gz)?")
return re.compile(rf"^(?:{'|'.join(forms)})$")
def _is_rolled_name(stem: str, name: str) -> bool: def _is_rolled_name(stem: str, name: str, live_stem: Optional[str] = None) -> bool:
"""return whether name has this lib's own rolled-log shape for stem""" """return whether name has this lib's own rolled-log shape for stem (or the live stem)"""
return _rolled_name_pattern(stem).match(name) is not None return _rolled_name_pattern(stem, live_stem).match(name) is not None
def _roll_counter(path: str) -> int: def _roll_counter(path: str) -> int:
@@ -335,7 +341,7 @@ def _roll_counter(path: str) -> int:
return int(tail) if tail.isdigit() else 0 return int(tail) if tail.isdigit() else 0
def prune(log_dir: str, stem: str, backup_count: int) -> None: def prune(log_dir: str, stem: str, backup_count: int, live_stem: Optional[str] = None) -> None:
"""keep only the newest `backup_count` rolled files for a given stem in log_dir """keep only the newest `backup_count` rolled files for a given stem in log_dir
matches files beginning with `<stem>.` (e.g. run.*), sorted newest-first by mtime matches files beginning with `<stem>.` (e.g. run.*), sorted newest-first by mtime
@@ -354,11 +360,12 @@ def prune(log_dir: str, stem: str, backup_count: int) -> None:
if backup_count <= 0: if backup_count <= 0:
return return
stem = os.path.basename(stem) stem = os.path.basename(stem)
live_stem = os.path.basename(live_stem) if live_stem else None
try: try:
entries = [ entries = [
os.path.join(log_dir, name) os.path.join(log_dir, name)
for name in os.listdir(log_dir) for name in os.listdir(log_dir)
if name != f"{stem}.log" and _is_rolled_name(stem, name) if name != f"{stem}.log" and _is_rolled_name(stem, name, live_stem)
] ]
except OSError: except OSError:
return return
@@ -403,9 +410,13 @@ def attach_rolling(
namer = make_history_namer( namer = make_history_namer(
os.path.basename(prune_stem or ""), log_dir, compress, plain=tiered, os.path.basename(prune_stem or ""), log_dir, compress, plain=tiered,
) )
# the live file name (stem of the handler's baseFilename, minus one .log) - so retention
# also recognizes pre-v0.5.0 legacy rolls, which were named off the LIVE name not the stem
live_base = os.path.basename(getattr(handler, "baseFilename", "") or "")
live_stem = live_base[:-4] if live_base.endswith(".log") else live_base
rotator = make_rotator( rotator = make_rotator(
compress, log_dir, prune_stem, backup_count, keep_uncompressed, keep_compressed, compress, log_dir, prune_stem, backup_count, keep_uncompressed, keep_compressed,
rotate_mode=rotate_mode, rotate_mode=rotate_mode, live_stem=live_stem or None,
) )
handler.namer = namer handler.namer = namer
handler.rotator = rotator handler.rotator = rotator
+3 -1
View File
@@ -185,7 +185,9 @@ def _file_handler(
) )
else: else:
rotate_on_start(live_path, log_dir, compress, history_stem=history_stem) rotate_on_start(live_path, log_dir, compress, history_stem=history_stem)
prune(log_dir, history_stem, backup_count) live_base = os.path.basename(live_path)
live_stem = live_base[:-4] if live_base.endswith(".log") else live_base
prune(log_dir, history_stem, backup_count, live_stem or None)
elif rotate is not None: elif rotate is not None:
# a typo'd value would otherwise silently fall through to a non-rotating # a typo'd value would otherwise silently fall through to a non-rotating
# FileHandler and grow forever - warn instead of degrade silently # FileHandler and grow forever - warn instead of degrade silently