fix: prune/retier recognize legacy rolls named off the live name; correct gzip docs
Legacy pre-v0.5.0 daily rolls were named off the LIVE file name (run.log.<date>), but
prune/retier only matched the history stem (the cwd basename), so in the default upgrade
path an existing legacy backlog was never recognized and piled up forever. prune/retier/
make_rotator/attach_rolling now thread the live name as an optional live_stem so both shapes
are pruned; a foreign same-stem file is still left alone (all forms stay date-bearing).
README + CLAUDE.md corrected to describe the atomic compress-or-skip behavior (40310b8 removed
the runtime _gz_intact reconciliation the docs still claimed).
Signed-off-by: disqualifier <dev@disqualifier.me>
This commit is contained in:
@@ -258,13 +258,13 @@ setup_logging(name="run", queue=True)
|
||||
rotate-mode aware: the zero-retention delete only ever fires for `"size"`, matching the
|
||||
contract in the bullet above — `"daily"`/`"on_start"` with `backup_count=0` were always
|
||||
meant to roll without pruning and now do again.
|
||||
- **Gzip writes are crash-safe (v0.5.1+).** `_gzip_file` now writes to a `.tmp` sibling and
|
||||
atomically `os.replace`s it onto the final `.gz` path, so a crash/OOM/power-loss mid-write
|
||||
can never leave a truncated `.gz` at the path retention logic trusts. Tiered retention's
|
||||
plain/gz dedupe additionally verifies a `.gz` decompresses cleanly before deleting its
|
||||
plain twin — a corrupt `.gz` (from before this fix, or an external cause) is never
|
||||
preferred over an intact plain copy; the plain is kept and the `.gz` gets rewritten
|
||||
cleanly on the next retier pass instead of being deleted.
|
||||
- **Gzip is atomic compress-or-skip (v0.5.1+).** `_gzip_file` writes to a `.tmp` sibling,
|
||||
verifies it decompresses cleanly, and only then `os.replace`s it onto the final `.gz` and
|
||||
removes the plain source — a crash/OOM/interrupt at any point leaves the plain `.log`
|
||||
intact with no `.gz` (or the `.tmp` cleaned up), so this code never produces a truncated
|
||||
`.gz` at the path retention trusts. Because a corrupt `.gz` can't arise from this path, the
|
||||
tiered retention dedupe drops a plain twin unconditionally when its `.gz` exists (no
|
||||
runtime `_gz_intact` reconciliation — that was removed once compression became atomic).
|
||||
- **Setup-time warnings reach the log file (v0.6.0+).** Previously, a warning raised
|
||||
during `setup_logging` itself (an invalid `module_levels` entry, a handler failing to
|
||||
close on re-setup, an unknown `rotate` value) was emitted *before* any handler was
|
||||
|
||||
+29
-18
@@ -132,7 +132,7 @@ def make_rotator(
|
||||
compress: bool, log_dir: Optional[str] = None,
|
||||
prune_stem: Optional[str] = None, backup_count: int = 0,
|
||||
keep_uncompressed: Optional[int] = None, keep_compressed: Optional[int] = None,
|
||||
rotate_mode: Optional[str] = None,
|
||||
rotate_mode: Optional[str] = None, live_stem: Optional[str] = None,
|
||||
) -> Callable[[str, str], None]:
|
||||
"""rotator: move (or gzip) the source live file to the destination rolled path
|
||||
|
||||
@@ -162,7 +162,7 @@ def make_rotator(
|
||||
plain_dest = _free_dest(dest[:-3] if dest.endswith(".gz") else dest)
|
||||
_move(source, plain_dest)
|
||||
if log_dir is not None and prune_stem is not None:
|
||||
retier(log_dir, prune_stem, keep_uncompressed or 0, keep_compressed or 0)
|
||||
retier(log_dir, prune_stem, keep_uncompressed or 0, keep_compressed or 0, live_stem)
|
||||
return
|
||||
if compress:
|
||||
_gzip_file(source, dest)
|
||||
@@ -175,7 +175,7 @@ def make_rotator(
|
||||
except OSError:
|
||||
pass
|
||||
elif log_dir is not None and prune_stem is not None:
|
||||
prune(log_dir, prune_stem, backup_count)
|
||||
prune(log_dir, prune_stem, backup_count, live_stem)
|
||||
return rotator
|
||||
|
||||
|
||||
@@ -225,7 +225,8 @@ def rotate_on_start(
|
||||
retier(log_dir, stem, keep_uncompressed or 0, keep_compressed or 0)
|
||||
|
||||
|
||||
def retier(log_dir: str, stem: str, keep_uncompressed: int, keep_compressed: int) -> None:
|
||||
def retier(log_dir: str, stem: str, keep_uncompressed: int, keep_compressed: int,
|
||||
live_stem: Optional[str] = None) -> None:
|
||||
"""re-tier rolled files for stem: newest plain, next gzipped, rest deleted
|
||||
|
||||
newest-first by mtime: the first `keep_uncompressed` stay uncompressed, the next
|
||||
@@ -248,10 +249,11 @@ def retier(log_dir: str, stem: str, keep_uncompressed: int, keep_compressed: int
|
||||
the same log_dir) is left alone rather than tiered/gzipped/deleted.
|
||||
"""
|
||||
stem = os.path.basename(stem)
|
||||
live_stem = os.path.basename(live_stem) if live_stem else None
|
||||
try:
|
||||
names = [
|
||||
name for name in os.listdir(log_dir)
|
||||
if name != f"{stem}.log" and _is_rolled_name(stem, name)
|
||||
if name != f"{stem}.log" and _is_rolled_name(stem, name, live_stem)
|
||||
]
|
||||
except OSError:
|
||||
return
|
||||
@@ -297,26 +299,30 @@ def retier(log_dir: str, stem: str, keep_uncompressed: int, keep_compressed: int
|
||||
pass
|
||||
|
||||
|
||||
def _rolled_name_pattern(stem: str) -> "re.Pattern":
|
||||
def _rolled_name_pattern(stem: str, live_stem: Optional[str] = None) -> "re.Pattern":
|
||||
"""compiled regex matching this lib's own rolled-file shapes for stem
|
||||
|
||||
two forms, both date-bearing so a foreign same-stem file (e.g. `proj.audit.log`) is
|
||||
all forms are date-bearing so a foreign same-stem file (e.g. `proj.audit.log`) is
|
||||
never mistaken for a roll and pruned/retiered/deleted:
|
||||
- current uniform namer: `<stem>.<Y-m-d_H-M-S>[.N].log[.gz]`
|
||||
(make_history_namer/rotate_on_start, see attach_rolling)
|
||||
- legacy pre-v0.5.0 daily: `<stem>.log.<Y-m-d>[.gz]` (the stdlib TimedRotatingFileHandler
|
||||
shape) - matched so an upgraded deployment's existing history is still pruned/retired
|
||||
instead of piling up forever
|
||||
instead of piling up forever. legacy rolls were named off the LIVE file's name, not the
|
||||
history stem, so when `live_stem` is given (and differs) its legacy shape is matched too
|
||||
"""
|
||||
esc = re.escape(stem)
|
||||
uniform = rf"{esc}\.\d{{4}}-\d{{2}}-\d{{2}}_\d{{2}}-\d{{2}}-\d{{2}}(?:\.\d+)?\.log(?:\.gz)?"
|
||||
legacy = rf"{esc}\.log\.\d{{4}}-\d{{2}}-\d{{2}}(?:\.gz)?"
|
||||
return re.compile(rf"^(?:{uniform}|{legacy})$")
|
||||
stems = [stem] if live_stem is None or live_stem == stem else [stem, live_stem]
|
||||
forms = []
|
||||
for s in stems:
|
||||
esc = re.escape(s)
|
||||
forms.append(rf"{esc}\.\d{{4}}-\d{{2}}-\d{{2}}_\d{{2}}-\d{{2}}-\d{{2}}(?:\.\d+)?\.log(?:\.gz)?")
|
||||
forms.append(rf"{esc}\.log\.\d{{4}}-\d{{2}}-\d{{2}}(?:\.gz)?")
|
||||
return re.compile(rf"^(?:{'|'.join(forms)})$")
|
||||
|
||||
|
||||
def _is_rolled_name(stem: str, name: str) -> bool:
|
||||
"""return whether name has this lib's own rolled-log shape for stem"""
|
||||
return _rolled_name_pattern(stem).match(name) is not None
|
||||
def _is_rolled_name(stem: str, name: str, live_stem: Optional[str] = None) -> bool:
|
||||
"""return whether name has this lib's own rolled-log shape for stem (or the live stem)"""
|
||||
return _rolled_name_pattern(stem, live_stem).match(name) is not None
|
||||
|
||||
|
||||
def _roll_counter(path: str) -> int:
|
||||
@@ -335,7 +341,7 @@ def _roll_counter(path: str) -> int:
|
||||
return int(tail) if tail.isdigit() else 0
|
||||
|
||||
|
||||
def prune(log_dir: str, stem: str, backup_count: int) -> None:
|
||||
def prune(log_dir: str, stem: str, backup_count: int, live_stem: Optional[str] = None) -> None:
|
||||
"""keep only the newest `backup_count` rolled files for a given stem in log_dir
|
||||
|
||||
matches files beginning with `<stem>.` (e.g. run.*), sorted newest-first by mtime
|
||||
@@ -354,11 +360,12 @@ def prune(log_dir: str, stem: str, backup_count: int) -> None:
|
||||
if backup_count <= 0:
|
||||
return
|
||||
stem = os.path.basename(stem)
|
||||
live_stem = os.path.basename(live_stem) if live_stem else None
|
||||
try:
|
||||
entries = [
|
||||
os.path.join(log_dir, name)
|
||||
for name in os.listdir(log_dir)
|
||||
if name != f"{stem}.log" and _is_rolled_name(stem, name)
|
||||
if name != f"{stem}.log" and _is_rolled_name(stem, name, live_stem)
|
||||
]
|
||||
except OSError:
|
||||
return
|
||||
@@ -403,9 +410,13 @@ def attach_rolling(
|
||||
namer = make_history_namer(
|
||||
os.path.basename(prune_stem or ""), log_dir, compress, plain=tiered,
|
||||
)
|
||||
# the live file name (stem of the handler's baseFilename, minus one .log) - so retention
|
||||
# also recognizes pre-v0.5.0 legacy rolls, which were named off the LIVE name not the stem
|
||||
live_base = os.path.basename(getattr(handler, "baseFilename", "") or "")
|
||||
live_stem = live_base[:-4] if live_base.endswith(".log") else live_base
|
||||
rotator = make_rotator(
|
||||
compress, log_dir, prune_stem, backup_count, keep_uncompressed, keep_compressed,
|
||||
rotate_mode=rotate_mode,
|
||||
rotate_mode=rotate_mode, live_stem=live_stem or None,
|
||||
)
|
||||
handler.namer = namer
|
||||
handler.rotator = rotator
|
||||
|
||||
@@ -185,7 +185,9 @@ def _file_handler(
|
||||
)
|
||||
else:
|
||||
rotate_on_start(live_path, log_dir, compress, history_stem=history_stem)
|
||||
prune(log_dir, history_stem, backup_count)
|
||||
live_base = os.path.basename(live_path)
|
||||
live_stem = live_base[:-4] if live_base.endswith(".log") else live_base
|
||||
prune(log_dir, history_stem, backup_count, live_stem or None)
|
||||
elif rotate is not None:
|
||||
# a typo'd value would otherwise silently fall through to a non-rotating
|
||||
# FileHandler and grow forever - warn instead of degrade silently
|
||||
|
||||
Reference in New Issue
Block a user