diff --git a/README.md b/README.md index 0071b5c..938a4a7 100644 --- a/README.md +++ b/README.md @@ -258,13 +258,13 @@ setup_logging(name="run", queue=True) rotate-mode aware: the zero-retention delete only ever fires for `"size"`, matching the contract in the bullet above — `"daily"`/`"on_start"` with `backup_count=0` were always meant to roll without pruning and now do again. -- **Gzip writes are crash-safe (v0.5.1+).** `_gzip_file` now writes to a `.tmp` sibling and - atomically `os.replace`s it onto the final `.gz` path, so a crash/OOM/power-loss mid-write - can never leave a truncated `.gz` at the path retention logic trusts. Tiered retention's - plain/gz dedupe additionally verifies a `.gz` decompresses cleanly before deleting its - plain twin — a corrupt `.gz` (from before this fix, or an external cause) is never - preferred over an intact plain copy; the plain is kept and the `.gz` gets rewritten - cleanly on the next retier pass instead of being deleted. +- **Gzip is atomic compress-or-skip (v0.5.1+).** `_gzip_file` writes to a `.tmp` sibling, + verifies it decompresses cleanly, and only then `os.replace`s it onto the final `.gz` and + removes the plain source — a crash/OOM/interrupt at any point leaves the plain `.log` + intact with no `.gz` (or the `.tmp` cleaned up), so this code never produces a truncated + `.gz` at the path retention trusts. Because a corrupt `.gz` can't arise from this path, the + tiered retention dedupe drops a plain twin unconditionally when its `.gz` exists (no + runtime `_gz_intact` reconciliation — that was removed once compression became atomic). - **Setup-time warnings reach the log file (v0.6.0+).** Previously, a warning raised during `setup_logging` itself (an invalid `module_levels` entry, a handler failing to close on re-setup, an unknown `rotate` value) was emitted *before* any handler was diff --git a/src/log_setup/rotation.py b/src/log_setup/rotation.py index 3a928b1..a560632 100644 --- a/src/log_setup/rotation.py +++ b/src/log_setup/rotation.py @@ -132,7 +132,7 @@ def make_rotator( compress: bool, log_dir: Optional[str] = None, prune_stem: Optional[str] = None, backup_count: int = 0, keep_uncompressed: Optional[int] = None, keep_compressed: Optional[int] = None, - rotate_mode: Optional[str] = None, + rotate_mode: Optional[str] = None, live_stem: Optional[str] = None, ) -> Callable[[str, str], None]: """rotator: move (or gzip) the source live file to the destination rolled path @@ -162,7 +162,7 @@ def make_rotator( plain_dest = _free_dest(dest[:-3] if dest.endswith(".gz") else dest) _move(source, plain_dest) if log_dir is not None and prune_stem is not None: - retier(log_dir, prune_stem, keep_uncompressed or 0, keep_compressed or 0) + retier(log_dir, prune_stem, keep_uncompressed or 0, keep_compressed or 0, live_stem) return if compress: _gzip_file(source, dest) @@ -175,7 +175,7 @@ def make_rotator( except OSError: pass elif log_dir is not None and prune_stem is not None: - prune(log_dir, prune_stem, backup_count) + prune(log_dir, prune_stem, backup_count, live_stem) return rotator @@ -225,7 +225,8 @@ def rotate_on_start( retier(log_dir, stem, keep_uncompressed or 0, keep_compressed or 0) -def retier(log_dir: str, stem: str, keep_uncompressed: int, keep_compressed: int) -> None: +def retier(log_dir: str, stem: str, keep_uncompressed: int, keep_compressed: int, + live_stem: Optional[str] = None) -> None: """re-tier rolled files for stem: newest plain, next gzipped, rest deleted newest-first by mtime: the first `keep_uncompressed` stay uncompressed, the next @@ -248,10 +249,11 @@ def retier(log_dir: str, stem: str, keep_uncompressed: int, keep_compressed: int the same log_dir) is left alone rather than tiered/gzipped/deleted. """ stem = os.path.basename(stem) + live_stem = os.path.basename(live_stem) if live_stem else None try: names = [ name for name in os.listdir(log_dir) - if name != f"{stem}.log" and _is_rolled_name(stem, name) + if name != f"{stem}.log" and _is_rolled_name(stem, name, live_stem) ] except OSError: return @@ -297,26 +299,30 @@ def retier(log_dir: str, stem: str, keep_uncompressed: int, keep_compressed: int pass -def _rolled_name_pattern(stem: str) -> "re.Pattern": +def _rolled_name_pattern(stem: str, live_stem: Optional[str] = None) -> "re.Pattern": """compiled regex matching this lib's own rolled-file shapes for stem - two forms, both date-bearing so a foreign same-stem file (e.g. `proj.audit.log`) is + all forms are date-bearing so a foreign same-stem file (e.g. `proj.audit.log`) is never mistaken for a roll and pruned/retiered/deleted: - current uniform namer: `.[.N].log[.gz]` (make_history_namer/rotate_on_start, see attach_rolling) - legacy pre-v0.5.0 daily: `.log.[.gz]` (the stdlib TimedRotatingFileHandler shape) - matched so an upgraded deployment's existing history is still pruned/retired - instead of piling up forever + instead of piling up forever. legacy rolls were named off the LIVE file's name, not the + history stem, so when `live_stem` is given (and differs) its legacy shape is matched too """ - esc = re.escape(stem) - uniform = rf"{esc}\.\d{{4}}-\d{{2}}-\d{{2}}_\d{{2}}-\d{{2}}-\d{{2}}(?:\.\d+)?\.log(?:\.gz)?" - legacy = rf"{esc}\.log\.\d{{4}}-\d{{2}}-\d{{2}}(?:\.gz)?" - return re.compile(rf"^(?:{uniform}|{legacy})$") + stems = [stem] if live_stem is None or live_stem == stem else [stem, live_stem] + forms = [] + for s in stems: + esc = re.escape(s) + forms.append(rf"{esc}\.\d{{4}}-\d{{2}}-\d{{2}}_\d{{2}}-\d{{2}}-\d{{2}}(?:\.\d+)?\.log(?:\.gz)?") + forms.append(rf"{esc}\.log\.\d{{4}}-\d{{2}}-\d{{2}}(?:\.gz)?") + return re.compile(rf"^(?:{'|'.join(forms)})$") -def _is_rolled_name(stem: str, name: str) -> bool: - """return whether name has this lib's own rolled-log shape for stem""" - return _rolled_name_pattern(stem).match(name) is not None +def _is_rolled_name(stem: str, name: str, live_stem: Optional[str] = None) -> bool: + """return whether name has this lib's own rolled-log shape for stem (or the live stem)""" + return _rolled_name_pattern(stem, live_stem).match(name) is not None def _roll_counter(path: str) -> int: @@ -335,7 +341,7 @@ def _roll_counter(path: str) -> int: return int(tail) if tail.isdigit() else 0 -def prune(log_dir: str, stem: str, backup_count: int) -> None: +def prune(log_dir: str, stem: str, backup_count: int, live_stem: Optional[str] = None) -> None: """keep only the newest `backup_count` rolled files for a given stem in log_dir matches files beginning with `.` (e.g. run.*), sorted newest-first by mtime @@ -354,11 +360,12 @@ def prune(log_dir: str, stem: str, backup_count: int) -> None: if backup_count <= 0: return stem = os.path.basename(stem) + live_stem = os.path.basename(live_stem) if live_stem else None try: entries = [ os.path.join(log_dir, name) for name in os.listdir(log_dir) - if name != f"{stem}.log" and _is_rolled_name(stem, name) + if name != f"{stem}.log" and _is_rolled_name(stem, name, live_stem) ] except OSError: return @@ -403,9 +410,13 @@ def attach_rolling( namer = make_history_namer( os.path.basename(prune_stem or ""), log_dir, compress, plain=tiered, ) + # the live file name (stem of the handler's baseFilename, minus one .log) - so retention + # also recognizes pre-v0.5.0 legacy rolls, which were named off the LIVE name not the stem + live_base = os.path.basename(getattr(handler, "baseFilename", "") or "") + live_stem = live_base[:-4] if live_base.endswith(".log") else live_base rotator = make_rotator( compress, log_dir, prune_stem, backup_count, keep_uncompressed, keep_compressed, - rotate_mode=rotate_mode, + rotate_mode=rotate_mode, live_stem=live_stem or None, ) handler.namer = namer handler.rotator = rotator diff --git a/src/log_setup/setup.py b/src/log_setup/setup.py index 2b5e9b1..f65cbfa 100644 --- a/src/log_setup/setup.py +++ b/src/log_setup/setup.py @@ -185,7 +185,9 @@ def _file_handler( ) else: rotate_on_start(live_path, log_dir, compress, history_stem=history_stem) - prune(log_dir, history_stem, backup_count) + live_base = os.path.basename(live_path) + live_stem = live_base[:-4] if live_base.endswith(".log") else live_base + prune(log_dir, history_stem, backup_count, live_stem or None) elif rotate is not None: # a typo'd value would otherwise silently fall through to a non-rotating # FileHandler and grow forever - warn instead of degrade silently