# -*- coding: utf-8 -*-
"""
统一日志模块。
Level 约定（生产默认 INFO）：
- DEBUG：HTTP/DB 明细、轮询重试、Handler 内部步骤、参数/响应 dump
- INFO：任务生命周期、派发/完成、FTP 出站成功、Token 换票、批次任务摘要
- WARNING：可恢复异常、跳过、降级路径
- ERROR/EXCEPTION：失败、需介入
维护脚本：scripts/optimize_log_levels.py
"""
from __future__ import annotations
import gzip
import logging
import os
import shutil
import sys
from datetime import datetime
from logging.handlers import TimedRotatingFileHandler
from pathlib import Path
from typing import cast
from src.core.config import LoggingFileConfig, get_settings
_DIR_MAINTAIN_INTERVAL_SEC = 60
_SIZE_CHECK_EVERY = 256 # 每 N 条日志检查一次单文件大小，降低 I/O
_LEVEL_SPECS: tuple[tuple[str, int], ...] = (
("debug", logging.DEBUG),
("info", logging.INFO),
("warning", logging.WARNING),
("error", logging.ERROR),
("critical", logging.CRITICAL),
)
_loggers: dict[str, logging.Logger] = {}
_file_handlers: dict[str, dict[str, SizeLimitedTimedRotatingFileHandler]] = {}
_console_handler: logging.StreamHandler | None = None
_formatter: logging.Formatter | None = None
_last_dir_maintain_at: float = 0.0
class _ExactLevelFilter(logging.Filter):
"""仅放行指定 level 的日志记录。"""
def __init__(self, level: int) -> None:
super().__init__()
self._level = level
def filter(self, record: logging.LogRecord) -> bool:
return record.levelno == self._level
class SizeLimitedTimedRotatingFileHandler(TimedRotatingFileHandler):
"""按天轮转，单文件超过 maxBytes 时压缩并截断。"""
def __init__(self, *args, max_bytes: int, **kwargs):
super().__init__(*args, **kwargs)
self.maxBytes = max_bytes
self._rollover_reason = "time"
self._records_since_size_check = 0
def shouldRollover(self, record: logging.LogRecord) -> bool:
if super().shouldRollover(record):
self._rollover_reason = "time"
return True
if self.maxBytes <= 0:
return False
self._records_since_size_check += 1
if self._records_since_size_check < _SIZE_CHECK_EVERY:
return False
self._records_since_size_check = 0
if self.stream is None:
self.stream = self._open()
self.stream.seek(0, 2)
if self.stream.tell() >= self.maxBytes:
self._rollover_reason = "size"
return True
return False
def doRollover(self) -> None:
if self._rollover_reason == "size":
if self.stream:
self.stream.close()
self.stream = None
_compress_and_truncate(self.baseFilename, self.maxBytes)
if not self.delay:
self.stream = self._open()
else:
super().doRollover()
_maintain_log_dir(Path(self.baseFilename).parent, force=True)
def _resolve_file_key(name: str) -> str:
"""
统一日志文件归类键，减少零散 log 文件数量。
- handlers.* / src.handlers.* -> handlers
- src.services.* -> services
- src.utils.* -> utils
- src.apps.rpatask.* -> rpatask
- src.apps.tokenpool.* -> tokenpool
- src.apps.* -> apps
- providers.* -> providers
- 其他模块按 logger name 各自落盘
"""
if name == "handlers" or name.startswith("handlers.") or name.startswith("src.handlers."):
return "handlers"
if name == "utils" or name.startswith("src.utils."):
return "utils"
if name.startswith("src.apps."):
return "apps"
return name
def _get_formatter() -> logging.Formatter:
global _formatter
if _formatter is None:
_formatter = logging.Formatter(
fmt="%(asctime)s | %(levelname)-8s | [%(name)s] | PID=%(process)d | %(filename)s:%(lineno)d | %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
return _formatter
def _ensure_level_handlers(
file_key: str,
log_dir: Path,
formatter: logging.Formatter,
min_level: int,
log_cfg: LoggingFileConfig,
) -> dict[str, SizeLimitedTimedRotatingFileHandler]:
"""为 file_key 创建各级别独立文件 handler（仅创建 >= min_level 的级别）。"""
level_handlers = _file_handlers.setdefault(file_key, {})
created = False
max_bytes = log_cfg.max_file_size_gb * 1024 ** 3
for level_name, level_no in _LEVEL_SPECS:
if level_no < min_level or level_name in level_handlers:
continue
log_file = log_dir / f"{file_key}.{level_name}.log"
handler = SizeLimitedTimedRotatingFileHandler(
filename=log_file,
when="midnight",
interval=1,
backupCount=log_cfg.retention_days,
encoding="utf-8",
max_bytes=max_bytes,
)
handler.suffix = "%Y-%m-%d.gz"
handler.setLevel(level_no)
handler.addFilter(_ExactLevelFilter(level_no))
handler.setFormatter(formatter)
handler.rotator = _log_rotator
level_handlers[level_name] = handler
created = True
if created:
_maintain_log_dir(log_dir)
return level_handlers
def get_logger(name: str = "rpa") -> logging.Logger:
"""获取 logger，相同 name 返回同一实例。"""
if name in _loggers:
return _loggers[name]
settings = get_settings()
log_cfg = settings.logging
level = getattr(logging, log_cfg.level.upper(), logging.INFO)
logger = logging.getLogger(name)
logger.setLevel(level)
logger.propagate = False
formatter = _get_formatter()
if log_cfg.file.enabled:
file_key = _resolve_file_key(name)
log_dir = Path(log_cfg.file.path)
log_dir.mkdir(parents=True, exist_ok=True)
for handler in _ensure_level_handlers(
file_key, log_dir, formatter, level, log_cfg.file
).values():
if handler not in logger.handlers:
logger.addHandler(handler)
if log_cfg.console:
global _console_handler
if _console_handler is None:
_console_handler = logging.StreamHandler(sys.stdout)
_console_handler.setFormatter(formatter)
logger.addHandler(_console_handler)
_loggers[name] = logger
return logger
def _get_active_log_filenames() -> set[str]:
names: set[str] = set()
for level_handlers in _file_handlers.values():
for handler in level_handlers.values():
names.add(Path(handler.baseFilename).name)
return names
def _is_active_log_file(path: Path) -> bool:
return path.name in _get_active_log_filenames()
def _is_log_backup(path: Path) -> bool:
if path.name in _get_active_log_filenames():
return False
return path.suffix == ".gz" or path.suffix == ".log" and ".log." in path.name
def _log_level_deletion_rank(path: Path) -> int:
"""删除优先级：数值越小越先删（debug 最先）。"""
lower = path.name.lower()
if ".debug." in lower or lower.endswith(".debug.log"):
return 0
return 1
def _should_run_dir_maintenance(force: bool) -> bool:
global _last_dir_maintain_at
now = datetime.now().timestamp()
if not force and now - _last_dir_maintain_at < _DIR_MAINTAIN_INTERVAL_SEC:
return False
_last_dir_maintain_at = now
return True
def _maintain_log_dir(log_dir: Path, *, force: bool = False) -> None:
"""压缩非活跃日志，并按保留天数与目录总大小上限清理。"""
if not _should_run_dir_maintenance(force):
return
log_cfg = get_settings().logging.file
_compress_inactive_logs(log_dir)
_cleanup_expired_logs(log_dir, log_cfg.retention_days)
_enforce_log_dir_size_limit(log_dir, log_cfg.max_dir_size_gb * 1024 ** 3)
def _compress_inactive_logs(log_dir: Path) -> None:
"""除正在写入的日志外，将目录内未压缩的日志备份 gzip 压缩。"""
active = _get_active_log_filenames()
try:
for path in log_dir.iterdir():
if not path.is_file() or path.name in active:
continue
if path.suffix == ".gz":
continue
if path.suffix != ".log" and ".log." not in path.name:
continue
dest = path.with_name(f"{path.name}.gz")
counter = 0
while dest.exists():
counter += 1
dest = path.with_name(f"{path.name}_{counter}.gz")
if _gzip_to_file(str(path), str(dest)):
path.unlink(missing_ok=True)
except Exception:
pass
def _cleanup_expired_logs(log_dir: Path, retention_days: int) -> None:
"""删除超过 retention_days 的全部日志备份（按 mtime）。"""
if retention_days <= 0:
return
try:
cutoff = datetime.now().timestamp() - retention_days * 86400
for path in log_dir.iterdir():
if not path.is_file() or not _is_log_backup(path):
continue
if path.stat().st_mtime < cutoff:
path.unlink(missing_ok=True)
except Exception:
pass
def _enforce_log_dir_size_limit(log_dir: Path, max_bytes: int) -> None:
"""目录总大小超限时删除备份；同条件下优先删 debug，再按时间从旧到新。"""
if max_bytes <= 0:
return
try:
files = [p for p in log_dir.iterdir() if p.is_file()]
if not files:
return
file_stats = [(p, p.stat()) for p in files]
total = sum(st.st_size for _, st in file_stats)
if total <= max_bytes:
return
file_stats.sort(key=lambda item: (
_is_active_log_file(item[0]),
_log_level_deletion_rank(item[0]),
item[1].st_mtime,
))
for path, st in file_stats:
if total <= max_bytes:
break
try:
path.unlink(missing_ok=True)
total -= st.st_size
except Exception:
pass
except Exception:
pass
def _gzip_to_file(source: str, dest: str) -> bool:
if not os.path.exists(source) or os.path.getsize(source) == 0:
return False
if os.path.exists(dest):
return False
with open(source, "rb") as f_in:
with gzip.open(dest, "wb", compresslevel=6) as f_out:
shutil.copyfileobj(f_in, f_out)
return True
def _truncate_file(source: str) -> None:
with open(source, "w", encoding="utf-8"):
pass
def _log_rotator(source: str, dest: str) -> None:
"""按天轮转：gzip 压缩备份并截断原文件（全平台 copy+truncate）。"""
try:
_gzip_to_file(source, dest)
_truncate_file(source)
except Exception:
pass
def _compress_and_truncate(source: str, max_bytes: int) -> None:
"""单文件超过大小上限时：gzip 压缩当前内容，再截断原文件。"""
try:
if not os.path.exists(source):
return
size = os.path.getsize(source)
if size == 0 or size < max_bytes:
return
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
dest = f"{source}.{timestamp}.gz"
counter = 0
while os.path.exists(dest):
counter += 1
dest = f"{source}.{timestamp}_{counter}.gz"
_gzip_to_file(source, dest)
_truncate_file(source)
except Exception:
pass
# 预定义 logger：模块级显式导出供 IDE 跳转；首次写日志时才 init handler。
class _LazyLogger:
__slots__ = ("_name", "_logger")
def __init__(self, name: str) -> None:
self._name = name
self._logger: logging.Logger | None = None
def _resolve(self) -> logging.Logger:
if self._logger is None:
self._logger = get_logger(self._name)
return self._logger
def __getattr__(self, name: str):
return getattr(self._resolve(), name)
def __repr__(self) -> str:
if self._logger is not None:
return repr(self._logger)
return f"<LazyLogger {self._name!r}>"
def _lazy_logger(name: str) -> logging.Logger:
return cast(logging.Logger, _LazyLogger(name))
db_logger = _lazy_logger("db")
web_logger = _lazy_logger("web")
__all__ = [
"get_logger",
"db_logger",
"web_logger",
]
浙公网安备 33010602011771号