From 64b2c0e7a8aca87145cfa6f9b2d4d5fc4091ed16 Mon Sep 17 00:00:00 2001 From: Jeong Yunwon Date: Tue, 19 Jul 2022 20:37:40 +0900 Subject: [PATCH] Update logging from CPython 3.10.5 --- Lib/logging/__init__.py | 321 +++++++++++++++++++++++++++++----------- Lib/logging/config.py | 29 ++-- Lib/logging/handlers.py | 173 +++++++++++++++------- 3 files changed, 375 insertions(+), 148 deletions(-) diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 1a652309a..19bd2bc20 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2001-2017 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -18,14 +18,16 @@ Logging package for Python. Based on PEP 282 and comments thereto in comp.lang.python. -Copyright (C) 2001-2017 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ -import sys, os, time, io, traceback, warnings, weakref, collections.abc +import sys, os, time, io, re, traceback, warnings, weakref, collections.abc from string import Template +from string import Formatter as StrFormatter + __all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR', 'FATAL', 'FileHandler', 'Filter', 'Formatter', 'Handler', 'INFO', @@ -63,7 +65,7 @@ raiseExceptions = True # # If you don't want threading information in the log, set this to zero # -logThreads = False +logThreads = True # # If you don't want multiprocessing information in the log, set this to zero @@ -116,7 +118,7 @@ _nameToLevel = { def getLevelName(level): """ - Return the textual representation of logging level 'level'. + Return the textual or numeric representation of logging level 'level'. If the level is one of the predefined levels (CRITICAL, ERROR, WARNING, INFO, DEBUG) then you get the corresponding string. If you have @@ -126,7 +128,11 @@ def getLevelName(level): If a numeric value corresponding to one of the defined levels is passed in, the corresponding string representation is returned. - Otherwise, the string "Level %s" % level is returned. + If a string representation of the level is passed in, the corresponding + numeric value is returned. + + If no matching numeric or string value is passed in, the string + 'Level %s' % level is returned. """ # See Issues #22386, #27937 and #29220 for why it's this way result = _levelToName.get(level) @@ -192,7 +198,8 @@ def _checkLevel(level): raise ValueError("Unknown level: %r" % level) rv = _nameToLevel[level] else: - raise TypeError("Level not an integer or a valid string: %r" % level) + raise TypeError("Level not an integer or a valid string: %r" + % (level,)) return rv #--------------------------------------------------------------------------- @@ -229,50 +236,33 @@ def _releaseLock(): # Prevent a held logging lock from blocking a child from logging. if not hasattr(os, 'register_at_fork'): # Windows and friends. - def _register_at_fork_acquire_release(instance): + def _register_at_fork_reinit_lock(instance): pass # no-op when os.register_at_fork does not exist. -else: # The os.register_at_fork API exists +else: + # A collection of instances with a _at_fork_reinit method (logging.Handler) + # to be called in the child after forking. The weakref avoids us keeping + # discarded Handler instances alive. + _at_fork_reinit_lock_weakset = weakref.WeakSet() + + def _register_at_fork_reinit_lock(instance): + _acquireLock() + try: + _at_fork_reinit_lock_weakset.add(instance) + finally: + _releaseLock() + + def _after_at_fork_child_reinit_locks(): + for handler in _at_fork_reinit_lock_weakset: + handler._at_fork_reinit() + + # _acquireLock() was called in the parent before forking. + # The lock is reinitialized to unlocked state. + _lock._at_fork_reinit() + os.register_at_fork(before=_acquireLock, - after_in_child=_releaseLock, + after_in_child=_after_at_fork_child_reinit_locks, after_in_parent=_releaseLock) - # A collection of instances with acquire and release methods (logging.Handler) - # to be called before and after fork. The weakref avoids us keeping discarded - # Handler instances alive forever in case an odd program creates and destroys - # many over its lifetime. - _at_fork_acquire_release_weakset = weakref.WeakSet() - - - def _register_at_fork_acquire_release(instance): - # We put the instance itself in a single WeakSet as we MUST have only - # one atomic weak ref. used by both before and after atfork calls to - # guarantee matched pairs of acquire and release calls. - _at_fork_acquire_release_weakset.add(instance) - - - def _at_fork_weak_calls(method_name): - for instance in _at_fork_acquire_release_weakset: - method = getattr(instance, method_name) - try: - method() - except Exception as err: - # Similar to what PyErr_WriteUnraisable does. - print("Ignoring exception from logging atfork", instance, - method_name, "method:", err, file=sys.stderr) - - - def _before_at_fork_weak_calls(): - _at_fork_weak_calls('acquire') - - - def _after_at_fork_weak_calls(): - _at_fork_weak_calls('release') - - - os.register_at_fork(before=_before_at_fork_weak_calls, - after_in_child=_after_at_fork_weak_calls, - after_in_parent=_after_at_fork_weak_calls) - #--------------------------------------------------------------------------- # The logging record @@ -362,12 +352,10 @@ class LogRecord(object): else: self.process = None - def __str__(self): + def __repr__(self): return ''%(self.name, self.levelno, self.pathname, self.lineno, self.msg) - __repr__ = __str__ - def getMessage(self): """ Return the message for this LogRecord. @@ -413,32 +401,79 @@ def makeLogRecord(dict): rv.__dict__.update(dict) return rv + #--------------------------------------------------------------------------- # Formatter classes and functions #--------------------------------------------------------------------------- +_str_formatter = StrFormatter() +del StrFormatter + class PercentStyle(object): default_format = '%(message)s' asctime_format = '%(asctime)s' asctime_search = '%(asctime)' + validation_pattern = re.compile(r'%\(\w+\)[#0+ -]*(\*|\d+)?(\.(\*|\d+))?[diouxefgcrsa%]', re.I) - def __init__(self, fmt): + def __init__(self, fmt, *, defaults=None): self._fmt = fmt or self.default_format + self._defaults = defaults def usesTime(self): return self._fmt.find(self.asctime_search) >= 0 + def validate(self): + """Validate the input format, ensure it matches the correct style""" + if not self.validation_pattern.search(self._fmt): + raise ValueError("Invalid format '%s' for '%s' style" % (self._fmt, self.default_format[0])) + + def _format(self, record): + if defaults := self._defaults: + values = defaults | record.__dict__ + else: + values = record.__dict__ + return self._fmt % values + def format(self, record): - return self._fmt % record.__dict__ + try: + return self._format(record) + except KeyError as e: + raise ValueError('Formatting field not found in record: %s' % e) + class StrFormatStyle(PercentStyle): default_format = '{message}' asctime_format = '{asctime}' asctime_search = '{asctime' - def format(self, record): - return self._fmt.format(**record.__dict__) + fmt_spec = re.compile(r'^(.?[<>=^])?[+ -]?#?0?(\d+|{\w+})?[,_]?(\.(\d+|{\w+}))?[bcdefgnosx%]?$', re.I) + field_spec = re.compile(r'^(\d+|\w+)(\.\w+|\[[^]]+\])*$') + + def _format(self, record): + if defaults := self._defaults: + values = defaults | record.__dict__ + else: + values = record.__dict__ + return self._fmt.format(**values) + + def validate(self): + """Validate the input format, ensure it is the correct string formatting style""" + fields = set() + try: + for _, fieldname, spec, conversion in _str_formatter.parse(self._fmt): + if fieldname: + if not self.field_spec.match(fieldname): + raise ValueError('invalid field name/expression: %r' % fieldname) + fields.add(fieldname) + if conversion and conversion not in 'rsa': + raise ValueError('invalid conversion: %r' % conversion) + if spec and not self.fmt_spec.match(spec): + raise ValueError('bad specifier: %r' % spec) + except ValueError as e: + raise ValueError('invalid format: %s' % e) + if not fields: + raise ValueError('invalid format: no fields') class StringTemplateStyle(PercentStyle): @@ -446,16 +481,35 @@ class StringTemplateStyle(PercentStyle): asctime_format = '${asctime}' asctime_search = '${asctime}' - def __init__(self, fmt): - self._fmt = fmt or self.default_format + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self._tpl = Template(self._fmt) def usesTime(self): fmt = self._fmt return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_format) >= 0 - def format(self, record): - return self._tpl.substitute(**record.__dict__) + def validate(self): + pattern = Template.pattern + fields = set() + for m in pattern.finditer(self._fmt): + d = m.groupdict() + if d['named']: + fields.add(d['named']) + elif d['braced']: + fields.add(d['braced']) + elif m.group(0) == '$': + raise ValueError('invalid format: bare \'$\' not allowed') + if not fields: + raise ValueError('invalid format: no fields') + + def _format(self, record): + if defaults := self._defaults: + values = defaults | record.__dict__ + else: + values = record.__dict__ + return self._tpl.substitute(**values) + BASIC_FORMAT = "%(levelname)s:%(name)s:%(message)s" @@ -473,7 +527,7 @@ class Formatter(object): responsible for converting a LogRecord to (usually) a string which can be interpreted by either a human or an external system. The base Formatter allows a formatting string to be specified. If none is supplied, the - the style-dependent default value, "%(message)s", "{message}", or + style-dependent default value, "%(message)s", "{message}", or "${message}", is used. The Formatter can be initialized with a format string which makes use of @@ -510,7 +564,8 @@ class Formatter(object): converter = time.localtime - def __init__(self, fmt=None, datefmt=None, style='%'): + def __init__(self, fmt=None, datefmt=None, style='%', validate=True, *, + defaults=None): """ Initialize the formatter with specified format strings. @@ -529,7 +584,10 @@ class Formatter(object): if style not in _STYLES: raise ValueError('Style must be one of: %s' % ','.join( _STYLES.keys())) - self._style = _STYLES[style][0](fmt) + self._style = _STYLES[style][0](fmt, defaults=defaults) + if validate: + self._style.validate() + self._fmt = self._style._fmt self.datefmt = datefmt @@ -558,8 +616,9 @@ class Formatter(object): if datefmt: s = time.strftime(datefmt, ct) else: - t = time.strftime(self.default_time_format, ct) - s = self.default_msec_format % (t, record.msecs) + s = time.strftime(self.default_time_format, ct) + if self.default_msec_format: + s = self.default_msec_format % (s, record.msecs) return s def formatException(self, ei): @@ -708,8 +767,8 @@ class Filter(object): """ Determine if the specified record is to be logged. - Is the specified record to be logged? Returns 0 for no, nonzero for - yes. If deemed appropriate, the record may be modified in-place. + Returns True if the record should be logged, or False otherwise. + If deemed appropriate, the record may be modified in-place. """ if self.nlen == 0: return True @@ -819,6 +878,7 @@ class Handler(Filterer): self._name = None self.level = _checkLevel(level) self.formatter = None + self._closed = False # Add the handler to the global _handlerList (for cleanup on shutdown) _addHandlerRef(self) self.createLock() @@ -844,7 +904,10 @@ class Handler(Filterer): Acquire a thread lock for serializing access to the underlying I/O. """ self.lock = threading.RLock() - _register_at_fork_acquire_release(self) + _register_at_fork_reinit_lock(self) + + def _at_fork_reinit(self): + self.lock._at_fork_reinit() def acquire(self): """ @@ -934,6 +997,7 @@ class Handler(Filterer): #get the module data lock, as we're updating a shared structure. _acquireLock() try: #unlikely to raise an exception, but you never know... + self._closed = True if self._name and self._name in _handlers: del _handlers[self._name] finally: @@ -974,6 +1038,8 @@ class Handler(Filterer): sys.stderr.write('Message: %r\n' 'Arguments: %s\n' % (record.msg, record.args)) + except RecursionError: # See issue 36272 + raise except Exception: sys.stderr.write('Unable to print the message and arguments' ' - possible formatting error.\nUse the' @@ -1036,6 +1102,8 @@ class StreamHandler(Handler): # issue 35046: merged two stream.writes into one. stream.write(msg + self.terminator) self.flush() + except RecursionError: # See issue 36272 + raise except Exception: self.handleError(record) @@ -1062,6 +1130,8 @@ class StreamHandler(Handler): def __repr__(self): level = getLevelName(self.level) name = getattr(self.stream, 'name', '') + # bpo-36015: name can be an int + name = str(name) if name: name += ' ' return '<%s %s(%s)>' % (self.__class__.__name__, name, level) @@ -1071,7 +1141,7 @@ class FileHandler(StreamHandler): """ A handler class which writes formatted logging records to disk files. """ - def __init__(self, filename, mode='a', encoding=None, delay=False): + def __init__(self, filename, mode='a', encoding=None, delay=False, errors=None): """ Open the specified file and use it as the stream for logging. """ @@ -1082,7 +1152,14 @@ class FileHandler(StreamHandler): self.baseFilename = os.path.abspath(filename) self.mode = mode self.encoding = encoding + if "b" not in mode: + self.encoding = io.text_encoding(encoding) + self.errors = errors self.delay = delay + # bpo-26789: FileHandler keeps a reference to the builtin open() + # function to be able to open or reopen the file during Python + # finalization. + self._builtin_open = open if delay: #We don't open the stream, but we still need to call the #Handler constructor to set level, formatter, lock etc. @@ -1109,6 +1186,8 @@ class FileHandler(StreamHandler): finally: # Issue #19523: call unconditionally to # prevent a handler leak when delay is set + # Also see Issue #42378: we also rely on + # self._closed being set to True there StreamHandler.close(self) finally: self.release() @@ -1118,7 +1197,9 @@ class FileHandler(StreamHandler): Open the current base file with the (original) mode and encoding. Return the resulting stream. """ - return open(self.baseFilename, self.mode, encoding=self.encoding) + open_func = self._builtin_open + return open_func(self.baseFilename, self.mode, + encoding=self.encoding, errors=self.errors) def emit(self, record): """ @@ -1126,10 +1207,15 @@ class FileHandler(StreamHandler): If the stream was not opened because 'delay' was specified in the constructor, open it before calling the superclass's emit. + + If stream is not open, current mode is 'w' and `_closed=True`, record + will not be emitted (see Issue #42378). """ if self.stream is None: - self.stream = self._open() - StreamHandler.emit(self, record) + if self.mode != 'w' or not self._closed: + self.stream = self._open() + if self.stream: + StreamHandler.emit(self, record) def __repr__(self): level = getLevelName(self.level) @@ -1218,6 +1304,14 @@ class Manager(object): self.loggerClass = None self.logRecordFactory = None + @property + def disable(self): + return self._disable + + @disable.setter + def disable(self, value): + self._disable = _checkLevel(value) + def getLogger(self, name): """ Get a logger with the specified name (channel name), creating it @@ -1429,7 +1523,11 @@ class Logger(Filterer): if self.isEnabledFor(CRITICAL): self._log(CRITICAL, msg, args, **kwargs) - fatal = critical + def fatal(self, msg, *args, **kwargs): + """ + Don't use this method, use critical() instead. + """ + self.critical(msg, *args, **kwargs) def log(self, level, msg, *args, **kwargs): """ @@ -1448,7 +1546,7 @@ class Logger(Filterer): if self.isEnabledFor(level): self._log(level, msg, args, **kwargs) - def findCaller(self, stack_info=False): + def findCaller(self, stack_info=False, stacklevel=1): """ Find the stack frame of the caller so that we can note the source file name, line number and function name. @@ -1458,6 +1556,12 @@ class Logger(Filterer): #IronPython isn't run with -X:Frames. if f is not None: f = f.f_back + orig_f = f + while f and stacklevel > 1: + f = f.f_back + stacklevel -= 1 + if not f: + f = orig_f rv = "(unknown file)", 0, "(unknown function)", None while hasattr(f, "f_code"): co = f.f_code @@ -1493,7 +1597,8 @@ class Logger(Filterer): rv.__dict__[key] = extra[key] return rv - def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False): + def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, + stacklevel=1): """ Low-level logging routine which creates a LogRecord and then calls all the handlers of this logger to handle the record. @@ -1504,7 +1609,7 @@ class Logger(Filterer): #exception on some versions of IronPython. We trap it here so that #IronPython can use logging. try: - fn, lno, func, sinfo = self.findCaller(stack_info) + fn, lno, func, sinfo = self.findCaller(stack_info, stacklevel) except ValueError: # pragma: no cover fn, lno, func = "(unknown file)", 0, "(unknown function)" else: # pragma: no cover @@ -1620,16 +1725,22 @@ class Logger(Filterer): """ Is this logger enabled for level 'level'? """ + if self.disabled: + return False + try: return self._cache[level] except KeyError: _acquireLock() - if self.manager.disable >= level: - is_enabled = self._cache[level] = False - else: - is_enabled = self._cache[level] = level >= self.getEffectiveLevel() - _releaseLock() - + try: + if self.manager.disable >= level: + is_enabled = self._cache[level] = False + else: + is_enabled = self._cache[level] = ( + level >= self.getEffectiveLevel() + ) + finally: + _releaseLock() return is_enabled def getChild(self, suffix): @@ -1687,7 +1798,7 @@ class LoggerAdapter(object): information in logging output. """ - def __init__(self, logger, extra): + def __init__(self, logger, extra=None): """ Initialize the adapter with a logger and a dict-like object which provides contextual information. This constructor signature allows @@ -1834,7 +1945,8 @@ def basicConfig(**kwargs): Do basic configuration for the logging system. This function does nothing if the root logger already has handlers - configured. It is a convenience method intended for use by simple scripts + configured, unless the keyword argument *force* is set to ``True``. + It is a convenience method intended for use by simple scripts to do one-shot configuration of the logging package. The default behaviour is to create a StreamHandler which writes to @@ -1862,6 +1974,17 @@ def basicConfig(**kwargs): handlers, which will be added to the root handler. Any handler in the list which does not have a formatter assigned will be assigned the formatter created in this function. + force If this keyword is specified as true, any existing handlers + attached to the root logger are removed and closed, before + carrying out the configuration as specified by the other + arguments. + encoding If specified together with a filename, this encoding is passed to + the created FileHandler, causing it to be used when the file is + opened. + errors If specified together with a filename, this value is passed to the + created FileHandler, causing it to be used when the file is + opened in text mode. If not specified, the default value is + `backslashreplace`. Note that you could specify a stream created using open(filename, mode) rather than passing the filename and mode in. However, it should be @@ -1878,11 +2001,24 @@ def basicConfig(**kwargs): ``filename``/``filemode``, or ``filename``/``filemode`` specified together with ``stream``, or ``handlers`` specified together with ``stream``. + + .. versionchanged:: 3.8 + Added the ``force`` parameter. + + .. versionchanged:: 3.9 + Added the ``encoding`` and ``errors`` parameters. """ # Add thread safety in case someone mistakenly calls # basicConfig() from multiple threads _acquireLock() try: + force = kwargs.pop('force', False) + encoding = kwargs.pop('encoding', None) + errors = kwargs.pop('errors', 'backslashreplace') + if force: + for h in root.handlers[:]: + root.removeHandler(h) + h.close() if len(root.handlers) == 0: handlers = kwargs.pop("handlers", None) if handlers is None: @@ -1897,7 +2033,12 @@ def basicConfig(**kwargs): filename = kwargs.pop("filename", None) mode = kwargs.pop("filemode", 'a') if filename: - h = FileHandler(filename, mode) + if 'b' in mode: + errors = None + else: + encoding = io.text_encoding(encoding) + h = FileHandler(filename, mode, + encoding=encoding, errors=errors) else: stream = kwargs.pop("stream", None) h = StreamHandler(stream) @@ -1933,10 +2074,9 @@ def getLogger(name=None): If no name is specified, return the root logger. """ - if name: - return Logger.manager.getLogger(name) - else: + if not name or isinstance(name, str) and name == root.name: return root + return Logger.manager.getLogger(name) def critical(msg, *args, **kwargs): """ @@ -1948,7 +2088,11 @@ def critical(msg, *args, **kwargs): basicConfig() root.critical(msg, *args, **kwargs) -fatal = critical +def fatal(msg, *args, **kwargs): + """ + Don't use this function, use critical() instead. + """ + critical(msg, *args, **kwargs) def error(msg, *args, **kwargs): """ @@ -2075,6 +2219,9 @@ class NullHandler(Handler): def createLock(self): self.lock = None + def _at_fork_reinit(self): + pass + # Warnings integration _warnings_showwarning = None diff --git a/Lib/logging/config.py b/Lib/logging/config.py index fa1a398ae..3bc63b786 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -1,4 +1,4 @@ -# Copyright 2001-2016 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -19,7 +19,7 @@ Configuration functions for the logging package for Python. The core package is based on PEP 282 and comments thereto in comp.lang.python, and influenced by Apache's log4j system. -Copyright (C) 2001-2016 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ @@ -48,7 +48,7 @@ RESET_ERROR = errno.ECONNRESET # _listener holds the server object doing the listening _listener = None -def fileConfig(fname, defaults=None, disable_existing_loggers=True): +def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=None): """ Read the logging configuration from a ConfigParser-format file. @@ -66,7 +66,8 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True): if hasattr(fname, 'readline'): cp.read_file(fname) else: - cp.read(fname) + encoding = io.text_encoding(encoding) + cp.read(fname, encoding=encoding) formatters = _create_formatters(cp) @@ -143,6 +144,7 @@ def _install_handlers(cp, formatters): kwargs = section.get("kwargs", '{}') kwargs = eval(kwargs, vars(logging)) h = klass(*args, **kwargs) + h.name = hand if "level" in section: level = section["level"] h.setLevel(level) @@ -173,9 +175,10 @@ def _handle_existing_loggers(existing, child_loggers, disable_existing): for log in existing: logger = root.manager.loggerDict[log] if log in child_loggers: - logger.level = logging.NOTSET - logger.handlers = [] - logger.propagate = True + if not isinstance(logger, logging.PlaceHolder): + logger.setLevel(logging.NOTSET) + logger.handlers = [] + logger.propagate = True else: logger.disabled = disable_existing @@ -446,7 +449,7 @@ class BaseConfigurator(object): value = ConvertingList(value) value.configurator = self elif not isinstance(value, ConvertingTuple) and\ - isinstance(value, tuple): + isinstance(value, tuple) and not hasattr(value, '_fields'): value = ConvertingTuple(value) value.configurator = self elif isinstance(value, str): # str for py3k @@ -666,11 +669,19 @@ class DictConfigurator(BaseConfigurator): dfmt = config.get('datefmt', None) style = config.get('style', '%') cname = config.get('class', None) + if not cname: c = logging.Formatter else: c = _resolve(cname) - result = c(fmt, dfmt, style) + + # A TypeError would be raised if "validate" key is passed in with a formatter callable + # that does not accept "validate" as a parameter + if 'validate' in config: # if user hasn't mentioned it, the default will be fine + result = c(fmt, dfmt, style, config['validate']) + else: + result = c(fmt, dfmt, style) + return result def configure_filter(self, config): diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index e213e438c..61a39958c 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -1,4 +1,4 @@ -# Copyright 2001-2016 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2021 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -18,15 +18,16 @@ Additional handlers for the logging package for Python. The core package is based on PEP 282 and comments thereto in comp.lang.python. -Copyright (C) 2001-2016 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2021 Vinay Sajip. All Rights Reserved. To use, simply 'import logging.handlers' and log away! """ -import logging, socket, os, pickle, struct, time, re +import io, logging, socket, os, pickle, struct, time, re from stat import ST_DEV, ST_INO, ST_MTIME import queue import threading +import copy # # Some constants... @@ -47,15 +48,19 @@ class BaseRotatingHandler(logging.FileHandler): Not meant to be instantiated directly. Instead, use RotatingFileHandler or TimedRotatingFileHandler. """ - def __init__(self, filename, mode, encoding=None, delay=False): + namer = None + rotator = None + + def __init__(self, filename, mode, encoding=None, delay=False, errors=None): """ Use the specified filename for streamed logging """ - logging.FileHandler.__init__(self, filename, mode, encoding, delay) + logging.FileHandler.__init__(self, filename, mode=mode, + encoding=encoding, delay=delay, + errors=errors) self.mode = mode self.encoding = encoding - self.namer = None - self.rotator = None + self.errors = errors def emit(self, record): """ @@ -116,7 +121,8 @@ class RotatingFileHandler(BaseRotatingHandler): Handler for logging to a set of files, which switches from one file to the next when the current file reaches a certain size. """ - def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=False): + def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, + encoding=None, delay=False, errors=None): """ Open the specified file and use it as the stream for logging. @@ -144,7 +150,10 @@ class RotatingFileHandler(BaseRotatingHandler): # on each run. if maxBytes > 0: mode = 'a' - BaseRotatingHandler.__init__(self, filename, mode, encoding, delay) + if "b" not in mode: + encoding = io.text_encoding(encoding) + BaseRotatingHandler.__init__(self, filename, mode, encoding=encoding, + delay=delay, errors=errors) self.maxBytes = maxBytes self.backupCount = backupCount @@ -178,14 +187,17 @@ class RotatingFileHandler(BaseRotatingHandler): Basically, see if the supplied record would cause the file to exceed the size limit we have. """ + # See bpo-45401: Never rollover anything other than regular files + if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): + return False if self.stream is None: # delay was set... self.stream = self._open() if self.maxBytes > 0: # are we rolling over? msg = "%s\n" % self.format(record) self.stream.seek(0, 2) #due to non-posix-compliant Windows feature if self.stream.tell() + len(msg) >= self.maxBytes: - return 1 - return 0 + return True + return False class TimedRotatingFileHandler(BaseRotatingHandler): """ @@ -195,8 +207,12 @@ class TimedRotatingFileHandler(BaseRotatingHandler): If backupCount is > 0, when rollover is done, no more than backupCount files are kept - the oldest ones are deleted. """ - def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None): - BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay) + def __init__(self, filename, when='h', interval=1, backupCount=0, + encoding=None, delay=False, utc=False, atTime=None, + errors=None): + encoding = io.text_encoding(encoding) + BaseRotatingHandler.__init__(self, filename, 'a', encoding=encoding, + delay=delay, errors=errors) self.when = when.upper() self.backupCount = backupCount self.utc = utc @@ -332,10 +348,13 @@ class TimedRotatingFileHandler(BaseRotatingHandler): record is not used, as we are just comparing times, but it is needed so the method signatures are the same """ + # See bpo-45401: Never rollover anything other than regular files + if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): + return False t = int(time.time()) if t >= self.rolloverAt: - return 1 - return 0 + return True + return False def getFilesToDelete(self): """ @@ -346,13 +365,32 @@ class TimedRotatingFileHandler(BaseRotatingHandler): dirName, baseName = os.path.split(self.baseFilename) fileNames = os.listdir(dirName) result = [] - prefix = baseName + "." + # See bpo-44753: Don't use the extension when computing the prefix. + n, e = os.path.splitext(baseName) + prefix = n + '.' plen = len(prefix) for fileName in fileNames: + if self.namer is None: + # Our files will always start with baseName + if not fileName.startswith(baseName): + continue + else: + # Our files could be just about anything after custom naming, but + # likely candidates are of the form + # foo.log.DATETIME_SUFFIX or foo.DATETIME_SUFFIX.log + if (not fileName.startswith(baseName) and fileName.endswith(e) and + len(fileName) > (plen + 1) and not fileName[plen+1].isdigit()): + continue + if fileName[:plen] == prefix: suffix = fileName[plen:] - if self.extMatch.match(suffix): - result.append(os.path.join(dirName, fileName)) + # See bpo-45628: The date/time suffix could be anywhere in the + # filename + parts = suffix.split('.') + for part in parts: + if self.extMatch.match(part): + result.append(os.path.join(dirName, fileName)) + break if len(result) < self.backupCount: result = [] else: @@ -430,8 +468,13 @@ class WatchedFileHandler(logging.FileHandler): This handler is based on a suggestion and patch by Chad J. Schroeder. """ - def __init__(self, filename, mode='a', encoding=None, delay=False): - logging.FileHandler.__init__(self, filename, mode, encoding, delay) + def __init__(self, filename, mode='a', encoding=None, delay=False, + errors=None): + if "b" not in mode: + encoding = io.text_encoding(encoding) + logging.FileHandler.__init__(self, filename, mode=mode, + encoding=encoding, delay=delay, + errors=errors) self.dev, self.ino = -1, -1 self._statstream() @@ -729,6 +772,10 @@ class SysLogHandler(logging.Handler): LOG_CRON = 9 # clock daemon LOG_AUTHPRIV = 10 # security/authorization messages (private) LOG_FTP = 11 # FTP daemon + LOG_NTP = 12 # NTP subsystem + LOG_SECURITY = 13 # Log audit + LOG_CONSOLE = 14 # Log alert + LOG_SOLCRON = 15 # Scheduling daemon (Solaris) # other codes through 15 reserved for system use LOG_LOCAL0 = 16 # reserved for local use @@ -756,27 +803,30 @@ class SysLogHandler(logging.Handler): } facility_names = { - "auth": LOG_AUTH, - "authpriv": LOG_AUTHPRIV, - "cron": LOG_CRON, - "daemon": LOG_DAEMON, - "ftp": LOG_FTP, - "kern": LOG_KERN, - "lpr": LOG_LPR, - "mail": LOG_MAIL, - "news": LOG_NEWS, - "security": LOG_AUTH, # DEPRECATED - "syslog": LOG_SYSLOG, - "user": LOG_USER, - "uucp": LOG_UUCP, - "local0": LOG_LOCAL0, - "local1": LOG_LOCAL1, - "local2": LOG_LOCAL2, - "local3": LOG_LOCAL3, - "local4": LOG_LOCAL4, - "local5": LOG_LOCAL5, - "local6": LOG_LOCAL6, - "local7": LOG_LOCAL7, + "auth": LOG_AUTH, + "authpriv": LOG_AUTHPRIV, + "console": LOG_CONSOLE, + "cron": LOG_CRON, + "daemon": LOG_DAEMON, + "ftp": LOG_FTP, + "kern": LOG_KERN, + "lpr": LOG_LPR, + "mail": LOG_MAIL, + "news": LOG_NEWS, + "ntp": LOG_NTP, + "security": LOG_SECURITY, + "solaris-cron": LOG_SOLCRON, + "syslog": LOG_SYSLOG, + "user": LOG_USER, + "uucp": LOG_UUCP, + "local0": LOG_LOCAL0, + "local1": LOG_LOCAL1, + "local2": LOG_LOCAL2, + "local3": LOG_LOCAL3, + "local4": LOG_LOCAL4, + "local5": LOG_LOCAL5, + "local6": LOG_LOCAL6, + "local7": LOG_LOCAL7, } #The map below appears to be trivially lowercasing the key. However, @@ -1122,7 +1172,7 @@ class NTEventLogHandler(logging.Handler): class HTTPHandler(logging.Handler): """ - A class which sends records to a Web server, using either GET or + A class which sends records to a web server, using either GET or POST semantics. """ def __init__(self, host, url, method="GET", secure=False, credentials=None, @@ -1153,19 +1203,30 @@ class HTTPHandler(logging.Handler): """ return record.__dict__ + def getConnection(self, host, secure): + """ + get a HTTP[S]Connection. + + Override when a custom connection is required, for example if + there is a proxy. + """ + import http.client + if secure: + connection = http.client.HTTPSConnection(host, context=self.context) + else: + connection = http.client.HTTPConnection(host) + return connection + def emit(self, record): """ Emit a record. - Send the record to the Web server as a percent-encoded dictionary + Send the record to the web server as a percent-encoded dictionary """ try: - import http.client, urllib.parse + import urllib.parse host = self.host - if self.secure: - h = http.client.HTTPSConnection(host, context=self.context) - else: - h = http.client.HTTPConnection(host) + h = self.getConnection(host, self.secure) url = self.url data = urllib.parse.urlencode(self.mapLogRecord(record)) if self.method == "GET": @@ -1241,7 +1302,7 @@ class BufferingHandler(logging.Handler): """ self.acquire() try: - self.buffer = [] + self.buffer.clear() finally: self.release() @@ -1293,7 +1354,11 @@ class MemoryHandler(BufferingHandler): """ Set the target handler for this handler. """ - self.target = target + self.acquire() + try: + self.target = target + finally: + self.release() def flush(self): """ @@ -1308,7 +1373,7 @@ class MemoryHandler(BufferingHandler): if self.target: for record in self.buffer: self.target.handle(record) - self.buffer = [] + self.buffer.clear() finally: self.release() @@ -1377,6 +1442,8 @@ class QueueHandler(logging.Handler): # exc_info and exc_text attributes, as they are no longer # needed and, if not None, will typically not be pickleable. msg = self.format(record) + # bpo-35726: make copy of record to avoid affecting other handlers in the chain. + record = copy.copy(record) record.message = msg record.msg = msg record.args = None @@ -1434,7 +1501,7 @@ class QueueListener(object): t.daemon = True t.start() - def prepare(self , record): + def prepare(self, record): """ Prepare a record for handling. @@ -1474,6 +1541,8 @@ class QueueListener(object): try: record = self.dequeue(True) if record is self._sentinel: + if has_task_done: + q.task_done() break self.handle(record) if has_task_done: