Ruud
13 years ago
4 changed files with 2 additions and 486 deletions
@ -1,326 +0,0 @@ |
|||
# Copyright 2008 Lowell Alleman |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not |
|||
# use this file except in compliance with the License. You may obtain a copy |
|||
# of the License at http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
""" cloghandler.py: A smart replacement for the standard RotatingFileHandler |
|||
|
|||
ConcurrentRotatingFileHandler: This class is a log handler which is a drop-in |
|||
replacement for the python standard log handler 'RotateFileHandler', the primary |
|||
difference being that this handler will continue to write to the same file if |
|||
the file cannot be rotated for some reason, whereas the RotatingFileHandler will |
|||
strictly adhere to the maximum file size. Unfortunately, if you are using the |
|||
RotatingFileHandler on Windows, you will find that once an attempted rotation |
|||
fails, all subsequent log messages are dropped. The other major advantage of |
|||
this module is that multiple processes can safely write to a single log file. |
|||
|
|||
To put it another way: This module's top priority is preserving your log |
|||
records, whereas the standard library attempts to limit disk usage, which can |
|||
potentially drop log messages. If you are trying to determine which module to |
|||
use, there are number of considerations: What is most important: strict disk |
|||
space usage or preservation of log messages? What OSes are you supporting? Can |
|||
you afford to have processes blocked by file locks? |
|||
|
|||
Concurrent access is handled by using file locks, which should ensure that log |
|||
messages are not dropped or clobbered. This means that a file lock is acquired |
|||
and released for every log message that is written to disk. (On Windows, you may |
|||
also run into a temporary situation where the log file must be opened and closed |
|||
for each log message.) This can have potentially performance implications. In my |
|||
testing, performance was more than adequate, but if you need a high-volume or |
|||
low-latency solution, I suggest you look elsewhere. |
|||
|
|||
This module currently only support the 'nt' and 'posix' platforms due to the |
|||
usage of the portalocker module. I do not have access to any other platforms |
|||
for testing, patches are welcome. |
|||
|
|||
See the README file for an example usage of this module. |
|||
|
|||
""" |
|||
|
|||
|
|||
__version__ = "$Id: cloghandler.py 6175 2009-11-02 18:40:35Z lowell $" |
|||
__author__ = "Lowell Alleman" |
|||
__all__ = [ |
|||
"ConcurrentRotatingFileHandler", |
|||
] |
|||
|
|||
|
|||
import os |
|||
import sys |
|||
from random import randint |
|||
from logging import Handler |
|||
from logging.handlers import BaseRotatingHandler |
|||
|
|||
try: |
|||
import codecs |
|||
except ImportError: |
|||
codecs = None |
|||
|
|||
|
|||
|
|||
# Question/TODO: Should we have a fallback mode if we can't load portalocker / |
|||
# we should still be better off than with the standard RotattingFileHandler |
|||
# class, right? We do some rename checking... that should prevent some file |
|||
# clobbering that the builtin class allows. |
|||
|
|||
# sibling module than handles all the ugly platform-specific details of file locking |
|||
from portalocker import lock, unlock, LOCK_EX, LOCK_NB, LockException |
|||
|
|||
|
|||
# A client can set this to true to automatically convert relative paths to |
|||
# absolute paths (which will also hide the absolute path warnings) |
|||
FORCE_ABSOLUTE_PATH = False |
|||
|
|||
|
|||
class ConcurrentRotatingFileHandler(BaseRotatingHandler): |
|||
""" |
|||
Handler for logging to a set of files, which switches from one file to the |
|||
next when the current file reaches a certain size. Multiple processes can |
|||
write to the log file concurrently, but this may mean that the file will |
|||
exceed the given size. |
|||
""" |
|||
def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, |
|||
encoding=None, debug=True, supress_abs_warn=False): |
|||
""" |
|||
Open the specified file and use it as the stream for logging. |
|||
|
|||
By default, the file grows indefinitely. You can specify particular |
|||
values of maxBytes and backupCount to allow the file to rollover at |
|||
a predetermined size. |
|||
|
|||
Rollover occurs whenever the current log file is nearly maxBytes in |
|||
length. If backupCount is >= 1, the system will successively create |
|||
new files with the same pathname as the base file, but with extensions |
|||
".1", ".2" etc. appended to it. For example, with a backupCount of 5 |
|||
and a base file name of "app.log", you would get "app.log", |
|||
"app.log.1", "app.log.2", ... through to "app.log.5". The file being |
|||
written to is always "app.log" - when it gets filled up, it is closed |
|||
and renamed to "app.log.1", and if files "app.log.1", "app.log.2" etc. |
|||
exist, then they are renamed to "app.log.2", "app.log.3" etc. |
|||
respectively. |
|||
|
|||
If maxBytes is zero, rollover never occurs. |
|||
|
|||
On Windows, it is not possible to rename a file that is currently opened |
|||
by another process. This means that it is not possible to rotate the |
|||
log files if multiple processes is using the same log file. In this |
|||
case, the current log file will continue to grow until the rotation can |
|||
be completed successfully. In order for rotation to be possible, all of |
|||
the other processes need to close the file first. A mechanism, called |
|||
"degraded" mode, has been created for this scenario. In degraded mode, |
|||
the log file is closed after each log message is written. So once all |
|||
processes have entered degraded mode, the next rotate log attempt should |
|||
be successful and then normal logging can be resumed. |
|||
|
|||
This log handler assumes that all concurrent processes logging to a |
|||
single file will are using only this class, and that the exact same |
|||
parameters are provided to each instance of this class. If, for |
|||
example, two different processes are using this class, but with |
|||
different values for 'maxBytes' or 'backupCount', then odd behavior is |
|||
expected. The same is true if this class is used by one application, but |
|||
the RotatingFileHandler is used by another. |
|||
|
|||
NOTE: You should always provide 'filename' as an absolute path, since |
|||
this class will need to re-open the file during rotation. If your |
|||
application call os.chdir() then subsequent log files could be created |
|||
in the wrong directory. |
|||
""" |
|||
# The question of absolute paths: I'm not sure what the 'right thing' is |
|||
# to do here. RotatingFileHander simply ignores this possibility. I was |
|||
# going call os.path.abspath(), but that potentially limits uses. For |
|||
# example, on Linux (any posix system?) you can rename a directory of a |
|||
# running app, and the app wouldn't notice as long as it only opens new |
|||
# files using relative paths. But since that's not a "normal" thing to |
|||
# do, and having an app call os.chdir() is a much more likely scenario |
|||
# that should be supported. For the moment, we are just going to warn |
|||
# the user if they provide a relative path and do some other voodoo |
|||
# logic that you'll just have to review for yourself. |
|||
|
|||
# if the given filename contains no path, we make an absolute path |
|||
if not os.path.isabs(filename): |
|||
if FORCE_ABSOLUTE_PATH or \ |
|||
not os.path.split(filename)[0]: |
|||
filename = os.path.abspath(filename) |
|||
elif not supress_abs_warn: |
|||
from warnings import warn |
|||
warn("The given 'filename' should be an absolute path. If your " |
|||
"application calls os.chdir(), your logs may get messed up. " |
|||
"Use 'supress_abs_warn=True' to hide this message.") |
|||
try: |
|||
BaseRotatingHandler.__init__(self, filename, mode, encoding) |
|||
except TypeError: # Due to a different logging release without encoding support (Python 2.4.1 and earlier?) |
|||
BaseRotatingHandler.__init__(self, filename, mode) |
|||
self.encoding = encoding |
|||
|
|||
self._rotateFailed = False |
|||
self.maxBytes = maxBytes |
|||
self.backupCount = backupCount |
|||
# Prevent multiple extensions on the lock file (Only handles the normal "*.log" case.) |
|||
if filename.endswith(".log"): |
|||
lock_file = filename[:-4] |
|||
else: |
|||
lock_file = filename |
|||
self.stream_lock = open(lock_file + ".lock", "w") |
|||
|
|||
# For debug mode, swap out the "_degrade()" method with a more a verbose one. |
|||
if debug: |
|||
self._degrade = self._degrade_debug |
|||
|
|||
def _openFile(self, mode): |
|||
if self.encoding: |
|||
self.stream = codecs.open(self.baseFilename, mode, self.encoding) |
|||
else: |
|||
self.stream = open(self.baseFilename, mode) |
|||
|
|||
def acquire(self): |
|||
""" Acquire thread and file locks. Also re-opening log file when running |
|||
in 'degraded' mode. """ |
|||
# handle thread lock |
|||
Handler.acquire(self) |
|||
lock(self.stream_lock, LOCK_EX) |
|||
if self.stream.closed: |
|||
self._openFile(self.mode) |
|||
|
|||
def release(self): |
|||
""" Release file and thread locks. Flush stream and take care of closing |
|||
stream in 'degraded' mode. """ |
|||
try: |
|||
self.stream.flush() |
|||
if self._rotateFailed: |
|||
self.stream.close() |
|||
finally: |
|||
try: |
|||
unlock(self.stream_lock) |
|||
finally: |
|||
# release thread lock |
|||
Handler.release(self) |
|||
|
|||
def close(self): |
|||
""" |
|||
Closes the stream. |
|||
""" |
|||
if not self.stream.closed: |
|||
self.stream.flush() |
|||
self.stream.close() |
|||
Handler.close(self) |
|||
|
|||
def flush(self): |
|||
""" flush(): Do nothing. |
|||
|
|||
Since a flush is issued in release(), we don't do it here. To do a flush |
|||
here, it would be necessary to re-lock everything, and it is just easier |
|||
and cleaner to do it all in release(), rather than requiring two lock |
|||
ops per handle() call. |
|||
|
|||
Doing a flush() here would also introduces a window of opportunity for |
|||
another process to write to the log file in between calling |
|||
stream.write() and stream.flush(), which seems like a bad thing. """ |
|||
pass |
|||
|
|||
def _degrade(self, degrade, msg, *args): |
|||
""" Set degrade mode or not. Ignore msg. """ |
|||
self._rotateFailed = degrade |
|||
del msg, args # avoid pychecker warnings |
|||
|
|||
def _degrade_debug(self, degrade, msg, *args): |
|||
""" A more colorful version of _degade(). (This is enabled by passing |
|||
"debug=True" at initialization). |
|||
""" |
|||
if degrade: |
|||
if not self._rotateFailed: |
|||
sys.stderr.write("Degrade mode - ENTERING - (pid=%d) %s\n" % |
|||
(os.getpid(), msg % args)) |
|||
self._rotateFailed = True |
|||
else: |
|||
if self._rotateFailed: |
|||
sys.stderr.write("Degrade mode - EXITING - (pid=%d) %s\n" % |
|||
(os.getpid(), msg % args)) |
|||
self._rotateFailed = False |
|||
|
|||
def doRollover(self): |
|||
""" |
|||
Do a rollover, as described in __init__(). |
|||
""" |
|||
if self.backupCount <= 0: |
|||
# Don't keep any backups, just overwrite the existing backup file |
|||
# Locking doesn't much matter here; since we are overwriting it anyway |
|||
self.stream.close() |
|||
self._openFile("w") |
|||
return |
|||
self.stream.close() |
|||
try: |
|||
# Attempt to rename logfile to tempname: There is a slight race-condition here, but it seems unavoidable |
|||
tmpname = None |
|||
while not tmpname or os.path.exists(tmpname): |
|||
tmpname = "%s.rotate.%08d" % (self.baseFilename, randint(0,99999999)) |
|||
try: |
|||
# Do a rename test to determine if we can successfully rename the log file |
|||
os.rename(self.baseFilename, tmpname) |
|||
except (IOError, OSError): |
|||
exc_value = sys.exc_info()[1] |
|||
self._degrade(True, "rename failed. File in use? " |
|||
"exception=%s", exc_value) |
|||
return |
|||
|
|||
# Q: Is there some way to protect this code from a KeboardInterupt? |
|||
# This isn't necessarily a data loss issue, but it certainly would |
|||
# break the rotation process during my stress testing. |
|||
|
|||
# There is currently no mechanism in place to handle the situation |
|||
# where one of these log files cannot be renamed. (Example, user |
|||
# opens "logfile.3" in notepad) |
|||
for i in range(self.backupCount - 1, 0, -1): |
|||
sfn = "%s.%d" % (self.baseFilename, i) |
|||
dfn = "%s.%d" % (self.baseFilename, i + 1) |
|||
if os.path.exists(sfn): |
|||
#print "%s -> %s" % (sfn, dfn) |
|||
if os.path.exists(dfn): |
|||
os.remove(dfn) |
|||
os.rename(sfn, dfn) |
|||
dfn = self.baseFilename + ".1" |
|||
if os.path.exists(dfn): |
|||
os.remove(dfn) |
|||
os.rename(tmpname, dfn) |
|||
#print "%s -> %s" % (self.baseFilename, dfn) |
|||
self._degrade(False, "Rotation completed") |
|||
finally: |
|||
self._openFile(self.mode) |
|||
|
|||
def shouldRollover(self, record): |
|||
""" |
|||
Determine if rollover should occur. |
|||
|
|||
For those that are keeping track. This differs from the standard |
|||
library's RotatingLogHandler class. Because there is no promise to keep |
|||
the file size under maxBytes we ignore the length of the current record. |
|||
""" |
|||
del record # avoid pychecker warnings |
|||
if self._shouldRollover(): |
|||
# if some other process already did the rollover we might |
|||
# checked log.1, so we reopen the stream and check again on |
|||
# the right log file |
|||
self.stream.close() |
|||
self._openFile(self.mode) |
|||
return self._shouldRollover() |
|||
return False |
|||
|
|||
def _shouldRollover(self): |
|||
if self.maxBytes > 0: # are we rolling over? |
|||
self.stream.seek(0, 2) #due to non-posix-compliant Windows feature |
|||
if self.stream.tell() >= self.maxBytes: |
|||
return True |
|||
else: |
|||
self._degrade(False, "Rotation done or not needed at this time") |
|||
return False |
|||
|
|||
|
|||
# Publish this class to the "logging.handlers" module so that it can be use |
|||
# from a logging config file via logging.config.fileConfig(). |
|||
import logging.handlers |
|||
logging.handlers.ConcurrentRotatingFileHandler = ConcurrentRotatingFileHandler |
|||
@ -1,158 +0,0 @@ |
|||
# portalocker.py - Cross-platform (posix/nt) API for flock-style file locking. |
|||
# Requires python 1.5.2 or better. |
|||
"""Cross-platform (posix/nt) API for flock-style file locking. |
|||
|
|||
Synopsis: |
|||
|
|||
import portalocker |
|||
file = open("somefile", "r+") |
|||
portalocker.lock(file, portalocker.LOCK_EX) |
|||
file.seek(12) |
|||
file.write("foo") |
|||
file.close() |
|||
|
|||
If you know what you're doing, you may choose to |
|||
|
|||
portalocker.unlock(file) |
|||
|
|||
before closing the file, but why? |
|||
|
|||
Methods: |
|||
|
|||
lock( file, flags ) |
|||
unlock( file ) |
|||
|
|||
Constants: |
|||
|
|||
LOCK_EX |
|||
LOCK_SH |
|||
LOCK_NB |
|||
|
|||
Exceptions: |
|||
|
|||
LockException |
|||
|
|||
Notes: |
|||
|
|||
For the 'nt' platform, this module requires the Python Extensions for Windows. |
|||
Be aware that this may not work as expected on Windows 95/98/ME. |
|||
|
|||
History: |
|||
|
|||
I learned the win32 technique for locking files from sample code |
|||
provided by John Nielsen <nielsenjf@my-deja.com> in the documentation |
|||
that accompanies the win32 modules. |
|||
|
|||
Author: Jonathan Feinberg <jdf@pobox.com>, |
|||
Lowell Alleman <lalleman@mfps.com> |
|||
Version: $Id: portalocker.py 5488 2008-05-21 20:49:38Z lowell $ |
|||
|
|||
|
|||
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65203 |
|||
""" |
|||
|
|||
|
|||
__all__ = [ |
|||
"lock", |
|||
"unlock", |
|||
"LOCK_EX", |
|||
"LOCK_SH", |
|||
"LOCK_NB", |
|||
"LockException", |
|||
] |
|||
|
|||
import os |
|||
from types import IntType |
|||
|
|||
class LockException(Exception): |
|||
# Error codes: |
|||
LOCK_FAILED = 1 |
|||
|
|||
if os.name == 'nt': |
|||
import win32con |
|||
import win32file |
|||
import pywintypes |
|||
LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK |
|||
LOCK_SH = 0 # the default |
|||
LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY |
|||
# is there any reason not to reuse the following structure? |
|||
__overlapped = pywintypes.OVERLAPPED() |
|||
elif os.name == 'posix': |
|||
import fcntl |
|||
LOCK_EX = fcntl.LOCK_EX |
|||
LOCK_SH = fcntl.LOCK_SH |
|||
LOCK_NB = fcntl.LOCK_NB |
|||
else: |
|||
raise RuntimeError, "PortaLocker only defined for nt and posix platforms" |
|||
|
|||
|
|||
|
|||
def _getfd(file): |
|||
""" Get a file-descriptor from a file object or file-descriptor. """ |
|||
if hasattr(file, "fileno"): |
|||
return file.fileno() |
|||
elif type(file) == IntType: |
|||
return file |
|||
else: |
|||
raise TypeError("File object or file descriptor required, but %s " |
|||
"was provided." % type(file)) |
|||
|
|||
|
|||
if os.name == 'nt': |
|||
def lock(file, flags): |
|||
hfile = win32file._get_osfhandle(_getfd(file)) |
|||
try: |
|||
win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped) |
|||
except pywintypes.error, exc_value: |
|||
# error: (33, 'LockFileEx', 'The process cannot access the file because another process has locked a portion of the file.') |
|||
if exc_value[0] == 33: |
|||
raise LockException(LockException.LOCK_FAILED, exc_value[2]) |
|||
else: |
|||
# Q: Are there exceptions/codes we should be dealing with here? |
|||
raise |
|||
|
|||
def unlock(file): |
|||
hfile = win32file._get_osfhandle(_getfd(file)) |
|||
try: |
|||
win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped) |
|||
except pywintypes.error, exc_value: |
|||
if exc_value[0] == 158: |
|||
# error: (158, 'UnlockFileEx', 'The segment is already unlocked.') |
|||
# To match the 'posix' implementation, silently ignore this error |
|||
pass |
|||
else: |
|||
# Q: Are there exceptions/codes we should be dealing with here? |
|||
raise |
|||
|
|||
elif os.name == 'posix': |
|||
def lock(file, flags): |
|||
try: |
|||
fcntl.flock(_getfd(file), flags) |
|||
except IOError, exc_value: |
|||
# IOError: [Errno 11] Resource temporarily unavailable |
|||
if exc_value[0] == 11: |
|||
raise LockException(LockException.LOCK_FAILED, exc_value[1]) |
|||
else: |
|||
raise |
|||
|
|||
def unlock(file): |
|||
fcntl.flock(_getfd(file), fcntl.LOCK_UN) |
|||
|
|||
|
|||
|
|||
if __name__ == '__main__': |
|||
from time import time, strftime, localtime |
|||
import sys |
|||
import portalocker |
|||
|
|||
log = open('log.txt', "a+") |
|||
portalocker.lock(log, portalocker.LOCK_EX) |
|||
|
|||
timestamp = strftime("%m/%d/%Y %H:%M:%S\n", localtime(time())) |
|||
log.write( timestamp ) |
|||
|
|||
print "Wrote lines. Hit enter to release lock." |
|||
dummy = sys.stdin.readline() |
|||
|
|||
log.close() |
|||
|
|||
Loading…
Reference in new issue