You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
262 lines
8.1 KiB
262 lines
8.1 KiB
"""
|
|
copied from
|
|
werkzeug.contrib.cache
|
|
~~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
:copyright: (c) 2011 by the Werkzeug Team, see AUTHORS for more details.
|
|
:license: BSD, see LICENSE for more details.
|
|
"""
|
|
from cache.posixemulation import rename
|
|
from itertools import izip
|
|
from time import time
|
|
import os
|
|
import re
|
|
import tempfile
|
|
try:
|
|
from hashlib import md5
|
|
except ImportError:
|
|
from md5 import new as md5
|
|
|
|
try:
|
|
import cPickle as pickle
|
|
except ImportError:
|
|
import pickle
|
|
|
|
|
|
def _items(mappingorseq):
|
|
"""Wrapper for efficient iteration over mappings represented by dicts
|
|
or sequences::
|
|
|
|
>>> for k, v in _items((i, i*i) for i in xrange(5)):
|
|
... assert k*k == v
|
|
|
|
>>> for k, v in _items(dict((i, i*i) for i in xrange(5))):
|
|
... assert k*k == v
|
|
|
|
"""
|
|
return mappingorseq.iteritems() if hasattr(mappingorseq, 'iteritems') \
|
|
else mappingorseq
|
|
|
|
|
|
class BaseCache(object):
|
|
"""Baseclass for the cache systems. All the cache systems implement this
|
|
API or a superset of it.
|
|
|
|
:param default_timeout: the default timeout that is used if no timeout is
|
|
specified on :meth:`set`.
|
|
"""
|
|
|
|
def __init__(self, default_timeout = 300):
|
|
self.default_timeout = default_timeout
|
|
|
|
def delete(self, key):
|
|
"""Deletes `key` from the cache. If it does not exist in the cache
|
|
nothing happens.
|
|
|
|
:param key: the key to delete.
|
|
"""
|
|
pass
|
|
|
|
def get_many(self, *keys):
|
|
"""Returns a list of values for the given keys.
|
|
For each key a item in the list is created. Example::
|
|
|
|
foo, bar = cache.get_many("foo", "bar")
|
|
|
|
If a key can't be looked up `None` is returned for that key
|
|
instead.
|
|
|
|
:param keys: The function accepts multiple keys as positional
|
|
arguments.
|
|
"""
|
|
return map(self.get, keys)
|
|
|
|
def get_dict(self, *keys):
|
|
"""Works like :meth:`get_many` but returns a dict::
|
|
|
|
d = cache.get_dict("foo", "bar")
|
|
foo = d["foo"]
|
|
bar = d["bar"]
|
|
|
|
:param keys: The function accepts multiple keys as positional
|
|
arguments.
|
|
"""
|
|
return dict(izip(keys, self.get_many(*keys)))
|
|
|
|
def set(self, key, value, timeout = None):
|
|
"""Adds a new key/value to the cache (overwrites value, if key already
|
|
exists in the cache).
|
|
|
|
:param key: the key to set
|
|
:param value: the value for the key
|
|
:param timeout: the cache timeout for the key (if not specified,
|
|
it uses the default timeout).
|
|
"""
|
|
pass
|
|
|
|
def add(self, key, value, timeout = None):
|
|
"""Works like :meth:`set` but does not overwrite the values of already
|
|
existing keys.
|
|
|
|
:param key: the key to set
|
|
:param value: the value for the key
|
|
:param timeout: the cache timeout for the key or the default
|
|
timeout if not specified.
|
|
"""
|
|
pass
|
|
|
|
def set_many(self, mapping, timeout = None):
|
|
"""Sets multiple keys and values from a mapping.
|
|
|
|
:param mapping: a mapping with the keys/values to set.
|
|
:param timeout: the cache timeout for the key (if not specified,
|
|
it uses the default timeout).
|
|
"""
|
|
for key, value in _items(mapping):
|
|
self.set(key, value, timeout)
|
|
|
|
def delete_many(self, *keys):
|
|
"""Deletes multiple keys at once.
|
|
|
|
:param keys: The function accepts multiple keys as positional
|
|
arguments.
|
|
"""
|
|
for key in keys:
|
|
self.delete(key)
|
|
|
|
def clear(self):
|
|
"""Clears the cache. Keep in mind that not all caches support
|
|
completely clearing the cache.
|
|
"""
|
|
pass
|
|
|
|
def inc(self, key, delta = 1):
|
|
"""Increments the value of a key by `delta`. If the key does
|
|
not yet exist it is initialized with `delta`.
|
|
|
|
For supporting caches this is an atomic operation.
|
|
|
|
:param key: the key to increment.
|
|
:param delta: the delta to add.
|
|
"""
|
|
self.set(key, (self.get(key) or 0) + delta)
|
|
|
|
def dec(self, key, delta = 1):
|
|
"""Decrements the value of a key by `delta`. If the key does
|
|
not yet exist it is initialized with `-delta`.
|
|
|
|
For supporting caches this is an atomic operation.
|
|
|
|
:param key: the key to increment.
|
|
:param delta: the delta to subtract.
|
|
"""
|
|
self.set(key, (self.get(key) or 0) - delta)
|
|
|
|
|
|
class FileSystemCache(BaseCache):
|
|
"""A cache that stores the items on the file system. This cache depends
|
|
on being the only user of the `cache_dir`. Make absolutely sure that
|
|
nobody but this cache stores files there or otherwise the cache will
|
|
randomly delete files therein.
|
|
|
|
:param cache_dir: the directory where cache files are stored.
|
|
:param threshold: the maximum number of items the cache stores before
|
|
it starts deleting some.
|
|
:param default_timeout: the default timeout that is used if no timeout is
|
|
specified on :meth:`~BaseCache.set`.
|
|
:param mode: the file mode wanted for the cache files, default 0600
|
|
"""
|
|
|
|
#: used for temporary files by the FileSystemCache
|
|
_fs_transaction_suffix = '.__wz_cache'
|
|
|
|
def __init__(self, cache_dir, threshold = 500, default_timeout = 300, mode = 0600):
|
|
BaseCache.__init__(self, default_timeout)
|
|
self._path = cache_dir
|
|
self._threshold = threshold
|
|
self._mode = mode
|
|
if not os.path.exists(self._path):
|
|
os.makedirs(self._path)
|
|
|
|
def _list_dir(self):
|
|
"""return a list of (fully qualified) cache filenames
|
|
"""
|
|
return [os.path.join(self._path, fn) for fn in os.listdir(self._path)
|
|
if not fn.endswith(self._fs_transaction_suffix)]
|
|
|
|
def _prune(self):
|
|
entries = self._list_dir()
|
|
if len(entries) > self._threshold:
|
|
now = time()
|
|
for idx, fname in enumerate(entries):
|
|
remove = False
|
|
f = None
|
|
try:
|
|
try:
|
|
f = open(fname, 'rb')
|
|
expires = pickle.load(f)
|
|
remove = expires <= now or idx % 3 == 0
|
|
finally:
|
|
if f is not None:
|
|
f.close()
|
|
except Exception:
|
|
pass
|
|
if remove:
|
|
try:
|
|
os.remove(fname)
|
|
except (IOError, OSError):
|
|
pass
|
|
|
|
def clear(self):
|
|
for fname in self._list_dir():
|
|
try:
|
|
os.remove(fname)
|
|
except (IOError, OSError):
|
|
pass
|
|
|
|
def _get_filename(self, key):
|
|
hash = md5(key).hexdigest()
|
|
return os.path.join(self._path, hash)
|
|
|
|
def get(self, key):
|
|
filename = self._get_filename(key)
|
|
try:
|
|
f = open(filename, 'rb')
|
|
try:
|
|
if pickle.load(f) >= time():
|
|
return pickle.load(f)
|
|
finally:
|
|
f.close()
|
|
os.remove(filename)
|
|
except Exception:
|
|
return None
|
|
|
|
def add(self, key, value, timeout = None):
|
|
filename = self._get_filename(key)
|
|
if not os.path.exists(filename):
|
|
self.set(key, value, timeout)
|
|
|
|
def set(self, key, value, timeout = None):
|
|
if timeout is None:
|
|
timeout = self.default_timeout
|
|
filename = self._get_filename(key)
|
|
self._prune()
|
|
try:
|
|
fd, tmp = tempfile.mkstemp(suffix = self._fs_transaction_suffix,
|
|
dir = self._path)
|
|
f = os.fdopen(fd, 'wb')
|
|
try:
|
|
pickle.dump(int(time() + timeout), f, 1)
|
|
pickle.dump(value, f, pickle.HIGHEST_PROTOCOL)
|
|
finally:
|
|
f.close()
|
|
rename(tmp, filename)
|
|
os.chmod(filename, self._mode)
|
|
except (IOError, OSError):
|
|
pass
|
|
|
|
def delete(self, key):
|
|
try:
|
|
os.remove(self._get_filename(key))
|
|
except (IOError, OSError):
|
|
pass
|
|
|