""" copied from werkzeug.contrib.cache ~~~~~~~~~~~~~~~~~~~~~~ :copyright: (c) 2011 by the Werkzeug Team, see AUTHORS for more details. :license: BSD, see LICENSE for more details. """ from cache.posixemulation import rename from itertools import izip from time import time import os import re import tempfile try: from hashlib import md5 except ImportError: from md5 import new as md5 try: import cPickle as pickle except ImportError: import pickle def _items(mappingorseq): """Wrapper for efficient iteration over mappings represented by dicts or sequences:: >>> for k, v in _items((i, i*i) for i in xrange(5)): ... assert k*k == v >>> for k, v in _items(dict((i, i*i) for i in xrange(5))): ... assert k*k == v """ return mappingorseq.iteritems() if hasattr(mappingorseq, 'iteritems') \ else mappingorseq class BaseCache(object): """Baseclass for the cache systems. All the cache systems implement this API or a superset of it. :param default_timeout: the default timeout that is used if no timeout is specified on :meth:`set`. """ def __init__(self, default_timeout = 300): self.default_timeout = default_timeout def delete(self, key): """Deletes `key` from the cache. If it does not exist in the cache nothing happens. :param key: the key to delete. """ pass def get_many(self, *keys): """Returns a list of values for the given keys. For each key a item in the list is created. Example:: foo, bar = cache.get_many("foo", "bar") If a key can't be looked up `None` is returned for that key instead. :param keys: The function accepts multiple keys as positional arguments. """ return map(self.get, keys) def get_dict(self, *keys): """Works like :meth:`get_many` but returns a dict:: d = cache.get_dict("foo", "bar") foo = d["foo"] bar = d["bar"] :param keys: The function accepts multiple keys as positional arguments. """ return dict(izip(keys, self.get_many(*keys))) def set(self, key, value, timeout = None): """Adds a new key/value to the cache (overwrites value, if key already exists in the cache). :param key: the key to set :param value: the value for the key :param timeout: the cache timeout for the key (if not specified, it uses the default timeout). """ pass def add(self, key, value, timeout = None): """Works like :meth:`set` but does not overwrite the values of already existing keys. :param key: the key to set :param value: the value for the key :param timeout: the cache timeout for the key or the default timeout if not specified. """ pass def set_many(self, mapping, timeout = None): """Sets multiple keys and values from a mapping. :param mapping: a mapping with the keys/values to set. :param timeout: the cache timeout for the key (if not specified, it uses the default timeout). """ for key, value in _items(mapping): self.set(key, value, timeout) def delete_many(self, *keys): """Deletes multiple keys at once. :param keys: The function accepts multiple keys as positional arguments. """ for key in keys: self.delete(key) def clear(self): """Clears the cache. Keep in mind that not all caches support completely clearing the cache. """ pass def inc(self, key, delta = 1): """Increments the value of a key by `delta`. If the key does not yet exist it is initialized with `delta`. For supporting caches this is an atomic operation. :param key: the key to increment. :param delta: the delta to add. """ self.set(key, (self.get(key) or 0) + delta) def dec(self, key, delta = 1): """Decrements the value of a key by `delta`. If the key does not yet exist it is initialized with `-delta`. For supporting caches this is an atomic operation. :param key: the key to increment. :param delta: the delta to subtract. """ self.set(key, (self.get(key) or 0) - delta) class FileSystemCache(BaseCache): """A cache that stores the items on the file system. This cache depends on being the only user of the `cache_dir`. Make absolutely sure that nobody but this cache stores files there or otherwise the cache will randomly delete files therein. :param cache_dir: the directory where cache files are stored. :param threshold: the maximum number of items the cache stores before it starts deleting some. :param default_timeout: the default timeout that is used if no timeout is specified on :meth:`~BaseCache.set`. :param mode: the file mode wanted for the cache files, default 0600 """ #: used for temporary files by the FileSystemCache _fs_transaction_suffix = '.__wz_cache' def __init__(self, cache_dir, threshold = 500, default_timeout = 300, mode = 0600): BaseCache.__init__(self, default_timeout) self._path = cache_dir self._threshold = threshold self._mode = mode if not os.path.exists(self._path): os.makedirs(self._path) def _list_dir(self): """return a list of (fully qualified) cache filenames """ return [os.path.join(self._path, fn) for fn in os.listdir(self._path) if not fn.endswith(self._fs_transaction_suffix)] def _prune(self): entries = self._list_dir() if len(entries) > self._threshold: now = time() for idx, fname in enumerate(entries): remove = False f = None try: try: f = open(fname, 'rb') expires = pickle.load(f) remove = expires <= now or idx % 3 == 0 finally: if f is not None: f.close() except Exception: pass if remove: try: os.remove(fname) except (IOError, OSError): pass def clear(self): for fname in self._list_dir(): try: os.remove(fname) except (IOError, OSError): pass def _get_filename(self, key): hash = md5(key).hexdigest() return os.path.join(self._path, hash) def get(self, key): filename = self._get_filename(key) try: f = open(filename, 'rb') try: if pickle.load(f) >= time(): return pickle.load(f) finally: f.close() os.remove(filename) except Exception: return None def add(self, key, value, timeout = None): filename = self._get_filename(key) if not os.path.exists(filename): self.set(key, value, timeout) def set(self, key, value, timeout = None): if timeout is None: timeout = self.default_timeout filename = self._get_filename(key) self._prune() try: fd, tmp = tempfile.mkstemp(suffix = self._fs_transaction_suffix, dir = self._path) f = os.fdopen(fd, 'wb') try: pickle.dump(int(time() + timeout), f, 1) pickle.dump(value, f, pickle.HIGHEST_PROTOCOL) finally: f.close() rename(tmp, filename) os.chmod(filename, self._mode) except (IOError, OSError): pass def delete(self, key): try: os.remove(self._get_filename(key)) except (IOError, OSError): pass