dockeruser
/
SickGear
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Update DiskCache library 3.1.1 (2649ac9) → 4.0.0 (2c79bb9).

pull/1200/head
JackDandy 6 years ago
parent
e80a66c97c
commit
3dcbe6e17c
9 changed files with 657 additions and 432 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      CHANGES.md
  2. 36
      lib/diskcache/__init__.py
  3. 170
      lib/diskcache/core.py
  4. 83
      lib/diskcache/djangocache.py
  5. 93
      lib/diskcache/fanout.py
  6. 106
      lib/diskcache/memo.py
  7. 198
      lib/diskcache/persistent.py
  8. 324
      lib/diskcache/recipes.py
  9. 78
      lib/diskcache/stampede.py

1
CHANGES.md
View File

@ -2,6 +2,7 @@
* Update attr 19.2.0.dev0 (de84609) to 19.2.0.dev0 (154b4e5)
* Update Certifi 2019.03.09 (401100f) to 2019.06.16 (84dc766)
* Update DiskCache library 3.1.1 (2649ac9) to 4.0.0 (2c79bb9)
[develop changelog]

36
lib/diskcache/__init__.py
View File

@ -1,23 +1,38 @@
"DiskCache: disk and file backed cache."
"""
DiskCache API Reference
=======================
from .core import Cache, Disk, UnknownFileWarning, EmptyDirWarning, Timeout
The :doc:`tutorial` provides a helpful walkthrough of most methods.
"""
from .core import Cache, Disk, EmptyDirWarning, UnknownFileWarning, Timeout
from .core import DEFAULT_SETTINGS, ENOVAL, EVICTION_POLICY, UNKNOWN
from .fanout import FanoutCache
from .persistent import Deque, Index
from .recipes import Averager, BoundedSemaphore, Lock, RLock
from .recipes import barrier, memoize_stampede, throttle
__all__ = [
'Averager',
'BoundedSemaphore',
'Cache',
'Disk',
'UnknownFileWarning',
'EmptyDirWarning',
'Timeout',
'DEFAULT_SETTINGS',
'Deque',
'Disk',
'ENOVAL',
'EVICTION_POLICY',
'UNKNOWN',
'EmptyDirWarning',
'FanoutCache',
'Deque',
'Index',
'Lock',
'RLock',
'Timeout',
'UNKNOWN',
'UnknownFileWarning',
'barrier',
'memoize_stampede',
'throttle',
]
try:
@ -27,10 +42,9 @@ except Exception: # pylint: disable=broad-except
# Django not installed or not setup so ignore.
pass
__title__ = 'diskcache'
__version__ = '3.1.1'
__build__ = 0x030101
__version__ = '4.0.0'
__build__ = 0x040000
__author__ = 'Grant Jenks'
__license__ = 'Apache 2.0'
__copyright__ = 'Copyright 2016-2018 Grant Jenks'

170
lib/diskcache/core.py
View File

@ -13,12 +13,15 @@ import pickletools
import sqlite3
import struct
import sys
import tempfile
import threading
import time
import warnings
import zlib
from .memo import memoize
############################################################################
# BEGIN Python 2/3 Shims
############################################################################
if sys.hexversion < 0x03000000:
import cPickle as pickle # pylint: disable=import-error
@ -39,6 +42,20 @@ else:
INT_TYPES = (int,)
io_open = open # pylint: disable=invalid-name
def full_name(func):
"Return full name of `func` by adding the module and function name."
try:
# The __qualname__ attribute is only available in Python 3.3 and later.
# GrantJ 2019-03-29 Remove after support for Python 2 is dropped.
name = func.__qualname__
except AttributeError:
name = func.__name__
return func.__module__ + '.' + name
############################################################################
# END Python 2/3 Shims
############################################################################
try:
WindowsError
except NameError:
@ -356,10 +373,38 @@ class EmptyDirWarning(UserWarning):
"Warning used by Cache.check for empty directories."
def args_to_key(base, args, kwargs, typed):
"""Create cache key out of function arguments.
:param tuple base: base of key
:param tuple args: function arguments
:param dict kwargs: function keyword arguments
:param bool typed: include types in cache key
:return: cache key tuple
"""
key = base + args
if kwargs:
key += (ENOVAL,)
sorted_items = sorted(kwargs.items())
for item in sorted_items:
key += item
if typed:
key += tuple(type(arg) for arg in args)
if kwargs:
key += tuple(type(value) for _, value in sorted_items)
return key
class Cache(object):
"Disk and file backed cache."
# pylint: disable=bad-continuation
def __init__(self, directory, timeout=60, disk=Disk, **settings):
def __init__(self, directory=None, timeout=60, disk=Disk, **settings):
"""Initialize cache instance.
:param str directory: cache directory
@ -373,6 +418,11 @@ class Cache(object):
except (TypeError, AssertionError):
raise ValueError('disk must subclass diskcache.Disk')
if directory is None:
directory = tempfile.mkdtemp(prefix='diskcache-')
directory = op.expanduser(directory)
directory = op.expandvars(directory)
self._directory = directory
self._timeout = 0 # Manually handle retries during initialization.
self._local = threading.local()
@ -621,8 +671,7 @@ class Cache(object):
Raises :exc:`Timeout` error when database timeout occurs and `retry` is
`False` (default).
>>> cache = Cache('/tmp/diskcache')
>>> _ = cache.clear()
>>> cache = Cache()
>>> with cache.transact(): # Atomically increment two keys.
... _ = cache.incr('total', 123.4)
... _ = cache.incr('count', 1)
@ -688,6 +737,9 @@ class Cache(object):
When `read` is `True`, `value` should be a file-like object opened
for reading in binary mode.
If `expire` is less than or equal to zero then immediately returns
`False`.
Raises :exc:`Timeout` error when database timeout occurs and `retry` is
`False` (default).
@ -702,6 +754,9 @@ class Cache(object):
:raises Timeout: if database timeout occurs
"""
if expire is not None and expire <= 0:
return False
now = time.time()
db_key, raw = self._disk.put(key)
expire_time = None if expire is None else now + expire
@ -1344,8 +1399,7 @@ class Cache(object):
See also `Cache.pull`.
>>> cache = Cache('/tmp/diskcache')
>>> _ = cache.clear()
>>> cache = Cache()
>>> print(cache.push('first value'))
500000000000000
>>> cache.get(500000000000000)
@ -1438,8 +1492,7 @@ class Cache(object):
See also `Cache.push` and `Cache.get`.
>>> cache = Cache('/tmp/diskcache')
>>> _ = cache.clear()
>>> cache = Cache()
>>> cache.pull()
(None, None)
>>> for letter in 'abc':
@ -1555,8 +1608,7 @@ class Cache(object):
See also `Cache.pull` and `Cache.push`.
>>> cache = Cache('/tmp/diskcache')
>>> _ = cache.clear()
>>> cache = Cache()
>>> for letter in 'abc':
... print(cache.push(letter))
500000000000000
@ -1654,8 +1706,7 @@ class Cache(object):
Raises :exc:`Timeout` error when database timeout occurs and `retry` is
`False` (default).
>>> cache = Cache('/tmp/diskcache')
>>> _ = cache.clear()
>>> cache = Cache()
>>> for num, letter in enumerate('abc'):
... cache[letter] = num
>>> cache.peekitem()
@ -1706,9 +1757,6 @@ class Cache(object):
continue
else:
raise
finally:
if name is not None:
self._disk.remove(name)
break
if expire_time and tag:
@ -1721,7 +1769,92 @@ class Cache(object):
return key, value
memoize = memoize
def memoize(self, name=None, typed=False, expire=None, tag=None):
"""Memoizing cache decorator.
Decorator to wrap callable with memoizing function using cache.
Repeated calls with the same arguments will lookup result in cache and
avoid function evaluation.
If name is set to None (default), the callable name will be determined
automatically.
If typed is set to True, function arguments of different types will be
cached separately. For example, f(3) and f(3.0) will be treated as
distinct calls with distinct results.
The original underlying function is accessible through the __wrapped__
attribute. This is useful for introspection, for bypassing the cache,
or for rewrapping the function with a different cache.
>>> from diskcache import Cache
>>> cache = Cache()
>>> @cache.memoize(expire=1, tag='fib')
... def fibonacci(number):
... if number == 0:
... return 0
... elif number == 1:
... return 1
... else:
... return fibonacci(number - 1) + fibonacci(number - 2)
>>> print(fibonacci(100))
354224848179261915075
An additional `__cache_key__` attribute can be used to generate the
cache key used for the given arguments.
>>> key = fibonacci.__cache_key__(100)
>>> print(cache[key])
354224848179261915075
Remember to call memoize when decorating a callable. If you forget,
then a TypeError will occur. Note the lack of parenthenses after
memoize below:
>>> @cache.memoize
... def test():
... pass
Traceback (most recent call last):
...
TypeError: name cannot be callable
:param cache: cache to store callable arguments and return values
:param str name: name given for callable (default None, automatic)
:param bool typed: cache different types separately (default False)
:param float expire: seconds until arguments expire
(default None, no expiry)
:param str tag: text to associate with arguments (default None)
:return: callable decorator
"""
# Caution: Nearly identical code exists in DjangoCache.memoize
if callable(name):
raise TypeError('name cannot be callable')
def decorator(func):
"Decorator created by memoize() for callable `func`."
base = (full_name(func),) if name is None else (name,)
@ft.wraps(func)
def wrapper(*args, **kwargs):
"Wrapper for callable to cache arguments and return values."
key = wrapper.__cache_key__(*args, **kwargs)
result = self.get(key, default=ENOVAL, retry=True)
if result is ENOVAL:
result = func(*args, **kwargs)
self.set(key, result, expire=expire, tag=tag, retry=True)
return result
def __cache_key__(*args, **kwargs):
"Make key for cache given function arguments."
return args_to_key(base, args, kwargs, typed)
wrapper.__cache_key__ = __cache_key__
return wrapper
return decorator
def check(self, fix=False, retry=False):
@ -2046,8 +2179,7 @@ class Cache(object):
def iterkeys(self, reverse=False):
"""Iterate Cache keys in database sort order.
>>> cache = Cache('/tmp/diskcache')
>>> _ = cache.clear()
>>> cache = Cache()
>>> for key in [4, 1, 3, 0, 2]:
... cache[key] = key
>>> list(cache.iterkeys())
@ -2200,6 +2332,8 @@ class Cache(object):
def __enter__(self):
# Create connection in thread.
connection = self._con # pylint: disable=unused-variable
return self

83
lib/diskcache/djangocache.py
View File

@ -9,10 +9,9 @@ except ImportError:
# For older versions of Django simply use 300 seconds.
DEFAULT_TIMEOUT = 300
from .core import ENOVAL, args_to_key, full_name
from .fanout import FanoutCache
MARK = object()
class DjangoCache(BaseCache):
"Django-compatible disk and file backed cache."
@ -27,20 +26,23 @@ class DjangoCache(BaseCache):
shards = params.get('SHARDS', 8)
timeout = params.get('DATABASE_TIMEOUT', 0.010)
options = params.get('OPTIONS', {})
self._directory = directory
self._cache = FanoutCache(directory, shards, timeout, **options)
@property
def cache(self):
"FanoutCache used by DjangoCache."
return self._cache
@property
def directory(self):
"""Cache directory."""
return self._directory
return self._cache.directory
def cache(self, name):
"""Return Cache with given `name` in subdirectory.
:param str name: subdirectory name for Cache
:return: Cache with given name
"""
return self._cache.cache(name)
def deque(self, name):
@ -293,7 +295,7 @@ class DjangoCache(BaseCache):
def create_tag_index(self):
"""Create tag index on cache database.
It is better to initialize cache with `tag_index=True` than use this.
Better to initialize cache with `tag_index=True` than use this.
:raises Timeout: if database timeout occurs
@ -374,8 +376,11 @@ class DjangoCache(BaseCache):
attribute. This is useful for introspection, for bypassing the cache,
or for rewrapping the function with a different cache.
Remember to call memoize when decorating a callable. If you forget, then a
TypeError will occur.
An additional `__cache_key__` attribute can be used to generate the
cache key used for the given arguments.
Remember to call memoize when decorating a callable. If you forget,
then a TypeError will occur.
:param str name: name given for callable (default None, automatic)
:param float timeout: seconds until the item expires
@ -386,51 +391,33 @@ class DjangoCache(BaseCache):
:return: callable decorator
"""
# Caution: Nearly identical code exists in memo.memoize
# Caution: Nearly identical code exists in Cache.memoize
if callable(name):
raise TypeError('name cannot be callable')
def decorator(function):
"Decorator created by memoize call for callable."
if name is None:
try:
reference = function.__qualname__
except AttributeError:
reference = function.__name__
def decorator(func):
"Decorator created by memoize() for callable `func`."
base = (full_name(func),) if name is None else (name,)
reference = function.__module__ + reference
else:
reference = name
reference = (reference,)
@wraps(function)
@wraps(func)
def wrapper(*args, **kwargs):
"Wrapper for callable to cache arguments and return values."
key = wrapper.__cache_key__(*args, **kwargs)
result = self.get(key, ENOVAL, version, retry=True)
key = reference + args
if kwargs:
key += (MARK,)
sorted_items = sorted(kwargs.items())
for item in sorted_items:
key += item
if typed:
key += tuple(type(arg) for arg in args)
if kwargs:
key += tuple(type(value) for _, value in sorted_items)
result = self.get(key, MARK, version, retry=True)
if result is MARK:
result = function(*args, **kwargs)
self.set(key, result, timeout, version, tag=tag, retry=True)
if result is ENOVAL:
result = func(*args, **kwargs)
self.set(
key, result, timeout, version, tag=tag, retry=True,
)
return result
def __cache_key__(*args, **kwargs):
"Make key for cache given function arguments."
return args_to_key(base, args, kwargs, typed)
wrapper.__cache_key__ = __cache_key__
return wrapper
return decorator

93
lib/diskcache/fanout.py
View File

@ -4,21 +4,28 @@ import itertools as it
import operator
import os.path as op
import sqlite3
import sys
import tempfile
import time
try:
from functools import reduce
except ImportError:
reduce # pylint: disable=pointless-statement
from .core import ENOVAL, DEFAULT_SETTINGS, Cache, Disk, Timeout
from .memo import memoize
from .persistent import Deque, Index
############################################################################
# BEGIN Python 2/3 Shims
############################################################################
if sys.hexversion >= 0x03000000:
from functools import reduce
############################################################################
# END Python 2/3 Shims
############################################################################
class FanoutCache(object):
"Cache that shards keys and values."
def __init__(self, directory, shards=8, timeout=0.010, disk=Disk,
def __init__(self, directory=None, shards=8, timeout=0.010, disk=Disk,
**settings):
"""Initialize cache instance.
@ -29,13 +36,19 @@ class FanoutCache(object):
:param settings: any of `DEFAULT_SETTINGS`
"""
self._directory = directory
self._count = shards
if directory is None:
directory = tempfile.mkdtemp(prefix='diskcache-')
directory = op.expanduser(directory)
directory = op.expandvars(directory)
default_size_limit = DEFAULT_SETTINGS['size_limit']
size_limit = settings.pop('size_limit', default_size_limit) / shards
self._count = shards
self._directory = directory
self._shards = tuple(
Cache(
op.join(directory, '%03d' % num),
directory=op.join(directory, '%03d' % num),
timeout=timeout,
disk=disk,
size_limit=size_limit,
@ -44,6 +57,7 @@ class FanoutCache(object):
for num in range(shards)
)
self._hash = self._shards[0].disk.hash
self._caches = {}
self._deques = {}
self._indexes = {}
@ -348,9 +362,6 @@ class FanoutCache(object):
del shard[key]
memoize = memoize
def check(self, fix=False, retry=False):
"""Check database and file system consistency.
@ -391,7 +402,7 @@ class FanoutCache(object):
def create_tag_index(self):
"""Create tag index on cache database.
It is better to initialize cache with `tag_index=True` than use this.
Better to initialize cache with `tag_index=True` than use this.
:raises Timeout: if database timeout occurs
@ -492,6 +503,7 @@ class FanoutCache(object):
"Close database connection."
for shard in self._shards:
shard.close()
self._caches.clear()
self._deques.clear()
self._indexes.clear()
@ -545,7 +557,6 @@ class FanoutCache(object):
:param str key: Settings key for item
:param value: value for item (optional)
:return: updated value for item
:raises Timeout: if database timeout occurs
"""
for shard in self._shards:
@ -559,12 +570,41 @@ class FanoutCache(object):
return result
def cache(self, name):
"""Return Cache with given `name` in subdirectory.
>>> fanout_cache = FanoutCache()
>>> cache = fanout_cache.cache('test')
>>> cache.set('abc', 123)
True
>>> cache.get('abc')
123
>>> len(cache)
1
>>> cache.delete('abc')
True
:param str name: subdirectory name for Cache
:return: Cache with given name
"""
_caches = self._caches
try:
return _caches[name]
except KeyError:
parts = name.split('/')
directory = op.join(self._directory, 'cache', *parts)
temp = Cache(directory=directory)
_caches[name] = temp
return temp
def deque(self, name):
"""Return Deque with given `name` in subdirectory.
>>> cache = FanoutCache('/tmp/diskcache/fanoutcache')
>>> cache = FanoutCache()
>>> deque = cache.deque('test')
>>> deque.clear()
>>> deque.extend('abc')
>>> deque.popleft()
'a'
@ -592,9 +632,8 @@ class FanoutCache(object):
def index(self, name):
"""Return Index with given `name` in subdirectory.
>>> cache = FanoutCache('/tmp/diskcache/fanoutcache')
>>> cache = FanoutCache()
>>> index = cache.index('test')
>>> index.clear()
>>> index['abc'] = 123
>>> index['def'] = 456
>>> index['ghi'] = 789
@ -620,3 +659,19 @@ class FanoutCache(object):
temp = Index(directory)
_indexes[name] = temp
return temp
############################################################################
# BEGIN Python 2/3 Shims
############################################################################
if sys.hexversion < 0x03000000:
import types
memoize_func = Cache.__dict__['memoize'] # pylint: disable=invalid-name
FanoutCache.memoize = types.MethodType(memoize_func, None, FanoutCache)
else:
FanoutCache.memoize = Cache.memoize
############################################################################
# END Python 2/3 Shims
############################################################################

106
lib/diskcache/memo.py
View File

@ -1,106 +0,0 @@
"""Memoization utilities.
"""
from functools import wraps
MARK = object()
def memoize(cache, name=None, typed=False, expire=None, tag=None):
"""Memoizing cache decorator.
Decorator to wrap callable with memoizing function using cache. Repeated
calls with the same arguments will lookup result in cache and avoid
function evaluation.
If name is set to None (default), the callable name will be determined
automatically.
If typed is set to True, function arguments of different types will be
cached separately. For example, f(3) and f(3.0) will be treated as distinct
calls with distinct results.
The original underlying function is accessible through the __wrapped__
attribute. This is useful for introspection, for bypassing the cache, or
for rewrapping the function with a different cache.
>>> from diskcache import FanoutCache
>>> cache = FanoutCache('/tmp/diskcache/fanoutcache')
>>> @cache.memoize(typed=True, expire=1, tag='fib')
... def fibonacci(number):
... if number == 0:
... return 0
... elif number == 1:
... return 1
... else:
... return fibonacci(number - 1) + fibonacci(number - 2)
>>> print(sum(fibonacci(number=value) for value in range(100)))
573147844013817084100
Remember to call memoize when decorating a callable. If you forget, then a
TypeError will occur. Note the lack of parenthenses after memoize below:
>>> @cache.memoize
... def test():
... pass
Traceback (most recent call last):
...
TypeError: name cannot be callable
:param cache: cache to store callable arguments and return values
:param str name: name given for callable (default None, automatic)
:param bool typed: cache different types separately (default False)
:param float expire: seconds until arguments expire
(default None, no expiry)
:param str tag: text to associate with arguments (default None)
:return: callable decorator
"""
# Caution: Nearly identical code exists in DjangoCache.memoize
if callable(name):
raise TypeError('name cannot be callable')
def decorator(function):
"Decorator created by memoize call for callable."
if name is None:
try:
reference = function.__qualname__
except AttributeError:
reference = function.__name__
reference = function.__module__ + reference
else:
reference = name
reference = (reference,)
@wraps(function)
def wrapper(*args, **kwargs):
"Wrapper for callable to cache arguments and return values."
key = reference + args
if kwargs:
key += (MARK,)
sorted_items = sorted(kwargs.items())
for item in sorted_items:
key += item
if typed:
key += tuple(type(arg) for arg in args)
if kwargs:
key += tuple(type(value) for _, value in sorted_items)
result = cache.get(key, default=MARK, retry=True)
if result is MARK:
result = function(*args, **kwargs)
cache.set(key, result, expire=expire, tag=tag, retry=True)
return result
return wrapper
return decorator

198
lib/diskcache/persistent.py
View File

@ -8,7 +8,6 @@ import sys
from collections import OrderedDict
from contextlib import contextmanager
from shutil import rmtree
from tempfile import mkdtemp
from .core import BytesType, Cache, ENOVAL, TextType
@ -70,10 +69,7 @@ class Deque(Sequence):
Items are serialized to disk. Deque may be initialized from directory path
where items are stored.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque
Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque += range(5)
>>> list(deque)
[0, 1, 2, 3, 4]
@ -102,8 +98,6 @@ class Deque(Sequence):
:param directory: deque directory (default None)
"""
if directory is None:
directory = mkdtemp()
self._cache = Cache(directory, eviction_policy='none')
with self.transact():
self.extend(iterable)
@ -113,9 +107,10 @@ class Deque(Sequence):
def fromcache(cls, cache, iterable=()):
"""Initialize deque using `cache`.
>>> cache = Cache('/tmp/diskcache/index')
>>> _ = cache.clear()
>>> cache = Cache()
>>> deque = Deque.fromcache(cache, [5, 6, 7, 8])
>>> deque.cache is cache
True
>>> len(deque)
4
>>> 7 in deque
@ -178,7 +173,6 @@ class Deque(Sequence):
raise IndexError('deque index out of range')
def __getitem__(self, index):
"""deque.__getitem__(index) <==> deque[index]
@ -187,8 +181,7 @@ class Deque(Sequence):
See also `Deque.peekleft` and `Deque.peek` for indexing deque at index
``0`` or ``-1``.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque.extend('abcde')
>>> deque[1]
'b'
@ -208,8 +201,7 @@ class Deque(Sequence):
Store `value` in deque at `index`.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque.extend([None] * 3)
>>> deque[0] = 'a'
>>> deque[1] = 'b'
@ -231,8 +223,7 @@ class Deque(Sequence):
Delete item in deque at `index`.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque.extend([None] * 3)
>>> del deque[0]
>>> del deque[1]
@ -307,8 +298,7 @@ class Deque(Sequence):
Return iterator of deque from back to front.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque.extend('abcd')
>>> iterator = reversed(deque)
>>> next(iterator)
@ -337,8 +327,7 @@ class Deque(Sequence):
def append(self, value):
"""Add `value` to back of deque.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque.append('a')
>>> deque.append('b')
>>> deque.append('c')
@ -354,8 +343,7 @@ class Deque(Sequence):
def appendleft(self, value):
"""Add `value` to front of deque.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque.appendleft('a')
>>> deque.appendleft('b')
>>> deque.appendleft('c')
@ -371,6 +359,13 @@ class Deque(Sequence):
def clear(self):
"""Remove all elements from deque.
>>> deque = Deque('abc')
>>> len(deque)
3
>>> deque.clear()
>>> list(deque)
[]
"""
self._cache.clear(retry=True)
@ -378,8 +373,7 @@ class Deque(Sequence):
def count(self, value):
"""Return number of occurrences of `value` in deque.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque += [num for num in range(1, 5) for _ in range(num)]
>>> deque.count(0)
0
@ -408,8 +402,7 @@ class Deque(Sequence):
def extendleft(self, iterable):
"""Extend front side of deque with value from `iterable`.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque.extendleft('abc')
>>> list(deque)
['c', 'b', 'a']
@ -428,8 +421,7 @@ class Deque(Sequence):
If deque is empty then raise IndexError.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque.peek()
Traceback (most recent call last):
...
@ -456,8 +448,7 @@ class Deque(Sequence):
If deque is empty then raise IndexError.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque.peekleft()
Traceback (most recent call last):
...
@ -482,8 +473,7 @@ class Deque(Sequence):
If deque is empty then raise IndexError.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque += 'ab'
>>> deque.pop()
'b'
@ -508,8 +498,7 @@ class Deque(Sequence):
def popleft(self):
"""Remove and return value at front of deque.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque += 'ab'
>>> deque.popleft()
'a'
@ -534,8 +523,7 @@ class Deque(Sequence):
def remove(self, value):
"""Remove first occurrence of `value` in deque.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque += 'aab'
>>> deque.remove('a')
>>> list(deque)
@ -573,8 +561,7 @@ class Deque(Sequence):
def reverse(self):
"""Reverse deque in place.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque += 'abc'
>>> deque.reverse()
>>> list(deque)
@ -599,8 +586,7 @@ class Deque(Sequence):
If steps is negative then rotate left.
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque += range(5)
>>> deque.rotate(2)
>>> list(deque)
@ -659,8 +645,7 @@ class Deque(Sequence):
Transactions may be nested and may not be shared between threads.
>>> from diskcache import Deque
>>> deque = Deque(directory='/tmp/diskcache/deque')
>>> deque.clear()
>>> deque = Deque()
>>> deque += range(5)
>>> with deque.transact(): # Atomically rotate elements.
... value = deque.pop()
@ -669,7 +654,6 @@ class Deque(Sequence):
[4, 0, 1, 2, 3]
:return: context manager for use in `with` statement
:raises Timeout: if database timeout occurs
"""
with self._cache.transact(retry=True):
@ -685,10 +669,7 @@ class Index(MutableMapping):
Hashing protocol is not used. Keys are looked up by their serialized
format. See ``diskcache.Disk`` for details.
>>> index = Index('/tmp/diskcache/index')
>>> index
Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update([('a', 1), ('b', 2), ('c', 3)])
>>> index['a']
1
@ -724,7 +705,7 @@ class Index(MutableMapping):
else:
if args and args[0] is None:
args = args[1:]
directory = mkdtemp(prefix='diskcache-')
directory = None
self._cache = Cache(directory, eviction_policy='none')
self.update(*args, **kwargs)
@ -733,9 +714,10 @@ class Index(MutableMapping):
def fromcache(cls, cache, *args, **kwargs):
"""Initialize index using `cache` and update items.
>>> cache = Cache('/tmp/diskcache/index')
>>> _ = cache.clear()
>>> cache = Cache()
>>> index = Index.fromcache(cache, {'a': 1, 'b': 2, 'c': 3})
>>> index.cache is cache
True
>>> len(index)
3
>>> 'b' in index
@ -773,8 +755,7 @@ class Index(MutableMapping):
Return corresponding value for `key` in index.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update({'a': 1, 'b': 2})
>>> index['a']
1
@ -798,8 +779,7 @@ class Index(MutableMapping):
Set `key` and `value` item in index.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index['a'] = 1
>>> index[0] = None
>>> len(index)
@ -817,8 +797,7 @@ class Index(MutableMapping):
Delete corresponding item for `key` from index.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update({'a': 1, 'b': 2})
>>> del index['a']
>>> del index['b']
@ -842,8 +821,7 @@ class Index(MutableMapping):
If `key` is not in index then set corresponding value to `default`. If
`key` is in index then ignore `default` and return existing value.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.setdefault('a', 0)
0
>>> index.setdefault('a', 1)
@ -865,8 +843,7 @@ class Index(MutableMapping):
def peekitem(self, last=True):
"""Peek at key and value item pair in index based on iteration order.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> for num, letter in enumerate('xyz'):
... index[letter] = num
>>> index.peekitem()
@ -888,7 +865,7 @@ class Index(MutableMapping):
If `key` is missing then return `default`. If `default` is `ENOVAL`
then raise KeyError.
>>> index = Index('/tmp/diskcache/index', {'a': 1, 'b': 2})
>>> index = Index({'a': 1, 'b': 2})
>>> index.pop('a')
1
>>> index.pop('b')
@ -920,8 +897,7 @@ class Index(MutableMapping):
True else first-in-first-out (FIFO) order. LIFO order imitates a stack
and FIFO order imitates a queue.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update([('a', 1), ('b', 2), ('c', 3)])
>>> index.popitem()
('c', 3)
@ -932,7 +908,7 @@ class Index(MutableMapping):
>>> index.popitem()
Traceback (most recent call last):
...
KeyError
KeyError: 'dictionary is empty'
:param bool last: pop last item pair (default True)
:return: key and value item pair
@ -942,21 +918,11 @@ class Index(MutableMapping):
# pylint: disable=arguments-differ
_cache = self._cache
while True:
try:
if last:
key = next(reversed(_cache))
else:
key = next(iter(_cache))
except StopIteration:
raise KeyError
with _cache.transact(retry=True):
key, value = _cache.peekitem(last=last)
del _cache[key]
try:
value = _cache.pop(key, retry=True)
except KeyError:
continue
else:
return key, value
return key, value
def push(self, value, prefix=None, side='back'):
@ -970,8 +936,7 @@ class Index(MutableMapping):
See also `Index.pull`.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> print(index.push('apples'))
500000000000000
>>> print(index.push('beans'))
@ -1006,8 +971,7 @@ class Index(MutableMapping):
See also `Index.push`.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> for letter in 'abc':
... print(index.push(letter))
500000000000000
@ -1037,6 +1001,13 @@ class Index(MutableMapping):
def clear(self):
"""Remove all items from index.
>>> index = Index({'a': 0, 'b': 1, 'c': 2})
>>> len(index)
3
>>> index.clear()
>>> dict(index)
{}
"""
self._cache.clear(retry=True)
@ -1055,8 +1026,7 @@ class Index(MutableMapping):
Return iterator of index keys in reversed insertion order.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update([('a', 1), ('b', 2), ('c', 3)])
>>> iterator = reversed(index)
>>> next(iterator)
@ -1081,8 +1051,7 @@ class Index(MutableMapping):
def keys(self):
"""List of index keys.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update([('a', 1), ('b', 2), ('c', 3)])
>>> index.keys()
['a', 'b', 'c']
@ -1096,8 +1065,7 @@ class Index(MutableMapping):
def values(self):
"""List of index values.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update([('a', 1), ('b', 2), ('c', 3)])
>>> index.values()
[1, 2, 3]
@ -1111,8 +1079,7 @@ class Index(MutableMapping):
def items(self):
"""List of index items.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update([('a', 1), ('b', 2), ('c', 3)])
>>> index.items()
[('a', 1), ('b', 2), ('c', 3)]
@ -1126,8 +1093,7 @@ class Index(MutableMapping):
def iterkeys(self):
"""Iterator of index keys.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update([('a', 1), ('b', 2), ('c', 3)])
>>> list(index.iterkeys())
['a', 'b', 'c']
@ -1141,8 +1107,7 @@ class Index(MutableMapping):
def itervalues(self):
"""Iterator of index values.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update([('a', 1), ('b', 2), ('c', 3)])
>>> list(index.itervalues())
[1, 2, 3]
@ -1164,8 +1129,7 @@ class Index(MutableMapping):
def iteritems(self):
"""Iterator of index items.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update([('a', 1), ('b', 2), ('c', 3)])
>>> list(index.iteritems())
[('a', 1), ('b', 2), ('c', 3)]
@ -1187,8 +1151,7 @@ class Index(MutableMapping):
def viewkeys(self):
"""Set-like object providing a view of index keys.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update({'a': 1, 'b': 2, 'c': 3})
>>> keys_view = index.viewkeys()
>>> 'b' in keys_view
@ -1203,8 +1166,7 @@ class Index(MutableMapping):
def viewvalues(self):
"""Set-like object providing a view of index values.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update({'a': 1, 'b': 2, 'c': 3})
>>> values_view = index.viewvalues()
>>> 2 in values_view
@ -1219,8 +1181,7 @@ class Index(MutableMapping):
def viewitems(self):
"""Set-like object providing a view of index items.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update({'a': 1, 'b': 2, 'c': 3})
>>> items_view = index.viewitems()
>>> ('b', 2) in items_view
@ -1236,8 +1197,7 @@ class Index(MutableMapping):
def keys(self):
"""Set-like object providing a view of index keys.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update({'a': 1, 'b': 2, 'c': 3})
>>> keys_view = index.keys()
>>> 'b' in keys_view
@ -1252,8 +1212,7 @@ class Index(MutableMapping):
def values(self):
"""Set-like object providing a view of index values.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update({'a': 1, 'b': 2, 'c': 3})
>>> values_view = index.values()
>>> 2 in values_view
@ -1268,8 +1227,7 @@ class Index(MutableMapping):
def items(self):
"""Set-like object providing a view of index items.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update({'a': 1, 'b': 2, 'c': 3})
>>> items_view = index.items()
>>> ('b', 2) in items_view
@ -1300,8 +1258,7 @@ class Index(MutableMapping):
Comparison to another index or ordered dictionary is
order-sensitive. Comparison to all other mappings is order-insensitive.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> pairs = [('a', 1), ('b', 2), ('c', 3)]
>>> index.update(pairs)
>>> from collections import OrderedDict
@ -1335,8 +1292,7 @@ class Index(MutableMapping):
Comparison to another index or ordered dictionary is
order-sensitive. Comparison to all other mappings is order-insensitive.
>>> index = Index('/tmp/diskcache/index')
>>> index.clear()
>>> index = Index()
>>> index.update([('a', 1), ('b', 2), ('c', 3)])
>>> from collections import OrderedDict
>>> od = OrderedDict([('c', 3), ('b', 2), ('a', 1)])
@ -1371,8 +1327,8 @@ class Index(MutableMapping):
or for rewrapping the function with a different cache.
>>> from diskcache import Index
>>> mapping = Index('/tmp/diskcache/index')
>>> @mapping.memoize(typed=True)
>>> mapping = Index()
>>> @mapping.memoize()
... def fibonacci(number):
... if number == 0:
... return 0
@ -1380,8 +1336,15 @@ class Index(MutableMapping):
... return 1
... else:
... return fibonacci(number - 1) + fibonacci(number - 2)
>>> print(sum(fibonacci(number=value) for value in range(100)))
573147844013817084100
>>> print(fibonacci(100))
354224848179261915075
An additional `__cache_key__` attribute can be used to generate the
cache key used for the given arguments.
>>> key = fibonacci.__cache_key__(100)
>>> print(mapping[key])
354224848179261915075
Remember to call memoize when decorating a callable. If you forget,
then a TypeError will occur. Note the lack of parenthenses after
@ -1414,7 +1377,7 @@ class Index(MutableMapping):
Transactions may be nested and may not be shared between threads.
>>> from diskcache import Index
>>> mapping = Index('/tmp/diskcache/index')
>>> mapping = Index()
>>> with mapping.transact(): # Atomically increment two keys.
... mapping['total'] = mapping.get('total', 0) + 123.4
... mapping['count'] = mapping.get('count', 0) + 1
@ -1424,7 +1387,6 @@ class Index(MutableMapping):
123.4
:return: context manager for use in `with` statement
:raises Timeout: if database timeout occurs
"""
with self._cache.transact(retry=True):

324
lib/diskcache/recipes.py
View File

@ -1,12 +1,30 @@
"""Cache Recipes
"""Disk Cache Recipes
"""
import functools
import math
import os
import random
import sys
import threading
import time
from .core import ENOVAL, args_to_key, full_name
############################################################################
# BEGIN Python 2/3 Shims
############################################################################
if sys.hexversion < 0x03000000:
from thread import get_ident # pylint: disable=import-error
else:
from threading import get_ident
############################################################################
# END Python 2/3 Shims
############################################################################
class Averager(object):
"""Recipe for calculating a running average.
@ -15,45 +33,51 @@ class Averager(object):
total and count. The average can then be calculated at any time.
>>> import diskcache
>>> cache = diskcache.Cache('/tmp/diskcache/recipes')
>>> cache = diskcache.FanoutCache()
>>> ave = Averager(cache, 'latency')
>>> ave.add(0.080)
>>> ave.add(0.120)
>>> ave.get()
0.1
>>> ave.add(0.160)
>>> ave.get()
>>> ave.pop()
0.12
>>> print(ave.get())
None
"""
def __init__(self, cache, key):
def __init__(self, cache, key, expire=None, tag=None):
self._cache = cache
self._key = key
self._expire = expire
self._tag = tag
def add(self, value):
"Add `value` to average."
with self._cache.transact():
with self._cache.transact(retry=True):
total, count = self._cache.get(self._key, default=(0.0, 0))
total += value
count += 1
self._cache.set(self._key, (total, count))
self._cache.set(
self._key, (total, count), expire=self._expire, tag=self._tag,
)
def get(self):
"Get current average."
"Get current average or return `None` if count equals zero."
total, count = self._cache.get(self._key, default=(0.0, 0), retry=True)
return 0.0 if count == 0 else total / count
return None if count == 0 else total / count
def pop(self):
"Return current average and reset average to 0.0."
"Return current average and delete key."
total, count = self._cache.pop(self._key, default=(0.0, 0), retry=True)
return 0.0 if count == 0 else total / count
return None if count == 0 else total / count
class Lock(object):
"""Recipe for cross-process and cross-thread lock.
>>> import diskcache
>>> cache = diskcache.Cache('/tmp/diskcache/recipes')
>>> cache = diskcache.Cache()
>>> lock = Lock(cache, 'report-123')
>>> lock.acquire()
>>> lock.release()
@ -61,15 +85,20 @@ class Lock(object):
... pass
"""
def __init__(self, cache, key):
def __init__(self, cache, key, expire=None, tag=None):
self._cache = cache
self._key = key
self._expire = expire
self._tag = tag
def acquire(self):
"Acquire lock using spin-lock algorithm."
while True:
if self._cache.add(self._key, None, retry=True):
return
added = self._cache.add(
self._key, None, expire=self._expire, tag=self._tag, retry=True,
)
if added:
break
time.sleep(0.001)
def release(self):
@ -87,44 +116,57 @@ class RLock(object):
"""Recipe for cross-process and cross-thread re-entrant lock.
>>> import diskcache
>>> cache = diskcache.Cache('/tmp/diskcache/recipes')
>>> cache = diskcache.Cache()
>>> rlock = RLock(cache, 'user-123')
>>> rlock.acquire()
>>> rlock.acquire()
>>> rlock.release()
>>> rlock.release()
>>> with rlock:
... pass
>>> rlock.release()
>>> rlock.release()
Traceback (most recent call last):
...
AssertionError: cannot release un-acquired lock
"""
def __init__(self, cache, key):
def __init__(self, cache, key, expire=None, tag=None):
self._cache = cache
self._key = key
pid = os.getpid()
tid = threading.get_ident()
self._value = '{}-{}'.format(pid, tid)
self._expire = expire
self._tag = tag
def acquire(self):
"Acquire lock by incrementing count using spin-lock algorithm."
pid = os.getpid()
tid = get_ident()
pid_tid = '{}-{}'.format(pid, tid)
while True:
with self._cache.transact():
with self._cache.transact(retry=True):
value, count = self._cache.get(self._key, default=(None, 0))
if self._value == value or count == 0:
self._cache.set(self._key, (self._value, count + 1))
if pid_tid == value or count == 0:
self._cache.set(
self._key, (pid_tid, count + 1),
expire=self._expire, tag=self._tag,
)
return
time.sleep(0.001)
def release(self):
"Release lock by decrementing count."
with self._cache.transact():
pid = os.getpid()
tid = get_ident()
pid_tid = '{}-{}'.format(pid, tid)
with self._cache.transact(retry=True):
value, count = self._cache.get(self._key, default=(None, 0))
is_owned = self._value == value and count > 0
is_owned = pid_tid == value and count > 0
assert is_owned, 'cannot release un-acquired lock'
self._cache.set(self._key, (value, count - 1))
self._cache.set(
self._key, (value, count - 1),
expire=self._expire, tag=self._tag,
)
def __enter__(self):
self.acquire()
@ -137,8 +179,8 @@ class BoundedSemaphore(object):
"""Recipe for cross-process and cross-thread bounded semaphore.
>>> import diskcache
>>> cache = diskcache.Cache('/tmp/diskcache/recipes')
>>> semaphore = BoundedSemaphore(cache, 'max-connections', value=2)
>>> cache = diskcache.Cache()
>>> semaphore = BoundedSemaphore(cache, 'max-cons', value=2)
>>> semaphore.acquire()
>>> semaphore.acquire()
>>> semaphore.release()
@ -151,31 +193,245 @@ class BoundedSemaphore(object):
AssertionError: cannot release un-acquired semaphore
"""
def __init__(self, cache, key, value=1):
def __init__(self, cache, key, value=1, expire=None, tag=None):
self._cache = cache
self._key = key
self._value = value
self._expire = expire
self._tag = tag
def acquire(self):
"Acquire semaphore by decrementing value using spin-lock algorithm."
while True:
with self._cache.transact():
with self._cache.transact(retry=True):
value = self._cache.get(self._key, default=self._value)
if value > 0:
self._cache.set(self._key, value - 1)
self._cache.set(
self._key, value - 1,
expire=self._expire, tag=self._tag,
)
return
time.sleep(0.001)
def release(self):
"Release semaphore by incrementing value."
with self._cache.transact():
with self._cache.transact(retry=True):
value = self._cache.get(self._key, default=self._value)
assert self._value > value, 'cannot release un-acquired semaphore'
value += 1
self._cache.set(self._key, value)
self._cache.set(
self._key, value, expire=self._expire, tag=self._tag,
)
def __enter__(self):
self.acquire()
def __exit__(self, *exc_info):
self.release()
def throttle(cache, count, seconds, name=None, expire=None, tag=None,
time_func=time.time, sleep_func=time.sleep):
"""Decorator to throttle calls to function.
>>> import diskcache, time
>>> cache = diskcache.Cache()
>>> count = 0
>>> @throttle(cache, 2, 1) # 2 calls per 1 second
... def increment():
... global count
... count += 1
>>> start = time.time()
>>> while (time.time() - start) <= 2:
... increment()
>>> count in (6, 7) # 6 or 7 calls depending on CPU load
True
"""
def decorator(func):
rate = count / float(seconds)
key = full_name(func) if name is None else name
now = time_func()
cache.set(key, (now, count), expire=expire, tag=tag, retry=True)
@functools.wraps(func)
def wrapper(*args, **kwargs):
while True:
with cache.transact(retry=True):
last, tally = cache.get(key)
now = time_func()
tally += (now - last) * rate
delay = 0
if tally > count:
cache.set(key, (now, count - 1), expire)
elif tally >= 1:
cache.set(key, (now, tally - 1), expire)
else:
delay = (1 - tally) / rate
if delay:
sleep_func(delay)
else:
break
return func(*args, **kwargs)
return wrapper
return decorator
def barrier(cache, lock_factory, name=None, expire=None, tag=None):
"""Barrier to calling decorated function.
Supports different kinds of locks: Lock, RLock, BoundedSemaphore.
>>> import diskcache, time
>>> cache = diskcache.Cache()
>>> @barrier(cache, Lock)
... def work(num):
... print('worker started')
... time.sleep(1)
... print('worker finished')
>>> import multiprocessing.pool
>>> pool = multiprocessing.pool.ThreadPool(2)
>>> _ = pool.map(work, range(2))
worker started
worker finished
worker started
worker finished
>>> pool.terminate()
"""
def decorator(func):
key = full_name(func) if name is None else name
lock = lock_factory(cache, key, expire=expire, tag=tag)
@functools.wraps(func)
def wrapper(*args, **kwargs):
with lock:
return func(*args, **kwargs)
return wrapper
return decorator
def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1):
"""Memoizing cache decorator with cache stampede protection.
Cache stampedes are a type of system overload that can occur when parallel
computing systems using memoization come under heavy load. This behaviour
is sometimes also called dog-piling, cache miss storm, cache choking, or
the thundering herd problem.
The memoization decorator implements cache stampede protection through
early recomputation. Early recomputation of function results will occur
probabilistically before expiration in a background thread of
execution. Early probabilistic recomputation is based on research by
Vattani, A.; Chierichetti, F.; Lowenstein, K. (2015), Optimal Probabilistic
Cache Stampede Prevention, VLDB, pp. 886-897, ISSN 2150-8097
If name is set to None (default), the callable name will be determined
automatically.
If typed is set to True, function arguments of different types will be
cached separately. For example, f(3) and f(3.0) will be treated as distinct
calls with distinct results.
The original underlying function is accessible through the `__wrapped__`
attribute. This is useful for introspection, for bypassing the cache, or
for rewrapping the function with a different cache.
>>> from diskcache import Cache
>>> cache = Cache()
>>> @memoize_stampede(cache, expire=1)
... def fib(number):
... if number == 0:
... return 0
... elif number == 1:
... return 1
... else:
... return fib(number - 1) + fib(number - 2)
>>> print(fib(100))
354224848179261915075
An additional `__cache_key__` attribute can be used to generate the cache
key used for the given arguments.
>>> key = fib.__cache_key__(100)
>>> del cache[key]
Remember to call memoize when decorating a callable. If you forget, then a
TypeError will occur.
:param cache: cache to store callable arguments and return values
:param float expire: seconds until arguments expire
:param str name: name given for callable (default None, automatic)
:param bool typed: cache different types separately (default False)
:param str tag: text to associate with arguments (default None)
:return: callable decorator
"""
# Caution: Nearly identical code exists in Cache.memoize
def decorator(func):
"Decorator created by memoize call for callable."
base = (full_name(func),) if name is None else (name,)
def timer(*args, **kwargs):
"Time execution of `func` and return result and time delta."
start = time.time()
result = func(*args, **kwargs)
delta = time.time() - start
return result, delta
@functools.wraps(func)
def wrapper(*args, **kwargs):
"Wrapper for callable to cache arguments and return values."
key = wrapper.__cache_key__(*args, **kwargs)
pair, expire_time = cache.get(
key, default=ENOVAL, expire_time=True, retry=True,
)
if pair is not ENOVAL:
result, delta = pair
now = time.time()
ttl = expire_time - now
if (-delta * beta * math.log(random.random())) < ttl:
return result # Cache hit.
# Check whether a thread has started for early recomputation.
thread_key = key + (ENOVAL,)
thread_added = cache.add(
thread_key, None, expire=delta, retry=True,
)
if thread_added:
# Start thread for early recomputation.
def recompute():
with cache:
pair = timer(*args, **kwargs)
cache.set(
key, pair, expire=expire, tag=tag, retry=True,
)
thread = threading.Thread(target=recompute)
thread.daemon = True
thread.start()
return result
pair = timer(*args, **kwargs)
cache.set(key, pair, expire=expire, tag=tag, retry=True)
return pair[0]
def __cache_key__(*args, **kwargs):
"Make key for cache given function arguments."
return args_to_key(base, args, kwargs, typed)
wrapper.__cache_key__ = __cache_key__
return wrapper
return decorator

78
lib/diskcache/stampede.py
View File

@ -1,78 +0,0 @@
"Stampede barrier implementation."
import functools as ft
import math
import random
import tempfile
import time
from .core import Cache, ENOVAL
class StampedeBarrier(object):
"""Stampede barrier mitigates cache stampedes.
Cache stampedes are also known as dog-piling, cache miss storm, cache
choking, or the thundering herd problem.
Based on research by Vattani, A.; Chierichetti, F.; Lowenstein, K. (2015),
Optimal Probabilistic Cache Stampede Prevention,
VLDB, pp. 886?897, ISSN 2150-8097
Example:
```python
stampede_barrier = StampedeBarrier('/tmp/user_data', expire=3)
@stampede_barrier
def load_user_info(user_id):
return database.lookup_user_info_by_id(user_id)
```
"""
# pylint: disable=too-few-public-methods
def __init__(self, cache=None, expire=None):
if isinstance(cache, Cache):
pass
elif cache is None:
cache = Cache(tempfile.mkdtemp())
else:
cache = Cache(cache)
self._cache = cache
self._expire = expire
def __call__(self, func):
cache = self._cache
expire = self._expire
@ft.wraps(func)
def wrapper(*args, **kwargs):
"Wrapper function to cache function result."
key = (args, kwargs)
try:
result, expire_time, delta = cache.get(
key, default=ENOVAL, expire_time=True, tag=True
)
if result is ENOVAL:
raise KeyError
now = time.time()
ttl = expire_time - now
if (-delta * math.log(random.random())) < ttl:
return result
except KeyError:
pass
now = time.time()
result = func(*args, **kwargs)
delta = time.time() - now
cache.set(key, result, expire=expire, tag=delta)
return result
return wrapper
Write Preview
Loading…
Cancel
Save