Added cache for rss feed items.

Fixed issues with searches

lib/feedcache/__init__.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+#
+# Copyright 2007 Doug Hellmann.
+#
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Doug
+# Hellmann not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+
+"""
+
+"""
+
+__module_id__ = "$Id$"
+
+#
+# Import system modules
+#
+
+
+#
+# Import local modules
+#
+from cache import Cache
+
+#
+# Module
+#

lib/feedcache/cache.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python
+#
+# Copyright 2007 Doug Hellmann.
+#
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Doug
+# Hellmann not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+
+"""
+
+"""
+
+__module_id__ = "$Id$"
+
+#
+# Import system modules
+#
+from feedparser import feedparser
+
+import logging
+import time
+
+#
+# Import local modules
+#
+
+
+#
+# Module
+#
+
+logger = logging.getLogger('feedcache.cache')
+
+
+class Cache:
+    """A class to wrap Mark Pilgrim's Universal Feed Parser module
+    (http://www.feedparser.org) so that parameters can be used to
+    cache the feed results locally instead of fetching the feed every
+    time it is requested. Uses both etag and modified times for
+    caching.
+    """
+
+    def __init__(self, storage, timeToLiveSeconds=300, userAgent='feedcache'):
+        """
+        Arguments:
+
+          storage -- Backing store for the cache.  It should follow
+          the dictionary API, with URLs used as keys.  It should
+          persist data.
+
+          timeToLiveSeconds=300 -- The length of time content should
+          live in the cache before an update is attempted.
+
+          userAgent='feedcache' -- User agent string to be used when
+          fetching feed contents.
+
+        """
+        self.storage = storage
+        self.time_to_live = timeToLiveSeconds
+        self.user_agent = userAgent
+        return
+
+    def purge(self, olderThanSeconds):
+        """Remove cached data from the storage if the data is older than the
+        date given.  If olderThanSeconds is None, the entire cache is purged.
+        """
+        if olderThanSeconds is None:
+            logger.debug('purging the entire cache')
+            for key in self.storage.keys():
+                del self.storage[key]
+        else:
+            now = time.time()
+            # Iterate over the keys and load each item one at a time
+            # to avoid having the entire cache loaded into memory
+            # at one time.
+            for url in self.storage.keys():
+                (cached_time, cached_data) = self.storage[url]
+                age = now - cached_time
+                if age >= olderThanSeconds:
+                    logger.debug('removing %s with age %d', url, age)
+                    del self.storage[url]
+        return
+
+    def fetch(self, url, force_update=False, offline=False):
+        """Return the feed at url.
+
+        url - The URL of the feed.
+
+        force_update=False - When True, update the cache whether the
+                                           current contents have
+                                           exceeded their time-to-live
+                                           or not.
+
+        offline=False - When True, only return data from the local
+                                 cache and never access the remote
+                                 URL.
+
+        If there is data for that feed in the cache already, check
+        the expiration date before accessing the server.  If the
+        cached data has not expired, return it without accessing the
+        server.
+
+        In cases where the server is accessed, check for updates
+        before deciding what to return.  If the server reports a
+        status of 304, the previously cached content is returned.
+
+        The cache is only updated if the server returns a status of
+        200, to avoid holding redirected data in the cache.
+        """
+        logger.debug('url="%s"' % url)
+
+        # Convert the URL to a value we can use
+        # as a key for the storage backend.
+        key = url
+        if isinstance(key, unicode):
+            key = key.encode('utf-8')
+
+        modified = None
+        etag = None
+        now = time.time()
+
+        cached_time, cached_content = self.storage.get(key, (None, None))
+
+        # Offline mode support (no networked requests)
+        # so return whatever we found in the storage.
+        # If there is nothing in the storage, we'll be returning None.
+        if offline:
+            logger.debug('offline mode')
+            return cached_content
+
+        # Does the storage contain a version of the data
+        # which is older than the time-to-live?
+        logger.debug('cache modified time: %s' % str(cached_time))
+        if cached_time is not None and not force_update:
+            if self.time_to_live:
+                age = now - cached_time
+                if age <= self.time_to_live:
+                    logger.debug('cache contents still valid')
+                    return cached_content
+                else:
+                    logger.debug('cache contents older than TTL')
+            else:
+                logger.debug('no TTL value')
+
+            # The cache is out of date, but we have
+            # something.  Try to use the etag and modified_time
+            # values from the cached content.
+            etag = cached_content.get('etag')
+            modified = cached_content.get('modified')
+            logger.debug('cached etag=%s' % etag)
+            logger.debug('cached modified=%s' % str(modified))
+        else:
+            logger.debug('nothing in the cache, or forcing update')
+
+        # We know we need to fetch, so go ahead and do it.
+        logger.debug('fetching...')
+        parsed_result = feedparser.parse(url,
+                                         agent=self.user_agent,
+                                         modified=modified,
+                                         etag=etag,
+                                         )
+
+        status = parsed_result.get('status', None)
+        logger.debug('HTTP status=%s' % status)
+        if status == 304:
+            # No new data, based on the etag or modified values.
+            # We need to update the modified time in the
+            # storage, though, so we know that what we have
+            # stored is up to date.
+            self.storage[key] = (now, cached_content)
+
+            # Return the data from the cache, since
+            # the parsed data will be empty.
+            parsed_result = cached_content
+        elif status == 200:
+            # There is new content, so store it unless there was an error.
+            error = parsed_result.get('bozo_exception')
+            if not error:
+                logger.debug('Updating stored data for %s' % url)
+                self.storage[key] = (now, parsed_result)
+            else:
+                logger.warning('Not storing data with exception: %s',
+                               error)
+        else:
+            logger.warning('Not updating cache with HTTP status %s', status)
+
+        return parsed_result

lib/feedcache/cachestoragelock.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+#
+# Copyright 2007 Doug Hellmann.
+#
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Doug
+# Hellmann not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+from __future__ import with_statement
+
+"""Lock wrapper for cache storage which do not permit multi-threaded access.
+
+"""
+
+__module_id__ = "$Id$"
+
+#
+# Import system modules
+#
+import threading
+
+#
+# Import local modules
+#
+
+
+#
+# Module
+#
+
+class CacheStorageLock:
+    """Lock wrapper for cache storage which do not permit multi-threaded access.
+    """
+
+    def __init__(self, shelf):
+        self.lock = threading.Lock()
+        self.shelf = shelf
+        return
+
+    def __getitem__(self, key):
+        with self.lock:
+            return self.shelf[key]
+
+    def get(self, key, default=None):
+        with self.lock:
+            try:
+                return self.shelf[key]
+            except KeyError:
+                return default
+
+    def __setitem__(self, key, value):
+        with self.lock:
+            self.shelf[key] = value

lib/feedcache/example.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+#
+# Copyright 2007 Doug Hellmann.
+#
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Doug
+# Hellmann not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+
+"""Example use of feedcache.Cache.
+
+"""
+
+__module_id__ = "$Id$"
+
+#
+# Import system modules
+#
+import sys
+import shelve
+
+#
+# Import local modules
+#
+import cache
+
+#
+# Module
+#
+
+def main(urls=[]):
+    print 'Saving feed data to ./.feedcache'
+    storage = shelve.open('.feedcache')
+    try:
+        fc = cache.Cache(storage)
+        for url in urls:
+            parsed_data = fc.fetch(url)
+            print parsed_data.feed.title
+            for entry in parsed_data.entries:
+                print '\t', entry.title
+    finally:
+        storage.close()
+    return
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
+

lib/feedcache/example_threads.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+#
+# Copyright 2007 Doug Hellmann.
+#
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Doug
+# Hellmann not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+
+"""Example use of feedcache.Cache combined with threads.
+
+"""
+
+__module_id__ = "$Id$"
+
+#
+# Import system modules
+#
+import Queue
+import sys
+import shove
+import threading
+
+#
+# Import local modules
+#
+import cache
+
+#
+# Module
+#
+
+MAX_THREADS=5
+OUTPUT_DIR='/tmp/feedcache_example'
+
+
+def main(urls=[]):
+
+    if not urls:
+        print 'Specify the URLs to a few RSS or Atom feeds on the command line.'
+        return
+
+    # Decide how many threads to start
+    num_threads = min(len(urls), MAX_THREADS)
+
+    # Add the URLs to a queue
+    url_queue = Queue.Queue()
+    for url in urls:
+        url_queue.put(url)
+
+    # Add poison pills to the url queue to cause
+    # the worker threads to break out of their loops
+    for i in range(num_threads):
+        url_queue.put(None)
+
+    # Track the entries in the feeds being fetched
+    entry_queue = Queue.Queue()
+
+    print 'Saving feed data to', OUTPUT_DIR
+    storage = shove.Shove('file://' + OUTPUT_DIR)
+    try:
+
+        # Start a few worker threads
+        worker_threads = []
+        for i in range(num_threads):
+            t = threading.Thread(target=fetch_urls, 
+                                 args=(storage, url_queue, entry_queue,))
+            worker_threads.append(t)
+            t.setDaemon(True)
+            t.start()
+
+        # Start a thread to print the results
+        printer_thread = threading.Thread(target=print_entries, args=(entry_queue,))
+        printer_thread.setDaemon(True)
+        printer_thread.start()
+
+        # Wait for all of the URLs to be processed
+        url_queue.join()
+
+        # Wait for the worker threads to finish
+        for t in worker_threads:
+            t.join()
+
+        # Poison the print thread and wait for it to exit
+        entry_queue.put((None,None))
+        entry_queue.join()
+        printer_thread.join()        
+        
+    finally:
+        storage.close()
+    return
+
+
+def fetch_urls(storage, input_queue, output_queue):
+    """Thread target for fetching feed data.
+    """
+    c = cache.Cache(storage)
+
+    while True:
+        next_url = input_queue.get()
+        if next_url is None: # None causes thread to exit
+            input_queue.task_done()
+            break
+
+        feed_data = c.fetch(next_url)
+        for entry in feed_data.entries:
+            output_queue.put( (feed_data.feed, entry) )
+        input_queue.task_done()
+    return
+
+
+def print_entries(input_queue):
+    """Thread target for printing the contents of the feeds.
+    """
+    while True:
+        feed, entry = input_queue.get()
+        if feed is None: # None causes thread to exist
+            input_queue.task_done()
+            break
+
+        print '%s: %s' % (feed.title, entry.title)
+        input_queue.task_done()
+    return
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
+

lib/feedcache/test_cache.py

@@ -0,0 +1,323 @@
+#!/usr/bin/env python
+#
+# Copyright 2007 Doug Hellmann.
+#
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Doug
+# Hellmann not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+
+"""Unittests for feedcache.cache
+
+"""
+
+__module_id__ = "$Id$"
+
+import logging
+logging.basicConfig(level=logging.DEBUG,
+                    format='%(asctime)s %(levelname)-8s %(name)s %(message)s',
+                    )
+logger = logging.getLogger('feedcache.test_cache')
+
+#
+# Import system modules
+#
+import copy
+import time
+import unittest
+import UserDict
+
+#
+# Import local modules
+#
+import cache
+from test_server import HTTPTestBase, TestHTTPServer
+
+#
+# Module
+#
+
+
+class CacheTestBase(HTTPTestBase):
+
+    CACHE_TTL = 30
+
+    def setUp(self):
+        HTTPTestBase.setUp(self)
+
+        self.storage = self.getStorage()
+        self.cache = cache.Cache(self.storage,
+                                 timeToLiveSeconds=self.CACHE_TTL,
+                                 userAgent='feedcache.test',
+                                 )
+        return
+
+    def getStorage(self):
+        "Return a cache storage for the test."
+        return {}
+
+
+class CacheTest(CacheTestBase):
+
+    CACHE_TTL = 30
+
+    def getServer(self):
+        "These tests do not want to use the ETag or If-Modified-Since headers"
+        return TestHTTPServer(applyModifiedHeaders=False)
+
+    def testRetrieveNotInCache(self):
+        # Retrieve data not already in the cache.
+        feed_data = self.cache.fetch(self.TEST_URL)
+        self.failUnless(feed_data)
+        self.failUnlessEqual(feed_data.feed.title, 'CacheTest test data')
+        return
+
+    def testRetrieveIsInCache(self):
+        # Retrieve data which is alread in the cache,
+        # and verify that the second copy is identitical
+        # to the first.
+
+        # First fetch
+        feed_data = self.cache.fetch(self.TEST_URL)
+
+        # Second fetch
+        feed_data2 = self.cache.fetch(self.TEST_URL)
+
+        # Since it is the in-memory storage, we should have the
+        # exact same object.
+        self.failUnless(feed_data is feed_data2)
+        return
+
+    def testExpireDataInCache(self):
+        # Retrieve data which is in the cache but which
+        # has expired and verify that the second copy
+        # is different from the first.
+
+        # First fetch
+        feed_data = self.cache.fetch(self.TEST_URL)
+
+        # Change the timeout and sleep to move the clock
+        self.cache.time_to_live = 0
+        time.sleep(1)
+
+        # Second fetch
+        feed_data2 = self.cache.fetch(self.TEST_URL)
+
+        # Since we reparsed, the cache response should be different.
+        self.failIf(feed_data is feed_data2)
+        return
+
+    def testForceUpdate(self):
+        # Force cache to retrieve data which is alread in the cache,
+        # and verify that the new data is different.
+
+        # Pre-populate the storage with bad data
+        self.cache.storage[self.TEST_URL] = (time.time() + 100, self.id())
+
+        # Fetch the data
+        feed_data = self.cache.fetch(self.TEST_URL, force_update=True)
+
+        self.failIfEqual(feed_data, self.id())
+        return
+
+    def testOfflineMode(self):
+        # Retrieve data which is alread in the cache,
+        # whether it is expired or not.
+
+        # Pre-populate the storage with data
+        self.cache.storage[self.TEST_URL] = (0, self.id())
+
+        # Fetch it
+        feed_data = self.cache.fetch(self.TEST_URL, offline=True)
+
+        self.failUnlessEqual(feed_data, self.id())
+        return
+
+    def testUnicodeURL(self):
+        # Pass in a URL which is unicode
+
+        url = unicode(self.TEST_URL)
+        feed_data = self.cache.fetch(url)
+
+        storage = self.cache.storage
+        key = unicode(self.TEST_URL).encode('UTF-8')
+
+        # Verify that the storage has a key
+        self.failUnless(key in storage)
+
+        # Now pull the data from the storage directly
+        storage_timeout, storage_data = self.cache.storage.get(key)
+        self.failUnlessEqual(feed_data, storage_data)
+        return
+
+
+class SingleWriteMemoryStorage(UserDict.UserDict):
+    """Cache storage which only allows the cache value
+    for a URL to be updated one time.
+    """
+
+    def __setitem__(self, url, data):
+        if url in self.keys():
+            modified, existing = self[url]
+            # Allow the modified time to change,
+            # but not the feed content.
+            if data[1] != existing:
+                raise AssertionError('Trying to update cache for %s to %s' \
+                                         % (url, data))
+        UserDict.UserDict.__setitem__(self, url, data)
+        return
+
+
+class CacheConditionalGETTest(CacheTestBase):
+
+    CACHE_TTL = 0
+
+    def getStorage(self):
+        return SingleWriteMemoryStorage()
+
+    def testFetchOnceForEtag(self):
+        # Fetch data which has a valid ETag value, and verify
+        # that while we hit the server twice the response
+        # codes cause us to use the same data.
+
+        # First fetch populates the cache
+        response1 = self.cache.fetch(self.TEST_URL)
+        self.failUnlessEqual(response1.feed.title, 'CacheTest test data')
+
+        # Remove the modified setting from the cache so we know
+        # the next time we check the etag will be used
+        # to check for updates.  Since we are using an in-memory
+        # cache, modifying response1 updates the cache storage
+        # directly.
+        response1['modified'] = None
+
+        # This should result in a 304 status, and no data from
+        # the server.  That means the cache won't try to
+        # update the storage, so our SingleWriteMemoryStorage
+        # should not raise and we should have the same
+        # response object.
+        response2 = self.cache.fetch(self.TEST_URL)
+        self.failUnless(response1 is response2)
+
+        # Should have hit the server twice
+        self.failUnlessEqual(self.server.getNumRequests(), 2)
+        return
+
+    def testFetchOnceForModifiedTime(self):
+        # Fetch data which has a valid Last-Modified value, and verify
+        # that while we hit the server twice the response
+        # codes cause us to use the same data.
+
+        # First fetch populates the cache
+        response1 = self.cache.fetch(self.TEST_URL)
+        self.failUnlessEqual(response1.feed.title, 'CacheTest test data')
+
+        # Remove the etag setting from the cache so we know
+        # the next time we check the modified time will be used
+        # to check for updates.  Since we are using an in-memory
+        # cache, modifying response1 updates the cache storage
+        # directly.
+        response1['etag'] = None
+
+        # This should result in a 304 status, and no data from
+        # the server.  That means the cache won't try to
+        # update the storage, so our SingleWriteMemoryStorage
+        # should not raise and we should have the same
+        # response object.
+        response2 = self.cache.fetch(self.TEST_URL)
+        self.failUnless(response1 is response2)
+
+        # Should have hit the server twice
+        self.failUnlessEqual(self.server.getNumRequests(), 2)
+        return
+
+
+class CacheRedirectHandlingTest(CacheTestBase):
+
+    def _test(self, response):
+        # Set up the server to redirect requests,
+        # then verify that the cache is not updated
+        # for the original or new URL and that the
+        # redirect status is fed back to us with
+        # the fetched data.
+
+        self.server.setResponse(response, '/redirected')
+
+        response1 = self.cache.fetch(self.TEST_URL)
+
+        # The response should include the status code we set
+        self.failUnlessEqual(response1.get('status'), response)
+
+        # The response should include the new URL, too
+        self.failUnlessEqual(response1.href, self.TEST_URL + 'redirected')
+
+        # The response should not have been cached under either URL
+        self.failIf(self.TEST_URL in self.storage)
+        self.failIf(self.TEST_URL + 'redirected' in self.storage)
+        return
+
+    def test301(self):
+        self._test(301)
+
+    def test302(self):
+        self._test(302)
+
+    def test303(self):
+        self._test(303)
+
+    def test307(self):
+        self._test(307)
+
+
+class CachePurgeTest(CacheTestBase):
+
+    def testPurgeAll(self):
+        # Remove everything from the cache
+
+        self.cache.fetch(self.TEST_URL)
+        self.failUnless(self.storage.keys(),
+                        'Have no data in the cache storage')
+
+        self.cache.purge(None)
+
+        self.failIf(self.storage.keys(),
+                    'Still have data in the cache storage')
+        return
+
+    def testPurgeByAge(self):
+        # Remove old content from the cache
+
+        self.cache.fetch(self.TEST_URL)
+        self.failUnless(self.storage.keys(),
+                        'have no data in the cache storage')
+
+        time.sleep(1)
+
+        remains = (time.time(), copy.deepcopy(self.storage[self.TEST_URL][1]))
+        self.storage['http://this.should.remain/'] = remains
+
+        self.cache.purge(1)
+
+        self.failUnlessEqual(self.storage.keys(),
+                             ['http://this.should.remain/'])
+        return
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/feedcache/test_cachestoragelock.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+#
+# Copyright 2007 Doug Hellmann.
+#
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Doug
+# Hellmann not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+
+"""Tests for shelflock.
+
+"""
+
+__module_id__ = "$Id$"
+
+#
+# Import system modules
+#
+import os
+import shelve
+import tempfile
+import threading
+import unittest
+
+#
+# Import local modules
+#
+from cache import Cache
+from cachestoragelock import CacheStorageLock
+from test_server import HTTPTestBase
+
+#
+# Module
+#
+
+class CacheShelveTest(HTTPTestBase):
+
+    def setUp(self):
+        HTTPTestBase.setUp(self)
+        handle, self.shelve_filename = tempfile.mkstemp('.shelve')
+        os.close(handle) # we just want the file name, so close the open handle
+        os.unlink(self.shelve_filename) # remove the empty file
+        return
+
+    def tearDown(self):
+        try:
+            os.unlink(self.shelve_filename)
+        except AttributeError:
+            pass
+        HTTPTestBase.tearDown(self)
+        return
+
+    def test(self):
+        storage = shelve.open(self.shelve_filename)
+        locking_storage = CacheStorageLock(storage)
+        try:
+            fc = Cache(locking_storage)
+
+            # First fetch the data through the cache
+            parsed_data = fc.fetch(self.TEST_URL)
+            self.failUnlessEqual(parsed_data.feed.title, 'CacheTest test data')
+
+            # Now retrieve the same data directly from the shelf
+            modified, shelved_data = storage[self.TEST_URL]
+            
+            # The data should be the same
+            self.failUnlessEqual(parsed_data, shelved_data)
+        finally:
+            storage.close()
+        return
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/feedcache/test_server.py

@@ -0,0 +1,241 @@
+#!/usr/bin/env python
+#
+# Copyright 2007 Doug Hellmann.
+#
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Doug
+# Hellmann not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+
+"""Simple HTTP server for testing the feed cache.
+
+"""
+
+__module_id__ = "$Id$"
+
+#
+# Import system modules
+#
+import BaseHTTPServer
+import logging
+import md5
+import threading
+import time
+import unittest
+import urllib
+
+#
+# Import local modules
+#
+
+
+#
+# Module
+#
+logger = logging.getLogger('feedcache.test_server')
+
+
+def make_etag(data):
+    """Given a string containing data to be returned to the client,
+    compute an ETag value for the data.
+    """
+    _md5 = md5.new()
+    _md5.update(data)
+    return _md5.hexdigest()
+
+
+class TestHTTPHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+    "HTTP request handler which serves the same feed data every time."
+
+    FEED_DATA = """<?xml version="1.0" encoding="utf-8"?>
+
+<feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en-us">
+  <title>CacheTest test data</title>
+  <link href="http://localhost/feedcache/" rel="alternate"></link>
+  <link href="http://localhost/feedcache/atom/" rel="self"></link>
+  <id>http://localhost/feedcache/</id>
+  <updated>2006-10-14T11:00:36Z</updated>
+  <entry>
+    <title>single test entry</title>
+    <link href="http://www.example.com/" rel="alternate"></link>
+    <updated>2006-10-14T11:00:36Z</updated>
+    <author>
+      <name>author goes here</name>
+      <email>authoremail@example.com</email>
+    </author>
+    <id>http://www.example.com/</id>
+    <summary type="html">description goes here</summary>
+    <link length="100" href="http://www.example.com/enclosure" type="text/html" rel="enclosure">
+    </link>
+  </entry>
+</feed>"""
+
+    # The data does not change, so save the ETag and modified times
+    # as class attributes.
+    ETAG = make_etag(FEED_DATA)
+    # Calculated using email.utils.formatdate(usegmt=True)
+    MODIFIED_TIME = 'Sun, 08 Apr 2012 20:16:48 GMT'
+
+    def do_GET(self):
+        "Handle GET requests."
+        logger.debug('GET %s', self.path)
+
+        if self.path == '/shutdown':
+            # Shortcut to handle stopping the server
+            logger.debug('Stopping server')
+            self.server.stop()
+            self.send_response(200)
+
+        else:
+            # Record the request for tests that count them
+            self.server.requests.append(self.path)
+            # Process the request
+            logger.debug('pre-defined response code: %d', self.server.response)
+            handler_method_name = 'do_GET_%d' % self.server.response
+            handler_method = getattr(self, handler_method_name)
+            handler_method()
+        return
+
+    def do_GET_3xx(self):
+        "Handle redirects"
+        if self.path.endswith('/redirected'):
+            logger.debug('already redirected')
+            # We have already redirected, so return the data.
+            return self.do_GET_200()
+        new_path = self.server.new_path
+        logger.debug('redirecting to %s', new_path)
+        self.send_response(self.server.response)
+        self.send_header('Location', new_path)
+        return
+
+    do_GET_301 = do_GET_3xx
+    do_GET_302 = do_GET_3xx
+    do_GET_303 = do_GET_3xx
+    do_GET_307 = do_GET_3xx
+
+    def do_GET_200(self):
+        logger.debug('Etag: %s' % self.ETAG)
+        logger.debug('Last-Modified: %s' % self.MODIFIED_TIME)
+
+        incoming_etag = self.headers.get('If-None-Match', None)
+        logger.debug('Incoming ETag: "%s"' % incoming_etag)
+
+        incoming_modified = self.headers.get('If-Modified-Since', None)
+        logger.debug('Incoming If-Modified-Since: %s' % incoming_modified)
+
+        send_data = True
+
+        # Does the client have the same version of the data we have?
+        if self.server.apply_modified_headers:
+            if incoming_etag == self.ETAG:
+                logger.debug('Response 304, etag')
+                self.send_response(304)
+                send_data = False
+
+            elif incoming_modified == self.MODIFIED_TIME:
+                logger.debug('Response 304, modified time')
+                self.send_response(304)
+                send_data = False
+
+        # Now optionally send the data, if the client needs it
+        if send_data:
+            logger.debug('Response 200')
+            self.send_response(200)
+
+            self.send_header('Content-Type', 'application/atom+xml')
+
+            logger.debug('Outgoing Etag: %s' % self.ETAG)
+            self.send_header('ETag', self.ETAG)
+
+            logger.debug('Outgoing modified time: %s' % self.MODIFIED_TIME)
+            self.send_header('Last-Modified', self.MODIFIED_TIME)
+
+            self.end_headers()
+
+            logger.debug('Sending data')
+            self.wfile.write(self.FEED_DATA)
+        return
+
+
+class TestHTTPServer(BaseHTTPServer.HTTPServer):
+    """HTTP Server which counts the number of requests made
+    and can stop based on client instructions.
+    """
+
+    def __init__(self, applyModifiedHeaders=True, handler=TestHTTPHandler):
+        self.apply_modified_headers = applyModifiedHeaders
+        self.keep_serving = True
+        self.requests = []
+        self.setResponse(200)
+        BaseHTTPServer.HTTPServer.__init__(self, ('', 9999), handler)
+        return
+
+    def setResponse(self, newResponse, newPath=None):
+        """Sets the response code to use for future requests, and a new
+        path to be used as a redirect target, if necessary.
+        """
+        self.response = newResponse
+        self.new_path = newPath
+        return
+
+    def getNumRequests(self):
+        "Return the number of requests which have been made on the server."
+        return len(self.requests)
+
+    def stop(self):
+        "Stop serving requests, after the next request."
+        self.keep_serving = False
+        return
+
+    def serve_forever(self):
+        "Main loop for server"
+        while self.keep_serving:
+            self.handle_request()
+        logger.debug('exiting')
+        return
+
+
+class HTTPTestBase(unittest.TestCase):
+    "Base class for tests that use a TestHTTPServer"
+
+    TEST_URL = 'http://localhost:9999/'
+
+    CACHE_TTL = 0
+
+    def setUp(self):
+        self.server = self.getServer()
+        self.server_thread = threading.Thread(target=self.server.serve_forever)
+        # set daemon flag so the tests don't hang if cleanup fails
+        self.server_thread.setDaemon(True)
+        self.server_thread.start()
+        return
+
+    def getServer(self):
+        "Return a web server for the test."
+        s = TestHTTPServer()
+        s.setResponse(200)
+        return s
+
+    def tearDown(self):
+        # Stop the server thread
+        urllib.urlretrieve(self.TEST_URL + 'shutdown')
+        time.sleep(1)
+        self.server.server_close()
+        self.server_thread.join()
+        return

lib/feedcache/test_shovefilesystem.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+#
+# Copyright 2007 Doug Hellmann.
+#
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Doug
+# Hellmann not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+
+"""Tests with shove filesystem storage.
+
+"""
+
+__module_id__ = "$Id$"
+
+#
+# Import system modules
+#
+import os
+import shove
+import tempfile
+import threading
+import unittest
+
+#
+# Import local modules
+#
+from cache import Cache
+from test_server import HTTPTestBase
+
+#
+# Module
+#
+
+class CacheShoveTest(HTTPTestBase):
+
+    def setUp(self):
+        HTTPTestBase.setUp(self)
+        self.shove_dirname = tempfile.mkdtemp('shove')
+        return
+
+    def tearDown(self):
+        try:
+            os.system('rm -rf %s' % self.storage_dirname)
+        except AttributeError:
+            pass
+        HTTPTestBase.tearDown(self)
+        return
+
+    def test(self):
+        # First fetch the data through the cache
+        storage = shove.Shove('file://' + self.shove_dirname)
+        try:
+            fc = Cache(storage)
+            parsed_data = fc.fetch(self.TEST_URL)
+            self.failUnlessEqual(parsed_data.feed.title, 'CacheTest test data')
+        finally:
+            storage.close()
+
+        # Now retrieve the same data directly from the shelf
+        storage = shove.Shove('file://' + self.shove_dirname)
+        try:
+            modified, shelved_data = storage[self.TEST_URL]
+        finally:
+            storage.close()
+            
+        # The data should be the same
+        self.failUnlessEqual(parsed_data, shelved_data)
+        return
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/__init__.py

@@ -0,0 +1,519 @@
+# -*- coding: utf-8 -*-
+'''Common object storage frontend.'''
+
+import os
+import zlib
+import urllib
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+from collections import deque
+
+try:
+    # Import store and cache entry points if setuptools installed
+    import pkg_resources
+    stores = dict((_store.name, _store) for _store in
+        pkg_resources.iter_entry_points('shove.stores'))
+    caches = dict((_cache.name, _cache) for _cache in
+        pkg_resources.iter_entry_points('shove.caches'))
+    # Pass if nothing loaded
+    if not stores and not caches:
+        raise ImportError()
+except ImportError:
+    # Static store backend registry
+    stores = dict(
+        bsddb='shove.store.bsdb:BsdStore',
+        cassandra='shove.store.cassandra:CassandraStore',
+        dbm='shove.store.dbm:DbmStore',
+        durus='shove.store.durusdb:DurusStore',
+        file='shove.store.file:FileStore',
+        firebird='shove.store.db:DbStore',
+        ftp='shove.store.ftp:FtpStore',
+        hdf5='shove.store.hdf5:HDF5Store',
+        leveldb='shove.store.leveldbstore:LevelDBStore',
+        memory='shove.store.memory:MemoryStore',
+        mssql='shove.store.db:DbStore',
+        mysql='shove.store.db:DbStore',
+        oracle='shove.store.db:DbStore',
+        postgres='shove.store.db:DbStore',
+        redis='shove.store.redisdb:RedisStore',
+        s3='shove.store.s3:S3Store',
+        simple='shove.store.simple:SimpleStore',
+        sqlite='shove.store.db:DbStore',
+        svn='shove.store.svn:SvnStore',
+        zodb='shove.store.zodb:ZodbStore',
+    )
+    # Static cache backend registry
+    caches = dict(
+        bsddb='shove.cache.bsdb:BsdCache',
+        file='shove.cache.file:FileCache',
+        filelru='shove.cache.filelru:FileLRUCache',
+        firebird='shove.cache.db:DbCache',
+        memcache='shove.cache.memcached:MemCached',
+        memlru='shove.cache.memlru:MemoryLRUCache',
+        memory='shove.cache.memory:MemoryCache',
+        mssql='shove.cache.db:DbCache',
+        mysql='shove.cache.db:DbCache',
+        oracle='shove.cache.db:DbCache',
+        postgres='shove.cache.db:DbCache',
+        redis='shove.cache.redisdb:RedisCache',
+        simple='shove.cache.simple:SimpleCache',
+        simplelru='shove.cache.simplelru:SimpleLRUCache',
+        sqlite='shove.cache.db:DbCache',
+    )
+
+
+def getbackend(uri, engines, **kw):
+    '''
+    Loads the right backend based on a URI.
+
+    @param uri Instance or name string
+    @param engines A dictionary of scheme/class pairs
+    '''
+    if isinstance(uri, basestring):
+        mod = engines[uri.split('://', 1)[0]]
+        # Load module if setuptools not present
+        if isinstance(mod, basestring):
+            # Isolate classname from dot path
+            module, klass = mod.split(':')
+            # Load module
+            mod = getattr(__import__(module, '', '', ['']), klass)
+        # Load appropriate class from setuptools entry point
+        else:
+            mod = mod.load()
+        # Return instance
+        return mod(uri, **kw)
+    # No-op for existing instances
+    return uri
+
+
+def synchronized(func):
+    '''
+    Decorator to lock and unlock a method (Phillip J. Eby).
+
+    @param func Method to decorate
+    '''
+    def wrapper(self, *__args, **__kw):
+        self._lock.acquire()
+        try:
+            return func(self, *__args, **__kw)
+        finally:
+            self._lock.release()
+    wrapper.__name__ = func.__name__
+    wrapper.__dict__ = func.__dict__
+    wrapper.__doc__ = func.__doc__
+    return wrapper
+
+
+class Base(object):
+
+    '''Base Mapping class.'''
+
+    def __init__(self, engine, **kw):
+        '''
+        @keyword compress True, False, or an integer compression level (1-9).
+        '''
+        self._compress = kw.get('compress', False)
+        self._protocol = kw.get('protocol', pickle.HIGHEST_PROTOCOL)
+
+    def __getitem__(self, key):
+        raise NotImplementedError()
+
+    def __setitem__(self, key, value):
+        raise NotImplementedError()
+
+    def __delitem__(self, key):
+        raise NotImplementedError()
+
+    def __contains__(self, key):
+        try:
+            value = self[key]
+        except KeyError:
+            return False
+        return True
+
+    def get(self, key, default=None):
+        '''
+        Fetch a given key from the mapping. If the key does not exist,
+        return the default.
+
+        @param key Keyword of item in mapping.
+        @param default Default value (default: None)
+        '''
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def dumps(self, value):
+        '''Optionally serializes and compresses an object.'''
+        # Serialize everything but ASCII strings
+        value = pickle.dumps(value, protocol=self._protocol)
+        if self._compress:
+            level = 9 if self._compress is True else self._compress
+            value = zlib.compress(value, level)
+        return value
+
+    def loads(self, value):
+        '''Deserializes and optionally decompresses an object.'''
+        if self._compress:
+            try:
+                value = zlib.decompress(value)
+            except zlib.error:
+                pass
+        value = pickle.loads(value)
+        return value
+
+
+class BaseStore(Base):
+
+    '''Base Store class (based on UserDict.DictMixin).'''
+
+    def __init__(self, engine, **kw):
+        super(BaseStore, self).__init__(engine, **kw)
+        self._store = None
+
+    def __cmp__(self, other):
+        if other is None:
+            return False
+        if isinstance(other, BaseStore):
+            return cmp(dict(self.iteritems()), dict(other.iteritems()))
+
+    def __del__(self):
+        # __init__ didn't succeed, so don't bother closing
+        if not hasattr(self, '_store'):
+            return
+        self.close()
+
+    def __iter__(self):
+        for k in self.keys():
+            yield k
+
+    def __len__(self):
+        return len(self.keys())
+
+    def __repr__(self):
+        return repr(dict(self.iteritems()))
+
+    def close(self):
+        '''Closes internal store and clears object references.'''
+        try:
+            self._store.close()
+        except AttributeError:
+            pass
+        self._store = None
+
+    def clear(self):
+        '''Removes all keys and values from a store.'''
+        for key in self.keys():
+            del self[key]
+
+    def items(self):
+        '''Returns a list with all key/value pairs in the store.'''
+        return list(self.iteritems())
+
+    def iteritems(self):
+        '''Lazily returns all key/value pairs in a store.'''
+        for k in self:
+            yield (k, self[k])
+
+    def iterkeys(self):
+        '''Lazy returns all keys in a store.'''
+        return self.__iter__()
+
+    def itervalues(self):
+        '''Lazily returns all values in a store.'''
+        for _, v in self.iteritems():
+            yield v
+
+    def keys(self):
+        '''Returns a list with all keys in a store.'''
+        raise NotImplementedError()
+
+    def pop(self, key, *args):
+        '''
+        Removes and returns a value from a store.
+
+        @param args Default to return if key not present.
+        '''
+        if len(args) > 1:
+            raise TypeError('pop expected at most 2 arguments, got ' + repr(
+                1 + len(args))
+            )
+        try:
+            value = self[key]
+        # Return default if key not in store
+        except KeyError:
+            if args:
+                return args[0]
+        del self[key]
+        return value
+
+    def popitem(self):
+        '''Removes and returns a key, value pair from a store.'''
+        try:
+            k, v = self.iteritems().next()
+        except StopIteration:
+            raise KeyError('Store is empty.')
+        del self[k]
+        return (k, v)
+
+    def setdefault(self, key, default=None):
+        '''
+        Returns the value corresponding to an existing key or sets the
+        to key to the default and returns the default.
+
+        @param default Default value (default: None)
+        '''
+        try:
+            return self[key]
+        except KeyError:
+            self[key] = default
+        return default
+
+    def update(self, other=None, **kw):
+        '''
+        Adds to or overwrites the values in this store with values from
+        another store.
+
+        other Another store
+        kw Additional keys and values to store
+        '''
+        if other is None:
+            pass
+        elif hasattr(other, 'iteritems'):
+            for k, v in other.iteritems():
+                self[k] = v
+        elif hasattr(other, 'keys'):
+            for k in other.keys():
+                self[k] = other[k]
+        else:
+            for k, v in other:
+                self[k] = v
+        if kw:
+            self.update(kw)
+
+    def values(self):
+        '''Returns a list with all values in a store.'''
+        return list(v for _, v in self.iteritems())
+
+
+class Shove(BaseStore):
+
+    '''Common object frontend class.'''
+
+    def __init__(self, store='simple://', cache='simple://', **kw):
+        super(Shove, self).__init__(store, **kw)
+        # Load store
+        self._store = getbackend(store, stores, **kw)
+        # Load cache
+        self._cache = getbackend(cache, caches, **kw)
+        # Buffer for lazy writing and setting for syncing frequency
+        self._buffer, self._sync = dict(), kw.get('sync', 2)
+
+    def __getitem__(self, key):
+        '''Gets a item from shove.'''
+        try:
+            return self._cache[key]
+        except KeyError:
+            # Synchronize cache and store
+            self.sync()
+            value = self._store[key]
+            self._cache[key] = value
+            return value
+
+    def __setitem__(self, key, value):
+        '''Sets an item in shove.'''
+        self._cache[key] = self._buffer[key] = value
+        # When the buffer reaches self._limit, writes the buffer to the store
+        if len(self._buffer) >= self._sync:
+            self.sync()
+
+    def __delitem__(self, key):
+        '''Deletes an item from shove.'''
+        try:
+            del self._cache[key]
+        except KeyError:
+            pass
+        self.sync()
+        del self._store[key]
+
+    def keys(self):
+        '''Returns a list of keys in shove.'''
+        self.sync()
+        return self._store.keys()
+
+    def sync(self):
+        '''Writes buffer to store.'''
+        for k, v in self._buffer.iteritems():
+            self._store[k] = v
+        self._buffer.clear()
+
+    def close(self):
+        '''Finalizes and closes shove.'''
+        # If close has been called, pass
+        if self._store is not None:
+            try:
+                self.sync()
+            except AttributeError:
+                pass
+            self._store.close()
+            self._store = self._cache = self._buffer = None
+
+
+class FileBase(Base):
+
+    '''Base class for file based storage.'''
+
+    def __init__(self, engine, **kw):
+        super(FileBase, self).__init__(engine, **kw)
+        if engine.startswith('file://'):
+            engine = urllib.url2pathname(engine.split('://')[1])
+        self._dir = engine
+        # Create directory
+        if not os.path.exists(self._dir):
+            self._createdir()
+
+    def __getitem__(self, key):
+        # (per Larry Meyn)
+        try:
+            item = open(self._key_to_file(key), 'rb')
+            data = item.read()
+            item.close()
+            return self.loads(data)
+        except:
+            raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        # (per Larry Meyn)
+        try:
+            item = open(self._key_to_file(key), 'wb')
+            item.write(self.dumps(value))
+            item.close()
+        except (IOError, OSError):
+            raise KeyError(key)
+
+    def __delitem__(self, key):
+        try:
+            os.remove(self._key_to_file(key))
+        except (IOError, OSError):
+            raise KeyError(key)
+
+    def __contains__(self, key):
+        return os.path.exists(self._key_to_file(key))
+
+    def __len__(self):
+        return len(os.listdir(self._dir))
+
+    def _createdir(self):
+        '''Creates the store directory.'''
+        try:
+            os.makedirs(self._dir)
+        except OSError:
+            raise EnvironmentError(
+                'Cache directory "%s" does not exist and ' \
+                'could not be created' % self._dir
+            )
+
+    def _key_to_file(self, key):
+        '''Gives the filesystem path for a key.'''
+        return os.path.join(self._dir, urllib.quote_plus(key))
+
+    def keys(self):
+        '''Returns a list of keys in the store.'''
+        return [urllib.unquote_plus(name) for name in os.listdir(self._dir)]
+
+
+class SimpleBase(Base):
+
+    '''Single-process in-memory store base class.'''
+
+    def __init__(self, engine, **kw):
+        super(SimpleBase, self).__init__(engine, **kw)
+        self._store = dict()
+
+    def __getitem__(self, key):
+        try:
+            return self._store[key]
+        except:
+            raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        self._store[key] = value
+
+    def __delitem__(self, key):
+        try:
+            del self._store[key]
+        except:
+            raise KeyError(key)
+
+    def __len__(self):
+        return len(self._store)
+
+    def keys(self):
+        '''Returns a list of keys in the store.'''
+        return self._store.keys()
+
+
+class LRUBase(SimpleBase):
+
+    def __init__(self, engine, **kw):
+        super(LRUBase, self).__init__(engine, **kw)
+        self._max_entries = kw.get('max_entries', 300)
+        self._hits = 0
+        self._misses = 0
+        self._queue = deque()
+        self._refcount = dict()
+
+    def __getitem__(self, key):
+        try:
+            value = super(LRUBase, self).__getitem__(key)
+            self._hits += 1
+        except KeyError:
+            self._misses += 1
+            raise
+        self._housekeep(key)
+        return value
+
+    def __setitem__(self, key, value):
+        super(LRUBase, self).__setitem__(key, value)
+        self._housekeep(key)
+        if len(self._store) > self._max_entries:
+            while len(self._store) > self._max_entries:
+                k = self._queue.popleft()
+                self._refcount[k] -= 1
+                if not self._refcount[k]:
+                    super(LRUBase, self).__delitem__(k)
+                    del self._refcount[k]
+
+    def _housekeep(self, key):
+        self._queue.append(key)
+        self._refcount[key] = self._refcount.get(key, 0) + 1
+        if len(self._queue) > self._max_entries * 4:
+            self._purge_queue()
+
+    def _purge_queue(self):
+        for i in [None] * len(self._queue):
+            k = self._queue.popleft()
+            if self._refcount[k] == 1:
+                self._queue.append(k)
+            else:
+                self._refcount[k] -= 1
+
+
+class DbBase(Base):
+
+    '''Database common base class.'''
+
+    def __init__(self, engine, **kw):
+        super(DbBase, self).__init__(engine, **kw)
+
+    def __delitem__(self, key):
+        self._store.delete(self._store.c.key == key).execute()
+
+    def __len__(self):
+        return self._store.count().execute().fetchone()[0]
+
+
+__all__ = ['Shove']

lib/shove/cache/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

lib/shove/cache/db.py

@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*-
+'''
+Database object cache.
+
+The shove psuedo-URL used for database object caches is the format used by
+SQLAlchemy:
+
+<driver>://<username>:<password>@<host>:<port>/<database>
+
+<driver> is the database engine. The engines currently supported SQLAlchemy are
+sqlite, mysql, postgres, oracle, mssql, and firebird.
+<username> is the database account user name
+<password> is the database accound password
+<host> is the database location
+<port> is the database port
+<database> is the name of the specific database
+
+For more information on specific databases see:
+
+http://www.sqlalchemy.org/docs/dbengine.myt#dbengine_supported
+'''
+
+import time
+import random
+from datetime import datetime
+try:
+    from sqlalchemy import (
+        MetaData, Table, Column, String, Binary, DateTime, select, update,
+        insert, delete,
+    )
+    from shove import DbBase
+except ImportError:
+    raise ImportError('Requires SQLAlchemy >= 0.4')
+
+__all__ = ['DbCache']
+
+
+class DbCache(DbBase):
+
+    '''database cache backend'''
+
+    def __init__(self, engine, **kw):
+        super(DbCache, self).__init__(engine, **kw)
+        # Get table name
+        tablename = kw.get('tablename', 'cache')
+        # Bind metadata
+        self._metadata = MetaData(engine)
+        # Make cache table
+        self._store = Table(tablename, self._metadata,
+            Column('key', String(60), primary_key=True, nullable=False),
+            Column('value', Binary, nullable=False),
+            Column('expires', DateTime, nullable=False),
+        )
+        # Create cache table if it does not exist
+        if not self._store.exists():
+            self._store.create()
+        # Set maximum entries
+        self._max_entries = kw.get('max_entries', 300)
+        # Maximum number of entries to cull per call if cache is full
+        self._maxcull = kw.get('maxcull', 10)
+        # Set timeout
+        self.timeout = kw.get('timeout', 300)
+
+    def __getitem__(self, key):
+        row = select(
+             [self._store.c.value, self._store.c.expires],
+            self._store.c.key == key
+        ).execute().fetchone()
+        if row is not None:
+            # Remove if item expired
+            if row.expires < datetime.now().replace(microsecond=0):
+                del self[key]
+                raise KeyError(key)
+            return self.loads(str(row.value))
+        raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        timeout, value, cache = self.timeout, self.dumps(value), self._store
+        # Cull if too many items
+        if len(self) >= self._max_entries:
+            self._cull()
+        # Generate expiration time
+        expires = datetime.fromtimestamp(
+            time.time() + timeout
+        ).replace(microsecond=0)
+        # Update database if key already present
+        if key in self:
+            update(
+                cache,
+                cache.c.key == key,
+                dict(value=value, expires=expires),
+            ).execute()
+        # Insert new key if key not present
+        else:
+            insert(
+                cache, dict(key=key, value=value, expires=expires)
+            ).execute()
+
+    def _cull(self):
+        '''Remove items in cache to make more room.'''
+        cache, maxcull = self._store, self._maxcull
+        # Remove items that have timed out
+        now = datetime.now().replace(microsecond=0)
+        delete(cache, cache.c.expires < now).execute()
+        # Remove any items over the maximum allowed number in the cache
+        if len(self) >= self._max_entries:
+            # Upper limit for key query
+            ul = maxcull * 2
+            # Get list of keys
+            keys = [
+                i[0] for i in select(
+                    [cache.c.key], limit=ul
+                ).execute().fetchall()
+            ]
+            # Get some keys at random
+            delkeys = list(random.choice(keys) for i in xrange(maxcull))
+            delete(cache, cache.c.key.in_(delkeys)).execute()

lib/shove/cache/file.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+'''
+File-based cache
+
+shove's psuedo-URL for file caches follows the form:
+
+file://<path>
+
+Where the path is a URL path to a directory on a local filesystem.
+Alternatively, a native pathname to the directory can be passed as the 'engine'
+argument.
+'''
+
+import time
+
+from shove import FileBase
+from shove.cache.simple import SimpleCache
+
+
+class FileCache(FileBase, SimpleCache):
+
+    '''File-based cache backend'''
+
+    def __init__(self, engine, **kw):
+        super(FileCache, self).__init__(engine, **kw)
+
+    def __getitem__(self, key):
+        try:
+            exp, value = super(FileCache, self).__getitem__(key)
+            # Remove item if time has expired.
+            if exp < time.time():
+                del self[key]
+                raise KeyError(key)
+            return value
+        except:
+            raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        if len(self) >= self._max_entries:
+            self._cull()
+        super(FileCache, self).__setitem__(
+            key, (time.time() + self.timeout, value)
+        )
+
+
+__all__ = ['FileCache']

lib/shove/cache/filelru.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+'''
+File-based LRU cache
+
+shove's psuedo-URL for file caches follows the form:
+
+file://<path>
+
+Where the path is a URL path to a directory on a local filesystem.
+Alternatively, a native pathname to the directory can be passed as the 'engine'
+argument.
+'''
+
+from shove import FileBase
+from shove.cache.simplelru import SimpleLRUCache
+
+
+class FileCache(FileBase, SimpleLRUCache):
+
+    '''File-based LRU cache backend'''
+
+
+__all__ = ['FileCache']

lib/shove/cache/memcached.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+'''
+"memcached" cache.
+
+The shove psuedo-URL for a memcache cache is:
+
+memcache://<memcache_server>
+'''
+
+try:
+    import memcache
+except ImportError:
+    raise ImportError("Memcache cache requires the 'memcache' library")
+
+from shove import Base
+
+
+class MemCached(Base):
+
+    '''Memcached cache backend'''
+
+    def __init__(self, engine, **kw):
+        super(MemCached, self).__init__(engine, **kw)
+        if engine.startswith('memcache://'):
+            engine = engine.split('://')[1]
+        self._store = memcache.Client(engine.split(';'))
+        # Set timeout
+        self.timeout = kw.get('timeout', 300)
+
+    def __getitem__(self, key):
+        value = self._store.get(key)
+        if value is None:
+            raise KeyError(key)
+        return self.loads(value)
+
+    def __setitem__(self, key, value):
+        self._store.set(key, self.dumps(value), self.timeout)
+
+    def __delitem__(self, key):
+        self._store.delete(key)
+
+
+__all__ = ['MemCached']

lib/shove/cache/memlru.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+'''
+Thread-safe in-memory cache using LRU.
+
+The shove psuedo-URL for a memory cache is:
+
+memlru://
+'''
+
+import copy
+import threading
+
+from shove import synchronized
+from shove.cache.simplelru import SimpleLRUCache
+
+
+class MemoryLRUCache(SimpleLRUCache):
+
+    '''Thread-safe in-memory cache backend using LRU.'''
+
+    def __init__(self, engine, **kw):
+        super(MemoryLRUCache, self).__init__(engine, **kw)
+        self._lock = threading.Condition()
+
+    @synchronized
+    def __setitem__(self, key, value):
+        super(MemoryLRUCache, self).__setitem__(key, value)
+
+    @synchronized
+    def __getitem__(self, key):
+        return copy.deepcopy(super(MemoryLRUCache, self).__getitem__(key))
+
+    @synchronized
+    def __delitem__(self, key):
+        super(MemoryLRUCache, self).__delitem__(key)
+
+
+__all__ = ['MemoryLRUCache']

lib/shove/cache/memory.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+'''
+Thread-safe in-memory cache.
+
+The shove psuedo-URL for a memory cache is:
+
+memory://
+'''
+
+import copy
+import threading
+
+from shove import synchronized
+from shove.cache.simple import SimpleCache
+
+
+class MemoryCache(SimpleCache):
+
+    '''Thread-safe in-memory cache backend.'''
+
+    def __init__(self, engine, **kw):
+        super(MemoryCache, self).__init__(engine, **kw)
+        self._lock = threading.Condition()
+
+    @synchronized
+    def __setitem__(self, key, value):
+        super(MemoryCache, self).__setitem__(key, value)
+
+    @synchronized
+    def __getitem__(self, key):
+        return copy.deepcopy(super(MemoryCache, self).__getitem__(key))
+
+    @synchronized
+    def __delitem__(self, key):
+        super(MemoryCache, self).__delitem__(key)
+
+
+__all__ = ['MemoryCache']

lib/shove/cache/redisdb.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+'''
+Redis-based object cache
+
+The shove psuedo-URL for a redis cache is:
+
+redis://<host>:<port>/<db>
+'''
+
+import urlparse
+
+try:
+    import redis
+except ImportError:
+    raise ImportError('This store requires the redis library')
+
+from shove import Base
+
+
+class RedisCache(Base):
+
+    '''Redis cache backend'''
+
+    init = 'redis://'
+
+    def __init__(self, engine, **kw):
+        super(RedisCache, self).__init__(engine, **kw)
+        spliturl = urlparse.urlsplit(engine)
+        host, port = spliturl[1].split(':')
+        db = spliturl[2].replace('/', '')
+        self._store = redis.Redis(host, int(port), db)
+        # Set timeout
+        self.timeout = kw.get('timeout', 300)
+
+    def __getitem__(self, key):
+        return self.loads(self._store[key])
+
+    def __setitem__(self, key, value):
+        self._store.setex(key, self.dumps(value), self.timeout)
+
+    def __delitem__(self, key):
+        self._store.delete(key)
+
+
+__all__ = ['RedisCache']

lib/shove/cache/simple.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+'''
+Single-process in-memory cache.
+
+The shove psuedo-URL for a simple cache is:
+
+simple://
+'''
+
+import time
+import random
+
+from shove import SimpleBase
+
+
+class SimpleCache(SimpleBase):
+
+    '''Single-process in-memory cache.'''
+
+    def __init__(self, engine, **kw):
+        super(SimpleCache, self).__init__(engine, **kw)
+        # Get random seed
+        random.seed()
+        # Set maximum number of items to cull if over max
+        self._maxcull = kw.get('maxcull', 10)
+        # Set max entries
+        self._max_entries = kw.get('max_entries', 300)
+        # Set timeout
+        self.timeout = kw.get('timeout', 300)
+
+    def __getitem__(self, key):
+        exp, value = super(SimpleCache, self).__getitem__(key)
+        # Delete if item timed out.
+        if exp < time.time():
+            super(SimpleCache, self).__delitem__(key)
+            raise KeyError(key)
+        return value
+
+    def __setitem__(self, key, value):
+        # Cull values if over max # of entries
+        if len(self) >= self._max_entries:
+            self._cull()
+        # Set expiration time and value
+        exp = time.time() + self.timeout
+        super(SimpleCache, self).__setitem__(key, (exp, value))
+
+    def _cull(self):
+        '''Remove items in cache to make room.'''
+        num, maxcull = 0, self._maxcull
+        # Cull number of items allowed (set by self._maxcull)
+        for key in self.keys():
+            # Remove only maximum # of items allowed by maxcull
+            if num <= maxcull:
+                # Remove items if expired
+                try:
+                    self[key]
+                except KeyError:
+                    num += 1
+            else:
+                break
+        # Remove any additional items up to max # of items allowed by maxcull
+        while len(self) >= self._max_entries and num <= maxcull:
+            # Cull remainder of allowed quota at random
+            del self[random.choice(self.keys())]
+            num += 1
+
+
+__all__ = ['SimpleCache']

lib/shove/cache/simplelru.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+'''
+Single-process in-memory LRU cache.
+
+The shove psuedo-URL for a simple cache is:
+
+simplelru://
+'''
+
+from shove import LRUBase
+
+
+class SimpleLRUCache(LRUBase):
+
+    '''In-memory cache that purges based on least recently used item.'''
+
+
+__all__ = ['SimpleLRUCache']

lib/shove/store/__init__.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+from urllib import url2pathname
+from shove.store.simple import SimpleStore
+
+
+class ClientStore(SimpleStore):
+
+    '''Base class for stores where updates have to be committed.'''
+
+    def __init__(self, engine, **kw):
+        super(ClientStore, self).__init__(engine, **kw)
+        if engine.startswith(self.init):
+            self._engine = url2pathname(engine.split('://')[1])
+
+    def __getitem__(self, key):
+        return self.loads(super(ClientStore, self).__getitem__(key))
+
+    def __setitem__(self, key, value):
+        super(ClientStore, self).__setitem__(key, self.dumps(value))
+
+
+class SyncStore(ClientStore):
+
+    '''Base class for stores where updates have to be committed.'''
+
+    def __getitem__(self, key):
+        return self.loads(super(SyncStore, self).__getitem__(key))
+
+    def __setitem__(self, key, value):
+        super(SyncStore, self).__setitem__(key, value)
+        try:
+            self.sync()
+        except AttributeError:
+            pass
+
+    def __delitem__(self, key):
+        super(SyncStore, self).__delitem__(key)
+        try:
+            self.sync()
+        except AttributeError:
+            pass
+
+
+__all__ = [
+    'bsdb', 'db', 'dbm', 'durusdb', 'file', 'ftp', 'memory', 's3', 'simple',
+    'svn', 'zodb', 'redisdb', 'hdf5db', 'leveldbstore', 'cassandra',
+]

lib/shove/store/bsdb.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+'''
+Berkeley Source Database Store.
+
+shove's psuedo-URL for BSDDB stores follows the form:
+
+bsddb://<path>
+
+Where the path is a URL path to a Berkeley database. Alternatively, the native
+pathname to a Berkeley database can be passed as the 'engine' parameter.
+'''
+try:
+    import bsddb
+except ImportError:
+    raise ImportError('requires bsddb library')
+
+import threading
+
+from shove import synchronized
+from shove.store import SyncStore
+
+
+class BsdStore(SyncStore):
+
+    '''Class for Berkeley Source Database Store.'''
+
+    init = 'bsddb://'
+
+    def __init__(self, engine, **kw):
+        super(BsdStore, self).__init__(engine, **kw)
+        self._store = bsddb.hashopen(self._engine)
+        self._lock = threading.Condition()
+        self.sync = self._store.sync
+
+    @synchronized
+    def __getitem__(self, key):
+        return super(BsdStore, self).__getitem__(key)
+
+    @synchronized
+    def __setitem__(self, key, value):
+        super(BsdStore, self).__setitem__(key, value)
+
+    @synchronized
+    def __delitem__(self, key):
+        super(BsdStore, self).__delitem__(key)
+
+
+__all__ = ['BsdStore']

lib/shove/store/cassandra.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+'''
+Cassandra-based object store
+
+The shove psuedo-URL for a cassandra-based store is:
+
+cassandra://<host>:<port>/<keyspace>/<columnFamily>
+'''
+
+import urlparse
+
+try:
+    import pycassa
+except ImportError:
+    raise ImportError('This store requires the pycassa library')
+
+from shove import BaseStore
+
+
+class CassandraStore(BaseStore):
+
+    '''Cassandra based store'''
+
+    init = 'cassandra://'
+
+    def __init__(self, engine, **kw):
+        super(CassandraStore, self).__init__(engine, **kw)
+        spliturl = urlparse.urlsplit(engine)
+        _, keyspace, column_family = spliturl[2].split('/')
+        try:
+            self._pool = pycassa.connect(keyspace, [spliturl[1]])
+            self._store = pycassa.ColumnFamily(self._pool, column_family)
+        except pycassa.InvalidRequestException:
+            from pycassa.system_manager import SystemManager
+            system_manager = SystemManager(spliturl[1])
+            system_manager.create_keyspace(
+                keyspace,
+                pycassa.system_manager.SIMPLE_STRATEGY,
+                {'replication_factor': str(kw.get('replication', 1))}
+            )
+            system_manager.create_column_family(keyspace, column_family)
+            self._pool = pycassa.connect(keyspace, [spliturl[1]])
+            self._store = pycassa.ColumnFamily(self._pool, column_family)
+
+    def __getitem__(self, key):
+        try:
+            item = self._store.get(key).get(key)
+            if item is not None:
+                return self.loads(item)
+            raise KeyError(key)
+        except pycassa.NotFoundException:
+            raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        self._store.insert(key, dict(key=self.dumps(value)))
+
+    def __delitem__(self, key):
+        # beware eventual consistency
+        try:
+            self._store.remove(key)
+        except pycassa.NotFoundException:
+            raise KeyError(key)
+
+    def clear(self):
+        # beware eventual consistency
+        self._store.truncate()
+
+    def keys(self):
+        return list(i[0] for i in self._store.get_range())
+
+
+__all__ = ['CassandraStore']

lib/shove/store/db.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+'''
+Database object store.
+
+The shove psuedo-URL used for database object stores is the format used by
+SQLAlchemy:
+
+<driver>://<username>:<password>@<host>:<port>/<database>
+
+<driver> is the database engine. The engines currently supported SQLAlchemy are
+sqlite, mysql, postgres, oracle, mssql, and firebird.
+<username> is the database account user name
+<password> is the database accound password
+<host> is the database location
+<port> is the database port
+<database> is the name of the specific database
+
+For more information on specific databases see:
+
+http://www.sqlalchemy.org/docs/dbengine.myt#dbengine_supported
+'''
+
+try:
+    from sqlalchemy import MetaData, Table, Column, String, Binary, select
+    from shove import BaseStore, DbBase
+except ImportError:
+    raise ImportError('Requires SQLAlchemy >= 0.4')
+
+
+class DbStore(BaseStore, DbBase):
+
+    '''Database cache backend.'''
+
+    def __init__(self, engine, **kw):
+        super(DbStore, self).__init__(engine, **kw)
+        # Get tablename
+        tablename = kw.get('tablename', 'store')
+        # Bind metadata
+        self._metadata = MetaData(engine)
+        # Make store table
+        self._store = Table(tablename, self._metadata,
+            Column('key', String(255), primary_key=True, nullable=False),
+            Column('value', Binary, nullable=False),
+        )
+        # Create store table if it does not exist
+        if not self._store.exists():
+            self._store.create()
+
+    def __getitem__(self, key):
+        row = select(
+            [self._store.c.value], self._store.c.key == key,
+        ).execute().fetchone()
+        if row is not None:
+            return self.loads(str(row.value))
+        raise KeyError(key)
+
+    def __setitem__(self, k, v):
+        v, store = self.dumps(v), self._store
+        # Update database if key already present
+        if k in self:
+            store.update(store.c.key == k).execute(value=v)
+        # Insert new key if key not present
+        else:
+            store.insert().execute(key=k, value=v)
+
+    def keys(self):
+        '''Returns a list of keys in the store.'''
+        return list(i[0] for i in select(
+            [self._store.c.key]
+        ).execute().fetchall())
+
+
+__all__ = ['DbStore']

lib/shove/store/dbm.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+'''
+DBM Database Store.
+
+shove's psuedo-URL for DBM stores follows the form:
+
+dbm://<path>
+
+Where <path> is a URL path to a DBM database. Alternatively, the native
+pathname to a DBM database can be passed as the 'engine' parameter.
+'''
+
+import anydbm
+
+from shove.store import SyncStore
+
+
+class DbmStore(SyncStore):
+
+    '''Class for variants of the DBM database.'''
+
+    init = 'dbm://'
+
+    def __init__(self, engine, **kw):
+        super(DbmStore, self).__init__(engine, **kw)
+        self._store = anydbm.open(self._engine, 'c')
+        try:
+            self.sync = self._store.sync
+        except AttributeError:
+            pass
+
+
+__all__ = ['DbmStore']

lib/shove/store/durusdb.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+'''
+Durus object database frontend.
+
+shove's psuedo-URL for Durus stores follows the form:
+
+durus://<path>
+
+
+Where the path is a URL path to a durus FileStorage database. Alternatively, a
+native pathname to a durus database can be passed as the 'engine' parameter.
+'''
+
+try:
+    from durus.connection import Connection
+    from durus.file_storage import FileStorage
+except ImportError:
+    raise ImportError('Requires Durus library')
+
+from shove.store import SyncStore
+
+
+class DurusStore(SyncStore):
+
+    '''Class for Durus object database frontend.'''
+
+    init = 'durus://'
+
+    def __init__(self, engine, **kw):
+        super(DurusStore, self).__init__(engine, **kw)
+        self._db = FileStorage(self._engine)
+        self._connection = Connection(self._db)
+        self.sync = self._connection.commit
+        self._store = self._connection.get_root()
+
+    def close(self):
+        '''Closes all open storage and connections.'''
+        self.sync()
+        self._db.close()
+        super(DurusStore, self).close()
+
+
+__all__ = ['DurusStore']

lib/shove/store/file.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+'''
+Filesystem-based object store
+
+shove's psuedo-URL for filesystem-based stores follows the form:
+
+file://<path>
+
+Where the path is a URL path to a directory on a local filesystem.
+Alternatively, a native pathname to the directory can be passed as the 'engine'
+argument.
+'''
+
+from shove import BaseStore, FileBase
+
+
+class FileStore(FileBase, BaseStore):
+
+    '''File-based store.'''
+
+    def __init__(self, engine, **kw):
+        super(FileStore, self).__init__(engine, **kw)
+
+
+__all__ = ['FileStore']

lib/shove/store/ftp.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+'''
+FTP-accessed stores
+
+shove's URL for FTP accessed stores follows the standard form for FTP URLs
+defined in RFC-1738:
+
+ftp://<user>:<password>@<host>:<port>/<url-path>
+'''
+
+import urlparse
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+from ftplib import FTP, error_perm
+
+from shove import BaseStore
+
+
+class FtpStore(BaseStore):
+
+    def __init__(self, engine, **kw):
+        super(FtpStore, self).__init__(engine, **kw)
+        user = kw.get('user', 'anonymous')
+        password = kw.get('password', '')
+        spliturl = urlparse.urlsplit(engine)
+        # Set URL, path, and strip 'ftp://' off
+        base, path = spliturl[1], spliturl[2] + '/'
+        if '@' in base:
+            auth, base = base.split('@')
+            user, password = auth.split(':')
+        self._store = FTP(base, user, password)
+        # Change to remote path if it exits
+        try:
+            self._store.cwd(path)
+        except error_perm:
+            self._makedir(path)
+        self._base, self._user, self._password = base, user, password
+        self._updated, self ._keys = True, None
+
+    def __getitem__(self, key):
+        try:
+            local = StringIO()
+            # Download item
+            self._store.retrbinary('RETR %s' % key, local.write)
+            self._updated = False
+            return self.loads(local.getvalue())
+        except:
+            raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        local = StringIO(self.dumps(value))
+        self._store.storbinary('STOR %s' % key, local)
+        self._updated = True
+
+    def __delitem__(self, key):
+        try:
+            self._store.delete(key)
+            self._updated = True
+        except:
+            raise KeyError(key)
+
+    def _makedir(self, path):
+        '''Makes remote paths on an FTP server.'''
+        paths = list(reversed([i for i in path.split('/') if i != '']))
+        while paths:
+            tpath = paths.pop()
+            self._store.mkd(tpath)
+            self._store.cwd(tpath)
+
+    def keys(self):
+        '''Returns a list of keys in a store.'''
+        if self._updated or self._keys is None:
+            rlist, nlist = list(), list()
+            # Remote directory listing
+            self._store.retrlines('LIST -a', rlist.append)
+            for rlisting in rlist:
+                # Split remote file based on whitespace
+                rfile = rlisting.split()
+                # Append tuple of remote item type & name
+                if rfile[-1] not in ('.', '..') and rfile[0].startswith('-'):
+                    nlist.append(rfile[-1])
+            self._keys = nlist
+        return self._keys
+
+
+__all__ = ['FtpStore']

lib/shove/store/hdf5.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+'''
+HDF5 Database Store.
+
+shove's psuedo-URL for HDF5 stores follows the form:
+
+hdf5://<path>/<group>
+
+Where <path> is a URL path to a HDF5 database. Alternatively, the native
+pathname to a HDF5 database can be passed as the 'engine' parameter.
+<group> is the name of the database.
+'''
+
+try:
+    import h5py
+except ImportError:
+    raise ImportError('This store requires h5py library')
+
+from shove.store import ClientStore
+
+
+class HDF5Store(ClientStore):
+
+    '''LevelDB based store'''
+
+    init = 'hdf5://'
+
+    def __init__(self, engine, **kw):
+        super(HDF5Store, self).__init__(engine, **kw)
+        engine, group = self._engine.rsplit('/')
+        self._store = h5py.File(engine).require_group(group).attrs
+
+
+__all__ = ['HDF5Store']

lib/shove/store/leveldbstore.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+'''
+LevelDB Database Store.
+
+shove's psuedo-URL for LevelDB stores follows the form:
+
+leveldb://<path>
+
+Where <path> is a URL path to a LevelDB database. Alternatively, the native
+pathname to a LevelDB database can be passed as the 'engine' parameter.
+'''
+
+try:
+    import leveldb
+except ImportError:
+    raise ImportError('This store requires py-leveldb library')
+
+from shove.store import ClientStore
+
+
+class LevelDBStore(ClientStore):
+
+    '''LevelDB based store'''
+
+    init = 'leveldb://'
+
+    def __init__(self, engine, **kw):
+        super(LevelDBStore, self).__init__(engine, **kw)
+        self._store = leveldb.LevelDB(self._engine)
+
+    def __getitem__(self, key):
+        item = self.loads(self._store.Get(key))
+        if item is not None:
+            return item
+        raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        self._store.Put(key, self.dumps(value))
+
+    def __delitem__(self, key):
+        self._store.Delete(key)
+
+    def keys(self):
+        return list(k for k in self._store.RangeIter(include_value=False))
+
+
+__all__ = ['LevelDBStore']

lib/shove/store/memory.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+'''
+Thread-safe in-memory store.
+
+The shove psuedo-URL for a memory store is:
+
+memory://
+'''
+
+import copy
+import threading
+
+from shove import synchronized
+from shove.store.simple import SimpleStore
+
+
+class MemoryStore(SimpleStore):
+
+    '''Thread-safe in-memory store.'''
+
+    def __init__(self, engine, **kw):
+        super(MemoryStore, self).__init__(engine, **kw)
+        self._lock = threading.Condition()
+
+    @synchronized
+    def __getitem__(self, key):
+        return copy.deepcopy(super(MemoryStore, self).__getitem__(key))
+
+    @synchronized
+    def __setitem__(self, key, value):
+        super(MemoryStore, self).__setitem__(key, value)
+
+    @synchronized
+    def __delitem__(self, key):
+        super(MemoryStore, self).__delitem__(key)
+
+
+__all__ = ['MemoryStore']

lib/shove/store/redisdb.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+'''
+Redis-based object store
+
+The shove psuedo-URL for a redis-based store is:
+
+redis://<host>:<port>/<db>
+'''
+
+import urlparse
+
+try:
+    import redis
+except ImportError:
+    raise ImportError('This store requires the redis library')
+
+from shove.store import ClientStore
+
+
+class RedisStore(ClientStore):
+
+    '''Redis based store'''
+
+    init = 'redis://'
+
+    def __init__(self, engine, **kw):
+        super(RedisStore, self).__init__(engine, **kw)
+        spliturl = urlparse.urlsplit(engine)
+        host, port = spliturl[1].split(':')
+        db = spliturl[2].replace('/', '')
+        self._store = redis.Redis(host, int(port), db)
+
+    def __contains__(self, key):
+        return self._store.exists(key)
+
+    def clear(self):
+        self._store.flushdb()
+
+    def keys(self):
+        return self._store.keys()
+
+    def setdefault(self, key, default=None):
+        return self._store.getset(key, default)
+
+    def update(self, other=None, **kw):
+        args = kw if other is not None else other
+        self._store.mset(args)
+
+
+__all__ = ['RedisStore']

lib/shove/store/s3.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+'''
+S3-accessed stores
+
+shove's psuedo-URL for stores found on Amazon.com's S3 web service follows this
+form:
+
+s3://<s3_key>:<s3_secret>@<bucket>
+
+<s3_key> is the Access Key issued by Amazon
+<s3_secret> is the Secret Access Key issued by Amazon
+<bucket> is the name of the bucket accessed through the S3 service
+'''
+
+try:
+    from boto.s3.connection import S3Connection
+    from boto.s3.key import Key
+except ImportError:
+    raise ImportError('Requires boto library')
+
+from shove import BaseStore
+
+
+class S3Store(BaseStore):
+
+    def __init__(self, engine=None, **kw):
+        super(S3Store, self).__init__(engine, **kw)
+        # key = Access Key, secret=Secret Access Key, bucket=bucket name
+        key, secret, bucket = kw.get('key'), kw.get('secret'), kw.get('bucket')
+        if engine is not None:
+            auth, bucket = engine.split('://')[1].split('@')
+            key, secret = auth.split(':')
+        # kw 'secure' = (True or False, use HTTPS)
+        self._conn = S3Connection(key, secret, kw.get('secure', False))
+        buckets = self._conn.get_all_buckets()
+        # Use bucket if it exists
+        for b in buckets:
+            if b.name == bucket:
+                self._store = b
+                break
+        # Create bucket if it doesn't exist
+        else:
+            self._store = self._conn.create_bucket(bucket)
+        # Set bucket permission ('private', 'public-read',
+        # 'public-read-write', 'authenticated-read'
+        self._store.set_acl(kw.get('acl', 'private'))
+        # Updated flag used for avoiding network calls
+        self._updated, self._keys = True, None
+
+    def __getitem__(self, key):
+        rkey = self._store.lookup(key)
+        if rkey is None:
+            raise KeyError(key)
+        # Fetch string
+        value = self.loads(rkey.get_contents_as_string())
+        # Flag that the store has not been updated
+        self._updated = False
+        return value
+
+    def __setitem__(self, key, value):
+        rkey = Key(self._store)
+        rkey.key = key
+        rkey.set_contents_from_string(self.dumps(value))
+        # Flag that the store has been updated
+        self._updated = True
+
+    def __delitem__(self, key):
+        try:
+            self._store.delete_key(key)
+            # Flag that the store has been updated
+            self._updated = True
+        except:
+            raise KeyError(key)
+
+    def keys(self):
+        '''Returns a list of keys in the store.'''
+        return list(i[0] for i in self.items())
+
+    def items(self):
+        '''Returns a list of items from the store.'''
+        if self._updated or self._keys is None:
+            self._keys = self._store.get_all_keys()
+        return list((str(k.key), k) for k in self._keys)
+
+    def iteritems(self):
+        '''Lazily returns items from the store.'''
+        for k in self.items():
+            yield (k.key, k)
+
+
+__all__ = ['S3Store']

lib/shove/store/simple.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+'''
+Single-process in-memory store.
+
+The shove psuedo-URL for a simple store is:
+
+simple://
+'''
+
+from shove import BaseStore, SimpleBase
+
+
+class SimpleStore(SimpleBase, BaseStore):
+
+    '''Single-process in-memory store.'''
+
+    def __init__(self, engine, **kw):
+        super(SimpleStore, self).__init__(engine, **kw)
+
+
+__all__ = ['SimpleStore']

lib/shove/store/svn.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+'''
+subversion managed store.
+
+The shove psuedo-URL used for a subversion store that is password protected is:
+
+svn:<username><password>:<path>?url=<url>
+
+or for non-password protected repositories:
+
+svn://<path>?url=<url>
+
+<path> is the local repository copy
+<url> is the URL of the subversion repository
+'''
+
+import os
+import urllib
+import threading
+
+try:
+    import pysvn
+except ImportError:
+    raise ImportError('Requires Python Subversion library')
+
+from shove import BaseStore, synchronized
+
+
+class SvnStore(BaseStore):
+
+    '''Class for subversion store.'''
+
+    def __init__(self, engine=None, **kw):
+        super(SvnStore, self).__init__(engine, **kw)
+        # Get path, url from keywords if used
+        path, url = kw.get('path'), kw.get('url')
+        # Get username. password from keywords if used
+        user, password = kw.get('user'), kw.get('password')
+        # Process psuedo URL if used
+        if engine is not None:
+            path, query = engine.split('n://')[1].split('?')
+            url = query.split('=')[1]
+            # Check for username, password
+            if '@' in path:
+                auth, path = path.split('@')
+                user, password = auth.split(':')
+            path = urllib.url2pathname(path)
+        # Create subversion client
+        self._client = pysvn.Client()
+        # Assign username, password
+        if user is not None:
+            self._client.set_username(user)
+        if password is not None:
+            self._client.set_password(password)
+        # Verify that store exists in repository
+        try:
+            self._client.info2(url)
+        # Create store in repository if it doesn't exist
+        except pysvn.ClientError:
+            self._client.mkdir(url, 'Adding directory')
+        # Verify that local copy exists
+        try:
+            if self._client.info(path) is None:
+                self._client.checkout(url, path)
+        # Check it out if it doesn't exist
+        except pysvn.ClientError:
+            self._client.checkout(url, path)
+        self._path, self._url = path, url
+        # Lock
+        self._lock = threading.Condition()
+
+    @synchronized
+    def __getitem__(self, key):
+        try:
+            return self.loads(self._client.cat(self._key_to_file(key)))
+        except:
+            raise KeyError(key)
+
+    @synchronized
+    def __setitem__(self, key, value):
+        fname = self._key_to_file(key)
+        # Write value to file
+        open(fname, 'wb').write(self.dumps(value))
+        # Add to repository
+        if key not in self:
+            self._client.add(fname)
+        self._client.checkin([fname], 'Adding %s' % fname)
+
+    @synchronized
+    def __delitem__(self, key):
+        try:
+            fname = self._key_to_file(key)
+            self._client.remove(fname)
+            # Remove deleted value from repository
+            self._client.checkin([fname], 'Removing %s' % fname)
+        except:
+            raise KeyError(key)
+
+    def _key_to_file(self, key):
+        '''Gives the filesystem path for a key.'''
+        return os.path.join(self._path, urllib.quote_plus(key))
+
+    @synchronized
+    def keys(self):
+        '''Returns a list of keys in the subversion repository.'''
+        return list(str(i.name.split('/')[-1]) for i
+            in self._client.ls(self._path))
+
+
+__all__ = ['SvnStore']

lib/shove/store/zodb.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+'''
+Zope Object Database store frontend.
+
+shove's psuedo-URL for ZODB stores follows the form:
+
+zodb:<path>
+
+
+Where the path is a URL path to a ZODB FileStorage database. Alternatively, a
+native pathname to a ZODB database can be passed as the 'engine' argument.
+'''
+
+try:
+    import transaction
+    from ZODB import FileStorage, DB
+except ImportError:
+    raise ImportError('Requires ZODB library')
+
+from shove.store import SyncStore
+
+
+class ZodbStore(SyncStore):
+
+    '''ZODB store front end.'''
+
+    init = 'zodb://'
+
+    def __init__(self, engine, **kw):
+        super(ZodbStore, self).__init__(engine, **kw)
+        # Handle psuedo-URL
+        self._storage = FileStorage.FileStorage(self._engine)
+        self._db = DB(self._storage)
+        self._connection = self._db.open()
+        self._store = self._connection.root()
+        # Keeps DB in synch through commits of transactions
+        self.sync = transaction.commit
+
+    def close(self):
+        '''Closes all open storage and connections.'''
+        self.sync()
+        super(ZodbStore, self).close()
+        self._connection.close()
+        self._db.close()
+        self._storage.close()
+
+
+__all__ = ['ZodbStore']

lib/shove/tests/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

lib/shove/tests/test_bsddb_store.py

@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestBsdbStore(unittest.TestCase):
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove('bsddb://test.db', compress=True)
+
+    def tearDown(self):
+        import os
+        self.store.close()
+        os.remove('test.db')
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.assertEqual(len(self.store), 2)
+
+    def test_close(self):
+        self.store.close()
+        self.assertEqual(self.store, None)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        item = self.store.popitem()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['powl'] = 7
+        self.store.setdefault('pow', 8)
+        self.assertEqual(self.store['pow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.update(tstore)
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_cassandra_store.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestCassandraStore(unittest.TestCase):
+
+    def setUp(self):
+        from shove import Shove
+        from pycassa.system_manager import SystemManager
+        system_manager = SystemManager('localhost:9160')
+        try:
+            system_manager.create_column_family('Foo', 'shove')
+        except:
+            pass
+        self.store = Shove('cassandra://localhost:9160/Foo/shove')
+
+    def tearDown(self):
+        self.store.clear()
+        self.store.close()
+        from pycassa.system_manager import SystemManager
+        system_manager = SystemManager('localhost:9160')
+        system_manager.drop_column_family('Foo', 'shove')
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.assertEqual(len(self.store), 2)
+
+#    def test_clear(self):
+#        self.store['max'] = 3
+#        self.store['min'] = 6
+#        self.store['pow'] = 7
+#        self.store.clear()
+#        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+#    def test_popitem(self):
+#        self.store['max'] = 3
+#        self.store['min'] = 6
+#        self.store['pow'] = 7
+#        item = self.store.popitem()
+#        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+#        self.store['pow'] = 7
+        self.store.setdefault('pow', 8)
+        self.assertEqual(self.store.setdefault('pow', 8), 8)
+        self.assertEqual(self.store['pow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.update(tstore)
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_db_cache.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestDbCache(unittest.TestCase):
+
+    initstring = 'sqlite:///'
+
+    def setUp(self):
+        from shove.cache.db import DbCache
+        self.cache = DbCache(self.initstring)
+
+    def tearDown(self):
+        self.cache = None
+
+    def test_getitem(self):
+        self.cache['test'] = 'test'
+        self.assertEqual(self.cache['test'], 'test')
+
+    def test_setitem(self):
+        self.cache['test'] = 'test'
+        self.assertEqual(self.cache['test'], 'test')
+
+    def test_delitem(self):
+        self.cache['test'] = 'test'
+        del self.cache['test']
+        self.assertEqual('test' in self.cache, False)
+
+    def test_get(self):
+        self.assertEqual(self.cache.get('min'), None)
+
+    def test_timeout(self):
+        import time
+        from shove.cache.db import DbCache
+        cache = DbCache(self.initstring, timeout=1)
+        cache['test'] = 'test'
+        time.sleep(2)
+
+        def tmp():
+            cache['test']
+        self.assertRaises(KeyError, tmp)
+
+    def test_cull(self):
+        from shove.cache.db import DbCache
+        cache = DbCache(self.initstring, max_entries=1)
+        cache['test'] = 'test'
+        cache['test2'] = 'test'
+        cache['test2'] = 'test'
+        self.assertEquals(len(cache), 1)
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_db_store.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestDbStore(unittest.TestCase):
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove('sqlite://', compress=True)
+
+    def tearDown(self):
+        self.store.close()
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.assertEqual(len(self.store), 2)
+
+    def test_close(self):
+        self.store.close()
+        self.assertEqual(self.store, None)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        item = self.store.popitem()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['powl'] = 7
+        self.store.setdefault('pow', 8)
+        self.assertEqual(self.store['pow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.update(tstore)
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_dbm_store.py

@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestDbmStore(unittest.TestCase):
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove('dbm://test.dbm', compress=True)
+
+    def tearDown(self):
+        import os
+        self.store.close()
+        try:
+            os.remove('test.dbm.db')
+        except OSError:
+            pass
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.assertEqual(len(self.store), 2)
+
+    def test_close(self):
+        self.store.close()
+        self.assertEqual(self.store, None)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        item = self.store.popitem()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.setdefault('how', 8)
+        self.assertEqual(self.store['how'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.update(tstore)
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_durus_store.py

@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestDurusStore(unittest.TestCase):
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove('durus://test.durus', compress=True)
+
+    def tearDown(self):
+        import os
+        self.store.close()
+        os.remove('test.durus')
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.assertEqual(len(self.store), 2)
+
+    def test_close(self):
+        self.store.close()
+        self.assertEqual(self.store, None)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        item = self.store.popitem()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['powl'] = 7
+        self.store.setdefault('pow', 8)
+        self.assertEqual(self.store['pow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.update(tstore)
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_file_cache.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestFileCache(unittest.TestCase):
+
+    initstring = 'file://test'
+
+    def setUp(self):
+        from shove.cache.file import FileCache
+        self.cache = FileCache(self.initstring)
+
+    def tearDown(self):
+        import os
+        self.cache = None
+        for x in os.listdir('test'):
+            os.remove(os.path.join('test', x))
+        os.rmdir('test')
+
+    def test_getitem(self):
+        self.cache['test'] = 'test'
+        self.assertEqual(self.cache['test'], 'test')
+
+    def test_setitem(self):
+        self.cache['test'] = 'test'
+        self.assertEqual(self.cache['test'], 'test')
+
+    def test_delitem(self):
+        self.cache['test'] = 'test'
+        del self.cache['test']
+        self.assertEqual('test' in self.cache, False)
+
+    def test_get(self):
+        self.assertEqual(self.cache.get('min'), None)
+
+    def test_timeout(self):
+        import time
+        from shove.cache.file import FileCache
+        cache = FileCache(self.initstring, timeout=1)
+        cache['test'] = 'test'
+        time.sleep(2)
+
+        def tmp():
+            cache['test']
+        self.assertRaises(KeyError, tmp)
+
+    def test_cull(self):
+        from shove.cache.file import FileCache
+        cache = FileCache(self.initstring, max_entries=1)
+        cache['test'] = 'test'
+        cache['test2'] = 'test'
+        num = len(cache)
+        self.assertEquals(num, 1)
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_file_store.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestFileStore(unittest.TestCase):
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove('file://test', compress=True)
+
+    def tearDown(self):
+        import os
+        self.store.close()
+        for x in os.listdir('test'):
+            os.remove(os.path.join('test', x))
+        os.rmdir('test')
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.store.sync()
+        tstore.sync()
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.assertEqual(len(self.store), 2)
+
+    def test_close(self):
+        self.store.close()
+        self.assertEqual(self.store, None)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        item = self.store.popitem()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['powl'] = 7
+        self.store.setdefault('pow', 8)
+        self.assertEqual(self.store['pow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.update(tstore)
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_ftp_store.py

@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestFtpStore(unittest.TestCase):
+
+    ftpstring = 'put ftp string here'
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove(self.ftpstring, compress=True)
+
+    def tearDown(self):
+        self.store.clear()
+        self.store.close()
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.store.sync()
+        tstore.sync()
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store.sync()
+        self.assertEqual(len(self.store), 2)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store.sync()
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        item = self.store.popitem()
+        self.store.sync()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['powl'] = 7
+        self.store.setdefault('pow', 8)
+        self.store.sync()
+        self.assertEqual(self.store['pow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.sync()
+        self.store.update(tstore)
+        self.store.sync()
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_hdf5_store.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+
+import unittest2
+
+
+class TestHDF5Store(unittest2.TestCase):
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove('hdf5://test.hdf5/test')
+
+    def tearDown(self):
+        import os
+        self.store.close()
+        try:
+            os.remove('test.hdf5')
+        except OSError:
+            pass
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.assertEqual(len(self.store), 2)
+
+    def test_close(self):
+        self.store.close()
+        self.assertEqual(self.store, None)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        item = self.store.popitem()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.setdefault('bow', 8)
+        self.assertEqual(self.store['bow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.update(tstore)
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+if __name__ == '__main__':
+    unittest2.main()

lib/shove/tests/test_leveldb_store.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+
+import unittest2
+
+
+class TestLevelDBStore(unittest2.TestCase):
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove('leveldb://test', compress=True)
+
+    def tearDown(self):
+        import shutil
+        shutil.rmtree('test')
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.assertEqual(len(self.store), 2)
+
+    def test_close(self):
+        self.store.close()
+        self.assertEqual(self.store, None)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        item = self.store.popitem()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.setdefault('bow', 8)
+        self.assertEqual(self.store['bow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.update(tstore)
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+
+if __name__ == '__main__':
+    unittest2.main()

lib/shove/tests/test_memcached_cache.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestMemcached(unittest.TestCase):
+
+    initstring = 'memcache://localhost:11211'
+
+    def setUp(self):
+        from shove.cache.memcached import MemCached
+        self.cache = MemCached(self.initstring)
+
+    def tearDown(self):
+        self.cache = None
+
+    def test_getitem(self):
+        self.cache['test'] = 'test'
+        self.assertEqual(self.cache['test'], 'test')
+
+    def test_setitem(self):
+        self.cache['test'] = 'test'
+        self.assertEqual(self.cache['test'], 'test')
+
+    def test_delitem(self):
+        self.cache['test'] = 'test'
+        del self.cache['test']
+        self.assertEqual('test' in self.cache, False)
+
+    def test_get(self):
+        self.assertEqual(self.cache.get('min'), None)
+
+    def test_timeout(self):
+        import time
+        from shove.cache.memcached import MemCached
+        cache = MemCached(self.initstring, timeout=1)
+        cache['test'] = 'test'
+        time.sleep(1)
+
+        def tmp():
+            cache['test']
+        self.assertRaises(KeyError, tmp)
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_memory_cache.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestMemoryCache(unittest.TestCase):
+
+    initstring = 'memory://'
+
+    def setUp(self):
+        from shove.cache.memory import MemoryCache
+        self.cache = MemoryCache(self.initstring)
+
+    def tearDown(self):
+        self.cache = None
+
+    def test_getitem(self):
+        self.cache['test'] = 'test'
+        self.assertEqual(self.cache['test'], 'test')
+
+    def test_setitem(self):
+        self.cache['test'] = 'test'
+        self.assertEqual(self.cache['test'], 'test')
+
+    def test_delitem(self):
+        self.cache['test'] = 'test'
+        del self.cache['test']
+        self.assertEqual('test' in self.cache, False)
+
+    def test_get(self):
+        self.assertEqual(self.cache.get('min'), None)
+
+    def test_timeout(self):
+        import time
+        from shove.cache.memory import MemoryCache
+        cache = MemoryCache(self.initstring, timeout=1)
+        cache['test'] = 'test'
+        time.sleep(1)
+
+        def tmp():
+            cache['test']
+        self.assertRaises(KeyError, tmp)
+
+    def test_cull(self):
+        from shove.cache.memory import MemoryCache
+        cache = MemoryCache(self.initstring, max_entries=1)
+        cache['test'] = 'test'
+        cache['test2'] = 'test'
+        cache['test2'] = 'test'
+        self.assertEquals(len(cache), 1)
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_memory_store.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestMemoryStore(unittest.TestCase):
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove('memory://', compress=True)
+
+    def tearDown(self):
+        self.store.close()
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.store.sync()
+        tstore.sync()
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.assertEqual(len(self.store), 2)
+
+    def test_close(self):
+        self.store.close()
+        self.assertEqual(self.store, None)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        item = self.store.popitem()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['powl'] = 7
+        self.store.setdefault('pow', 8)
+        self.assertEqual(self.store['pow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.update(tstore)
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_redis_cache.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestRedisCache(unittest.TestCase):
+
+    initstring = 'redis://localhost:6379/0'
+
+    def setUp(self):
+        from shove.cache.redisdb import RedisCache
+        self.cache = RedisCache(self.initstring)
+
+    def tearDown(self):
+        self.cache = None
+
+    def test_getitem(self):
+        self.cache['test'] = 'test'
+        self.assertEqual(self.cache['test'], 'test')
+
+    def test_setitem(self):
+        self.cache['test'] = 'test'
+        self.assertEqual(self.cache['test'], 'test')
+
+    def test_delitem(self):
+        self.cache['test'] = 'test'
+        del self.cache['test']
+        self.assertEqual('test' in self.cache, False)
+
+    def test_get(self):
+        self.assertEqual(self.cache.get('min'), None)
+
+    def test_timeout(self):
+        import time
+        from shove.cache.redisdb import RedisCache
+        cache = RedisCache(self.initstring, timeout=1)
+        cache['test'] = 'test'
+        time.sleep(3)
+        def tmp(): #@IgnorePep8
+            return cache['test']
+        self.assertRaises(KeyError, tmp)
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_redis_store.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestRedisStore(unittest.TestCase):
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove('redis://localhost:6379/0')
+
+    def tearDown(self):
+        self.store.clear()
+        self.store.close()
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.assertEqual(len(self.store), 2)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        item = self.store.popitem()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['powl'] = 7
+        self.store.setdefault('pow', 8)
+        self.assertEqual(self.store.setdefault('pow', 8), 8)
+        self.assertEqual(self.store['pow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.update(tstore)
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_s3_store.py

@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestS3Store(unittest.TestCase):
+
+    s3string = 's3 test string here'
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove(self.s3string, compress=True)
+
+    def tearDown(self):
+        self.store.clear()
+        self.store.close()
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.store.sync()
+        tstore.sync()
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store.sync()
+        self.assertEqual(len(self.store), 2)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store.sync()
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        item = self.store.popitem()
+        self.store.sync()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['powl'] = 7
+        self.store.setdefault('pow', 8)
+        self.store.sync()
+        self.assertEqual(self.store['pow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.sync()
+        self.store.update(tstore)
+        self.store.sync()
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_simple_cache.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestSimpleCache(unittest.TestCase):
+
+    initstring = 'simple://'
+
+    def setUp(self):
+        from shove.cache.simple import SimpleCache
+        self.cache = SimpleCache(self.initstring)
+
+    def tearDown(self):
+        self.cache = None
+
+    def test_getitem(self):
+        self.cache['test'] = 'test'
+        self.assertEqual(self.cache['test'], 'test')
+
+    def test_setitem(self):
+        self.cache['test'] = 'test'
+        self.assertEqual(self.cache['test'], 'test')
+
+    def test_delitem(self):
+        self.cache['test'] = 'test'
+        del self.cache['test']
+        self.assertEqual('test' in self.cache, False)
+
+    def test_get(self):
+        self.assertEqual(self.cache.get('min'), None)
+
+    def test_timeout(self):
+        import time
+        from shove.cache.simple import SimpleCache
+        cache = SimpleCache(self.initstring, timeout=1)
+        cache['test'] = 'test'
+        time.sleep(1)
+
+        def tmp():
+            cache['test']
+        self.assertRaises(KeyError, tmp)
+
+    def test_cull(self):
+        from shove.cache.simple import SimpleCache
+        cache = SimpleCache(self.initstring, max_entries=1)
+        cache['test'] = 'test'
+        cache['test2'] = 'test'
+        cache['test2'] = 'test'
+        self.assertEquals(len(cache), 1)
+
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_simple_store.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestSimpleStore(unittest.TestCase):
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove('simple://', compress=True)
+
+    def tearDown(self):
+        self.store.close()
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.store.sync()
+        tstore.sync()
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.assertEqual(len(self.store), 2)
+
+    def test_close(self):
+        self.store.close()
+        self.assertEqual(self.store, None)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        item = self.store.popitem()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['powl'] = 7
+        self.store.setdefault('pow', 8)
+        self.assertEqual(self.store['pow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.update(tstore)
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_svn_store.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestSvnStore(unittest.TestCase):
+
+    svnstring = 'SVN test string here'
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove(self.svnstring, compress=True)
+
+    def tearDown(self):
+        self.store.clear()
+        self.store.close()
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.store.sync()
+        tstore.sync()
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store.sync()
+        self.assertEqual(len(self.store), 2)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store.sync()
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        item = self.store.popitem()
+        self.store.sync()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['powl'] = 7
+        self.store.setdefault('pow', 8)
+        self.store.sync()
+        self.assertEqual(self.store['pow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.sync()
+        self.store.update(tstore)
+        self.store.sync()
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.sync()
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+if __name__ == '__main__':
+    unittest.main()

lib/shove/tests/test_zodb_store.py

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+
+class TestZodbStore(unittest.TestCase):
+
+    init = 'zodb://test.db'
+
+    def setUp(self):
+        from shove import Shove
+        self.store = Shove(self.init, compress=True)
+
+    def tearDown(self):
+        self.store.close()
+        import os
+        os.remove('test.db')
+        os.remove('test.db.index')
+        os.remove('test.db.tmp')
+        os.remove('test.db.lock')
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.assertEqual(self.store.get('min'), None)
+
+    def test__cmp__(self):
+        from shove import Shove
+        tstore = Shove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.assertEqual(len(self.store), 2)
+
+    def test_close(self):
+        self.store.close()
+        self.assertEqual(self.store, None)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        item = self.store.popitem()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['powl'] = 7
+        self.store.setdefault('pow', 8)
+        self.assertEqual(self.store['pow'], 8)
+
+    def test_update(self):
+        from shove import Shove
+        tstore = Shove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.update(tstore)
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+
+if __name__ == '__main__':
+    unittest.main()

sickbeard/generic_queue.py

@@ -22,6 +22,7 @@ import threading
 from sickbeard import logger
 
 
+
 class QueuePriorities:
     LOW = 10
     NORMAL = 20

sickbeard/helpers.py

@@ -952,7 +952,7 @@ def get_show_by_name(name):
         return
 
     indexerid = sickbeard.name_cache.retrieveNameFromCache(name)
-    if indexerid or indexerid == 0:
+    if indexerid:
         in_cache = True
 
     showNames = list(set(sickbeard.show_name_helpers.sceneToNormalShowNames(name)))
@@ -966,10 +966,6 @@ def get_show_by_name(name):
         if indexerid:
             break
 
-    # add to name cache if we didn't get it from the cache
-    if not in_cache:
-        sickbeard.name_cache.addNameToCache(name, indexerid if indexerid else 0)
-
     if indexerid:
         logger.log(u"Found Indexer ID:[" + repr(indexerid) + "], using that for [" + str(name) + "}",logger.DEBUG)
         if not showObj:

sickbeard/name_parser/parser.py

@@ -378,7 +378,7 @@ class ParseResult(object):
         new_episode_numbers = []
         new_season_numbers = []
         for epNo in self.episode_numbers:
-            (s, e) = scene_numbering.get_indexer_numbering(self.show.indexerid, self.season_number, epNo)
+            (s, e) = scene_numbering.get_indexer_numbering(self.show.indexerid, self.show.indexer, self.season_number, epNo)
             new_episode_numbers.append(e)
             new_season_numbers.append(s)
 

sickbeard/providers/dtt.py

@@ -80,7 +80,7 @@ class DTTProvider(generic.TorrentProvider):
 
         logger.log(u"Search string: " + searchURL, logger.DEBUG)
 
-        data = self.getRSSFeed(searchURL)
+        data = self.cache.getRSSFeed(searchURL)
 
         if not data:
             return []
@@ -126,7 +126,7 @@ class DTTCache(tvcache.TVCache):
 
         url = self.provider.url + 'rss/allshows?' + urllib.urlencode(params)
         logger.log(u"DTT cache update URL: " + url, logger.DEBUG)
-        return self.provider.getRSSFeed(url)
+        return self.getRSSFeed(url)
 
     def _parseItem(self, item):
         title, url = self.provider._get_title_and_url(item)

sickbeard/providers/ezrss.py

@@ -117,7 +117,7 @@ class EZRSSProvider(generic.TorrentProvider):
 
         logger.log(u"Search string: " + search_url, logger.DEBUG)
 
-        data = self.getRSSFeed(search_url)
+        data = self.cache.getRSSFeed(search_url)
 
         if not data:
             return []
@@ -173,7 +173,7 @@ class EZRSSCache(tvcache.TVCache):
         rss_url = self.provider.url + 'feed/'
         logger.log(self.provider.name + " cache update URL: " + rss_url, logger.DEBUG)
 
-        return self.provider.getRSSFeed(rss_url)
+        return self.getRSSFeed(rss_url)
 
     def _parseItem(self, item):
 

sickbeard/providers/generic.py

@@ -121,28 +121,6 @@ class GenericProvider:
 
         return data
 
-    def getRSSFeed(self, url, post_data=None):
-        parsed = list(urlparse.urlparse(url))
-        parsed[2] = re.sub("/{2,}", "/", parsed[2])  # replace two or more / with one
-
-        if post_data:
-            url = url + 'api?' + urllib.urlencode(post_data)
-
-        f = feedparser.parse(url)
-
-        if not f:
-            logger.log(u"Error loading " + self.name + " URL: " + url, logger.ERROR)
-            return None
-        elif 'error' in f.feed:
-            logger.log(u"Newznab ERROR:[%s] CODE:[%s]" % (f.feed['error']['description'], f.feed['error']['code']),
-                       logger.DEBUG)
-            return None
-        elif not f.entries:
-            logger.log(u"No items found on " + self.name + " using URL: " + url, logger.WARNING)
-            return None
-
-        return f
-
     def downloadResult(self, result):
         """
         Save the result to disk.
@@ -252,13 +230,11 @@ class GenericProvider:
         self._checkAuth()
         self.show = show
 
-        itemList = []
         results = {}
+        searchStrings = []
+        itemList = []
 
         for epObj in episodes:
-            scene_season = epObj.scene_season
-            scene_episode = epObj.scene_episode
-
             cacheResult = self.cache.searchCache(epObj, manualSearch)
             if len(cacheResult):
                 return cacheResult
@@ -268,94 +244,111 @@ class GenericProvider:
             else:
                 logger.log(u'Searching "%s" for "%s" as "%s"' % (self.name, epObj.prettyName(), epObj.prettySceneName()))
 
-            # get our season pack search results
+            # get our search strings
             if seasonSearch:
-                for curString in self._get_season_search_strings(epObj):
-                    itemList += self._doSearch(curString)
+                searchStrings += self._get_season_search_strings(epObj)
+            searchStrings += self._get_episode_search_strings(epObj)
 
-            # get our season/episode search results
-            for curString in self._get_episode_search_strings(epObj):
-                itemList += self._doSearch(curString)
+        # remove duplicate search strings
+        searchStrings = [i for n, i in enumerate(searchStrings) if i not in searchStrings[n + 1:]]
 
-            for item in itemList:
+        for curString in sorted(searchStrings):
+            itemList += self._doSearch(curString)
 
-                (title, url) = self._get_title_and_url(item)
+        # remove duplicate items
+        itemList = list(set(itemList))
+        for item in itemList:
 
-                quality = self.getQuality(item)
+            (title, url) = self._get_title_and_url(item)
 
-                # parse the file name
-                try:
-                    myParser = NameParser(False)
-                    parse_result = myParser.parse(title)
-                except InvalidNameException:
-                    logger.log(u"Unable to parse the filename " + title + " into a valid episode", logger.WARNING)
+            quality = self.getQuality(item)
+
+            # parse the file name
+            try:
+                myParser = NameParser(False)
+                parse_result = myParser.parse(title).convert()
+            except InvalidNameException:
+                logger.log(u"Unable to parse the filename " + title + " into a valid episode", logger.WARNING)
+                continue
+
+            if not (self.show.air_by_date or self.show.sports):
+                if not parse_result.episode_numbers and (parse_result.season_number != None and parse_result.season_number != season) or (
+                                parse_result.season_number == None and season != 1):
+                    logger.log(u"The result " + title + " doesn't seem to be a valid season for season " + str(
+                        season) + ", ignoring", logger.DEBUG)
+                    continue
+                elif len(parse_result.episode_numbers) and (parse_result.season_number != season or parse_result.episode_numbers[0] not in parse_result.episode_numbers):
+                    logger.log(u"Episode " + title + " isn't " + str(season) + "x" + str(
+                        parse_result.episode_numbers[0]) + ", skipping it", logger.DEBUG)
                     continue
 
-                if not (self.show.air_by_date or self.show.sports):
-                    if not parse_result.episode_numbers and (parse_result.season_number != None and parse_result.season_number != season) or (
-                                    parse_result.season_number == None and season != 1):
-                        logger.log(u"The result " + title + " doesn't seem to be a valid season for season " + str(
-                            season) + ", ignoring", logger.DEBUG)
-                        continue
-                    elif len(parse_result.episode_numbers) and (parse_result.season_number != scene_season or scene_episode not in parse_result.episode_numbers):
-                        logger.log(u"Episode " + title + " isn't " + str(scene_season) + "x" + str(
-                            scene_episode) + ", skipping it", logger.DEBUG)
-                        continue
-
-                else:
-                    if not (parse_result.air_by_date or parse_result.sports):
-                        logger.log(
-                            u"This is supposed to be a date search but the result " + title + " didn't parse as one, skipping it",
-                            logger.DEBUG)
-                        continue
-
-                    if ((parse_result.air_date != epObj.airdate and parse_result.air_by_date) or (
-                                    parse_result.sports_event_date != epObj.airdate and parse_result.sports)):
-                        logger.log(u"Episode " + title + " didn't air on " + str(epObj.airdate) + ", skipping it",
-                                   logger.DEBUG)
-                        continue
-
-                # make sure we want the episode
-                if not self.show.wantEpisode(epObj.season, epObj.episode, quality, manualSearch=manualSearch):
+                # we just use the existing info for normal searches
+                actual_season = season
+                actual_episodes = parse_result.episode_numbers
+            else:
+                if not (parse_result.air_by_date or parse_result.sports):
                     logger.log(
-                        u"Ignoring result " + title + " because we don't want an episode that is " + Quality.qualityStrings[
-                            quality], logger.DEBUG)
+                        u"This is supposed to be a date search but the result " + title + " didn't parse as one, skipping it",
+                        logger.DEBUG)
                     continue
 
-                logger.log(u"Found result " + title + " at " + url, logger.DEBUG)
-
-                # make a result object
-                epObjs = [epObj]
-
-                result = self.getResult(epObjs)
-                result.url = url
-                result.name = title
-                result.quality = quality
-                result.provider = self
-                result.content = None
-
-                if len(epObjs) == 1:
-                    epNum = epObj.episode
-                    logger.log(u"Single episode result.", logger.DEBUG)
-                elif len(epObjs) > 1:
-                    epNum = MULTI_EP_RESULT
-                    logger.log(u"Separating multi-episode result to check for later - result contains episodes: " + str(
-                        parse_result.episode_numbers), logger.DEBUG)
-                elif len(epObjs) == 0:
-                    epNum = SEASON_RESULT
-                    result.extraInfo = [self.show]
-                    logger.log(u"Separating full season result to check for later", logger.DEBUG)
-
-                if epNum in results:
-                    results[epNum].append(result)
-                else:
-                    results[epNum] = [result]
-
-                # remove duplicate results
-                results[epNum] = list(set(results[epNum]))
-
-                # found the result we wanted
-                break
+                myDB = db.DBConnection()
+                sql_results = myDB.select("SELECT season, episode FROM tv_episodes WHERE showid = ? AND airdate = ?",
+                                          [show.indexerid, parse_result.air_date.toordinal() or parse_result.sports_event_date.toordinal()])
+
+                if len(sql_results) != 1:
+                    logger.log(
+                        u"Tried to look up the date for the episode " + title + " but the database didn't give proper results, skipping it",
+                        logger.WARNING)
+                    continue
+
+                actual_season = int(sql_results[0]["season"])
+                actual_episodes = [int(sql_results[0]["episode"])]
+
+            # make sure we want the episode
+            wantEp = True
+            for epNo in actual_episodes:
+                if not show.wantEpisode(actual_season, epNo, quality):
+                    wantEp = False
+                    break
+
+            if not wantEp:
+                logger.log(u"Ignoring result " + title + " because we don't want an episode that is " + Quality.qualityStrings[quality], logger.DEBUG)
+                continue
+
+            logger.log(u"Found result " + title + " at " + url, logger.DEBUG)
+
+            # make a result object
+            epObj = []
+            for curEp in actual_episodes:
+                epObj.append(show.getEpisode(actual_season, curEp))
+
+            result = self.getResult(epObj)
+            result.url = url
+            result.name = title
+            result.quality = quality
+            result.provider = self
+            result.content = None
+
+            if len(epObj) == 1:
+                epNum = epObj[0].episode
+                logger.log(u"Single episode result.", logger.DEBUG)
+            elif len(epObj) > 1:
+                epNum = MULTI_EP_RESULT
+                logger.log(u"Separating multi-episode result to check for later - result contains episodes: " + str(
+                    parse_result.episode_numbers), logger.DEBUG)
+            elif len(epObj) == 0:
+                epNum = SEASON_RESULT
+                result.extraInfo = [show]
+                logger.log(u"Separating full season result to check for later", logger.DEBUG)
+
+            if epNum in results:
+                results[epNum].append(result)
+            else:
+                results[epNum] = [result]
+
+            # remove duplicate results
+            results[epNum] = list(set(results[epNum]))
 
         return results
 

sickbeard/providers/newzbin.py

@@ -319,7 +319,7 @@ class NewzbinProvider(generic.NZBProvider):
         url = self.url + "search/?%s" % urllib.urlencode(params)
         logger.log("Newzbin search URL: " + url, logger.DEBUG)
 
-        return self.getRSSFeed(url)
+        return self.cache.getRSSFeed(url)
 
     def _checkAuth(self):
         if sickbeard.NEWZBIN_USERNAME in (None, "") or sickbeard.NEWZBIN_PASSWORD in (None, ""):

sickbeard/providers/newznab.py

@@ -268,7 +268,7 @@ class NewznabCache(tvcache.TVCache):
 
         logger.log(self.provider.name + " cache update URL: " + rss_url, logger.DEBUG)
 
-        return self.provider.getRSSFeed(rss_url)
+        return self.getRSSFeed(rss_url)
 
     def _checkAuth(self, data):
         return self.provider._checkAuthFromData(data)

sickbeard/providers/nyaatorrents.py

@@ -133,7 +133,7 @@ class NyaaCache(tvcache.TVCache):
 
         logger.log(u"NyaaTorrents cache update URL: " + url, logger.DEBUG)
 
-        return self.provider.getRSSFeed(url)
+        return self.getRSSFeed(url)
 
     def _parseItem(self, item):
         (title, url) = self.provider._get_title_and_url(item)

sickbeard/providers/nzbs_org_old.py

@@ -76,7 +76,7 @@ class NZBsProvider(generic.NZBProvider):
 
         logger.log(u"Search string: " + searchURL, logger.DEBUG)
 
-        data = self.getRSSFeed(searchURL)
+        data = self.cache.getRSSFeed(searchURL)
 
         # Pause to avoid 503's
         time.sleep(5)

sickbeard/providers/nzbsrus.py

@@ -67,7 +67,7 @@ class NZBsRUSProvider(generic.NZBProvider):
         searchURL = self.url + 'api.php?' + urllib.urlencode(params)
         logger.log(u"NZBS'R'US search url: " + searchURL, logger.DEBUG)
 
-        data = self.getRSSFeed(searchURL)
+        data = self.cache.getRSSFeed(searchURL)
         if not data:
             return []
 
@@ -107,7 +107,7 @@ class NZBsRUSCache(tvcache.TVCache):
         url += urllib.urlencode(urlArgs)
         logger.log(u"NZBs'R'US cache update URL: " + url, logger.DEBUG)
 
-        return self.provider.getRSSFeed(url)
+        return self.getRSSFeed(url)
 
     def _checkAuth(self, data):
         return data != 'Invalid Link'

sickbeard/providers/omgwtfnzbs.py

@@ -164,7 +164,7 @@ class OmgwtfnzbsCache(tvcache.TVCache):
 
         logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG)
 
-        return self.provider.getRSSFeed(rss_url)
+        return self.getRSSFeed(rss_url)
 
     def _checkAuth(self, data):
         return self.provider._checkAuthFromData(data)

sickbeard/providers/rsstorrent.py

@@ -33,7 +33,6 @@ from sickbeard.exceptions import ex
 from lib import requests
 from lib.requests import exceptions
 from lib.bencode import bdecode
-from lib.feedparser import feedparser
 
 class TorrentRssProvider(generic.TorrentProvider):
     def __init__(self, name, url):
@@ -156,7 +155,7 @@ class TorrentRssCache(tvcache.TVCache):
     def _getRSSData(self):
         url = self.provider.url
         logger.log(u"TorrentRssCache cache update URL: " + self.provider.url, logger.DEBUG)
-        return self.provider.getRSSFeed(url)
+        return self.getRSSFeed(url)
 
     def _parseItem(self, item):
 

sickbeard/providers/tvtorrents.py

@@ -84,7 +84,7 @@ class TvTorrentsCache(tvcache.TVCache):
         rss_url = self.provider.url + 'RssServlet?digest=' + sickbeard.TVTORRENTS_DIGEST + '&hash=' + sickbeard.TVTORRENTS_HASH + '&fname=true&exclude=(' + ignore_regex + ')'
         logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG)
 
-        return self.provider.getRSSFeed(rss_url)
+        return self.getRSSFeed(rss_url)
 
     def _checkAuth(self, data):
         return self.provider._checkAuthFromData(data)

sickbeard/providers/womble.py

@@ -42,7 +42,7 @@ class WombleCache(tvcache.TVCache):
     def _getRSSData(self):
         url = self.provider.url + 'rss/?sec=TV-x264&fr=false'
         logger.log(u"Womble's Index cache update URL: " + url, logger.DEBUG)
-        return self.provider.getRSSFeed(url)
+        return self.getRSSFeed(url)
 
     def _checkAuth(self, data):
         return data != 'Invalid Link'

sickbeard/search_queue.py

@@ -47,6 +47,8 @@ class SearchQueue(generic_queue.GenericQueue):
         for cur_item in self.queue:
             if isinstance(cur_item, ManualSearchQueueItem) and cur_item.ep_obj == ep_obj:
                 return True
+            if isinstance(cur_item, BacklogQueueItem) and cur_item.ep_obj == ep_obj:
+                return True
         return False
 
     def pause_backlog(self):

sickbeard/tvcache.py

@@ -15,21 +15,28 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Sick Beard.  If not, see <http://www.gnu.org/licenses/>.
+import os
 
 import time
 import datetime
 import sqlite3
+import urllib
+import urlparse
+import re
 
 import sickbeard
 
+from shove import Shove
+from feedcache import cache
+
 from sickbeard import db
 from sickbeard import logger
 from sickbeard.common import Quality
 
 from sickbeard import helpers, show_name_helpers
-from sickbeard import name_cache, scene_exceptions
 from sickbeard.exceptions import MultipleShowObjectsException, ex
-from sickbeard.exceptions import ex, AuthException
+from sickbeard.exceptions import AuthException
+from sickbeard import encodingKludge as ek
 
 from name_parser.parser import NameParser, InvalidNameException
 
@@ -85,6 +92,34 @@ class TVCache():
     def _checkItemAuth(self, title, url):
         return True
 
+    def getRSSFeed(self, url, post_data=None):
+        # create provider storaqe cache
+        storage = Shove('file://' + ek.ek(os.path.join, sickbeard.CACHE_DIR, self.providerID))
+        fc = cache.Cache(storage)
+
+        parsed = list(urlparse.urlparse(url))
+        parsed[2] = re.sub("/{2,}", "/", parsed[2])  # replace two or more / with one
+
+        if post_data:
+            url = url + 'api?' + urllib.urlencode(post_data)
+
+        f = fc.fetch(url)
+
+        if not f:
+            logger.log(u"Error loading " + self.providerID + " URL: " + url, logger.ERROR)
+            return None
+        elif 'error' in f.feed:
+            logger.log(u"Newznab ERROR:[%s] CODE:[%s]" % (f.feed['error']['description'], f.feed['error']['code']),
+                       logger.DEBUG)
+            return None
+        elif not f.entries:
+            logger.log(u"No items found on " + self.providerID + " using URL: " + url, logger.WARNING)
+            return None
+
+        storage.close()
+
+        return f
+
     def updateCache(self):
 
         if not self.shouldUpdate():
@@ -174,6 +209,7 @@ class TVCache():
     lastUpdate = property(_getLastUpdate)
 
     def shouldUpdate(self):
+        return True
         # if we've updated recently then skip the update
         if datetime.datetime.today() - self.lastUpdate < datetime.timedelta(minutes=self.minTime):
             logger.log(u"Last update was too soon, using old cache: today()-" + str(self.lastUpdate) + "<" + str(
@@ -184,8 +220,11 @@ class TVCache():
 
     def _addCacheEntry(self, name, url, quality=None):
 
-        season = None
-        episodes = None
+
+        cacheResult = sickbeard.name_cache.retrieveNameFromCache(name)
+        if cacheResult:
+            logger.log(u"Found Indexer ID:[" + repr(cacheResult) + "], using that for [" + str(name) + "}", logger.DEBUG)
+            return
 
         # if we don't have complete info then parse the filename to get it
         try:
@@ -208,12 +247,17 @@ class TVCache():
             logger.log(u"Could not find a show matching " + parse_result.series_name + " in the database, skipping ...", logger.DEBUG)
             return None
 
+        logger.log(u"Added RSS item: [" + name + "] to cache: [" + self.providerID + "]", logger.DEBUG)
+        sickbeard.name_cache.addNameToCache(name, showObj.indexerid)
+
+        season = episodes = None
         if parse_result.air_by_date:
             myDB = db.DBConnection()
 
             airdate = parse_result.air_date.toordinal()
-            sql_results = myDB.select("SELECT season, episode FROM tv_episodes WHERE showid = ? AND indexer = ? AND airdate = ?",
-                                      [showObj.indexerid, showObj.indexer, airdate])
+            sql_results = myDB.select(
+                "SELECT season, episode FROM tv_episodes WHERE showid = ? AND indexer = ? AND airdate = ?",
+                [showObj.indexerid, showObj.indexer, airdate])
             if sql_results > 0:
                 season = int(sql_results[0]["season"])
                 episodes = [int(sql_results[0]["episode"])]
@@ -235,11 +279,12 @@ class TVCache():
             if not isinstance(name, unicode):
                 name = unicode(name, 'utf-8')
 
-
             logger.log(u"Added RSS item: [" + name + "] to cache: [" + self.providerID + "]", logger.DEBUG)
-            return ["INSERT INTO [" + self.providerID + "] (name, season, episodes, indexerid, url, time, quality) VALUES (?,?,?,?,?,?,?)",
+            return [
+                "INSERT INTO [" + self.providerID + "] (name, season, episodes, indexerid, url, time, quality) VALUES (?,?,?,?,?,?,?)",
                 [name, season, episodeText, showObj.indexerid, url, curTimestamp, quality]]
 
+
     def searchCache(self, episode, manualSearch=False):
         neededEps = self.findNeededEpisodes(episode, manualSearch)
         return neededEps

tests/show name/show name - s04e02.mkv

@@ -0,0 +1 @@
+foo bar