diff --git a/couchpotato/core/database.py b/couchpotato/core/database.py index 09f2105..37841bf 100644 --- a/couchpotato/core/database.py +++ b/couchpotato/core/database.py @@ -32,6 +32,7 @@ class Database(object): addEvent('database.setup.after', self.startup_compact) addEvent('database.setup_index', self.setupIndex) + addEvent('database.delete_corrupted', self.deleteCorrupted) addEvent('app.migrate', self.migrate) addEvent('app.after_shutdown', self.close) @@ -147,6 +148,17 @@ class Database(object): return results + def deleteCorrupted(self, _id, traceback_error = ''): + + db = self.getDB() + + try: + log.debug('Deleted corrupted document "%s": %s', (_id, traceback_error)) + corrupted = db.get('id', _id, with_storage = False) + db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None) + except: + log.debug('Failed deleting corrupted: %s', traceback.format_exc()) + def reindex(self, **kwargs): success = True diff --git a/couchpotato/core/event.py b/couchpotato/core/event.py index 7246cde..35818e7 100644 --- a/couchpotato/core/event.py +++ b/couchpotato/core/event.py @@ -90,7 +90,7 @@ def fireEvent(name, *args, **kwargs): else: - e = Event(name = name, threads = 10, exc_info = True, traceback = True, lock = threading.RLock()) + e = Event(name = name, threads = 10, exc_info = True, traceback = True) for event in events[name]: e.handle(event['handler'], priority = event['priority']) diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index 4e319fc..549ed0d 100755 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -26,9 +26,9 @@ class MediaBase(Plugin): def onComplete(): try: media = fireEvent('media.get', media_id, single = True) - event_name = '%s.searcher.single' % media.get('type') - - fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id), manual = True) + if media: + event_name = '%s.searcher.single' % media.get('type') + fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id), manual = True) except: log.error('Failed creating onComplete: %s', traceback.format_exc()) @@ -39,9 +39,9 @@ class MediaBase(Plugin): def notifyFront(): try: media = fireEvent('media.get', media_id, single = True) - event_name = '%s.update' % media.get('type') - - fireEvent('notify.frontend', type = event_name, data = media) + if media: + event_name = '%s.update' % media.get('type') + fireEvent('notify.frontend', type = event_name, data = media) except: log.error('Failed creating onComplete: %s', traceback.format_exc()) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index d5e54d7..ccee1fe 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -178,8 +178,10 @@ class MediaPlugin(MediaBase): continue yield doc - except RecordNotFound: + except (RecordDeleted, RecordNotFound): log.debug('Record not found, skipping: %s', ms['_id']) + except (ValueError, EOFError): + fireEvent('database.delete_corrupted', ms.get('_id'), traceback_error = traceback.format_exc(0)) else: yield ms @@ -280,6 +282,10 @@ class MediaPlugin(MediaBase): media = fireEvent('media.get', media_id, single = True) + # Skip if no media has been found + if not media: + continue + # Merge releases with movie dict medias.append(media) diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 0dc39e5..1863d89 100755 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -179,6 +179,9 @@ class MovieBase(MovieTypeBase): db.delete(rel) movie_dict = fireEvent('media.get', m['_id'], single = True) + if not movie_dict: + log.debug('Failed adding media, can\'t find it anymore') + return False if do_search and search_after: onComplete = self.createOnComplete(m['_id']) @@ -268,6 +271,10 @@ class MovieBase(MovieTypeBase): if self.shuttingDown(): return + lock_key = 'media.get.%s' % media_id if media_id else identifier + self.acquireLock(lock_key) + + media = {} try: db = get_db() @@ -316,11 +323,11 @@ class MovieBase(MovieTypeBase): self.getPoster(media, image_urls) db.update(media) - return media except: log.error('Failed update media: %s', traceback.format_exc()) - return {} + self.releaseLock(lock_key) + return media def updateReleaseDate(self, media_id): """ diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index 97181ae..7ebaaf5 100755 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -89,6 +89,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): for media_id in medias: media = fireEvent('media.get', media_id, single = True) + if not media: continue try: self.single(media, search_protocols, manual = manual) @@ -388,9 +389,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase): rel['status'] = 'ignored' db.update(rel) - movie_dict = fireEvent('media.get', media_id, single = True) - log.info('Trying next release for: %s', getTitle(movie_dict)) - self.single(movie_dict, manual = manual, force_download = force_download) + media = fireEvent('media.get', media_id, single = True) + if media: + log.info('Trying next release for: %s', getTitle(media)) + self.single(media, manual = manual, force_download = force_download) return True diff --git a/couchpotato/core/plugins/automation.py b/couchpotato/core/plugins/automation.py index 39d7c9e..e98a00a 100644 --- a/couchpotato/core/plugins/automation.py +++ b/couchpotato/core/plugins/automation.py @@ -46,7 +46,8 @@ class Automation(Plugin): break movie_dict = fireEvent('media.get', movie_id, single = True) - fireEvent('movie.searcher.single', movie_dict) + if movie_dict: + fireEvent('movie.searcher.single', movie_dict) return True diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index ecc77b3..73bc935 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -1,3 +1,4 @@ +import threading from urllib import quote from urlparse import urlparse import glob @@ -35,6 +36,8 @@ class Plugin(object): _needs_shutdown = False _running = None + _locks = {} + user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20130519 Firefox/24.0' http_last_use = {} http_time_between_calls = 0 @@ -400,3 +403,19 @@ class Plugin(object): def isEnabled(self): return self.conf(self.enabled_option) or self.conf(self.enabled_option) is None + + def acquireLock(self, key): + + lock = self._locks.get(key) + if not lock: + self._locks[key] = threading.RLock() + + log.debug('Acquiring lock: %s', key) + self._locks.get(key).acquire() + + def releaseLock(self, key): + + lock = self._locks.get(key) + if lock: + log.debug('Releasing lock: %s', key) + self._locks.get(key).release() diff --git a/couchpotato/core/plugins/dashboard.py b/couchpotato/core/plugins/dashboard.py index d4af7ad..afead44 100644 --- a/couchpotato/core/plugins/dashboard.py +++ b/couchpotato/core/plugins/dashboard.py @@ -1,5 +1,6 @@ import random as rndm import time +from CodernityDB.database import RecordDeleted from couchpotato import get_db from couchpotato.api import addApiView @@ -58,7 +59,11 @@ class Dashboard(Plugin): rndm.shuffle(active_ids) for media_id in active_ids: - media = db.get('id', media_id) + try: + media = db.get('id', media_id) + except RecordDeleted: + log.debug('Record already deleted: %s', media_id) + continue pp = profile_pre.get(media.get('profile_id')) if not pp: continue diff --git a/couchpotato/core/plugins/manage.py b/couchpotato/core/plugins/manage.py index 132f7ec..75c550b 100755 --- a/couchpotato/core/plugins/manage.py +++ b/couchpotato/core/plugins/manage.py @@ -234,7 +234,8 @@ class Manage(Plugin): total = self.in_progress[folder]['total'] movie_dict = fireEvent('media.get', identifier, single = True) - fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict)) + if movie_dict: + fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict)) return afterUpdate diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index b8417c3..cc92b9f 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -95,9 +95,7 @@ class Release(Plugin): except: log.error('Failed fixing mis-status tag: %s', traceback.format_exc()) except ValueError: - log.debug('Deleted corrupted document "%s": %s', (release.get('key'), traceback.format_exc(0))) - corrupted = db.get('id', release.get('key'), with_storage = False) - db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None) + fireEvent('database.delete_corrupted', release.get('key'), traceback_error = traceback.format_exc(0)) reindex += 1 except RecordDeleted: db.delete(doc) @@ -112,7 +110,7 @@ class Release(Plugin): del media_exist # get movies last_edit more than a week ago - medias = fireEvent('media.with_status', ['done','active'], single = True) + medias = fireEvent('media.with_status', ['done', 'active'], single = True) for media in medias: if media.get('last_edit', 0) > (now - week):