Improved manage scanning

Expire after db get

couchpotato/core/helpers/variable.py

@@ -278,4 +278,4 @@ def dictIsSubset(a, b):
 
 def isSubFolder(sub_folder, base_folder):
     # Returns True if sub_folder is the same as or inside base_folder
-    return base_folder and sub_folder and os.path.normpath(base_folder).rstrip(os.path.sep) + os.path.sep in os.path.normpath(sub_folder).rstrip(os.path.sep) + os.path.sep
+    return base_folder and sub_folder and ss(os.path.normpath(base_folder).rstrip(os.path.sep) + os.path.sep) in ss(os.path.normpath(sub_folder).rstrip(os.path.sep) + os.path.sep)

couchpotato/core/logger.py

@@ -38,7 +38,7 @@ class CPLog(object):
     def safeMessage(self, msg, replace_tuple = ()):
 
         from couchpotato.environment import Env
-        from couchpotato.core.helpers.encoding import ss
+        from couchpotato.core.helpers.encoding import ss, toUnicode
 
         msg = ss(msg)
 
@@ -67,4 +67,4 @@ class CPLog(object):
             except:
                 pass
 
-        return msg
+        return toUnicode(msg)

couchpotato/core/media/_base/media/main.py

@@ -70,8 +70,6 @@ class MediaPlugin(MediaBase):
         addEvent('media.restatus', self.restatus)
 
     def refresh(self, id = '', **kwargs):
-        db = get_session()
-
         handlers = []
         ids = splitString(id)
 
@@ -97,11 +95,14 @@ class MediaPlugin(MediaBase):
 
             default_title = getTitle(media.library)
             identifier = media.library.identifier
-            db.expire_all()
+            event = 'library.update.%s' % media.type
 
             def handler():
-                fireEvent('library.update.%s' % media.type, identifier = identifier, default_title = default_title, force = True, on_complete = self.createOnComplete(id))
+                fireEvent(event, identifier = identifier, default_title = default_title, on_complete = self.createOnComplete(id))
 
+        db.expire_all()
+
+        if handler:
             return handler
 
     def addSingleRefreshView(self):
@@ -467,6 +468,7 @@ class MediaPlugin(MediaBase):
             m.status_id = active_status.get('id') if move_to_wanted else done_status.get('id')
 
         db.commit()
+        db.expire_all()
 
         return True
 

couchpotato/core/media/movie/library/movie/main.py

@@ -60,7 +60,7 @@ class MovieLibraryPlugin(LibraryBase):
         db.expire_all()
         return library_dict
 
-    def update(self, identifier, default_title = '', force = False):
+    def update(self, identifier, default_title = '', extended = False):
 
         if self.shuttingDown():
             return
@@ -69,13 +69,7 @@ class MovieLibraryPlugin(LibraryBase):
         library = db.query(Library).filter_by(identifier = identifier).first()
         done_status = fireEvent('status.get', 'done', single = True)
 
-        library_dict = None
-        if library:
-            library_dict = library.to_dict(self.default_dict)
-
-        do_update = True
-
-        info = fireEvent('movie.info', merge = True, identifier = identifier)
+        info = fireEvent('movie.info', merge = True, extended = extended, identifier = identifier)
 
         # Don't need those here
         try: del info['in_wanted']
@@ -88,55 +82,54 @@ class MovieLibraryPlugin(LibraryBase):
             return False
 
         # Main info
-        if do_update:
-            library.plot = toUnicode(info.get('plot', ''))
-            library.tagline = toUnicode(info.get('tagline', ''))
-            library.year = info.get('year', 0)
-            library.status_id = done_status.get('id')
-            library.info.update(info)
-            db.commit()
+        library.plot = toUnicode(info.get('plot', ''))
+        library.tagline = toUnicode(info.get('tagline', ''))
+        library.year = info.get('year', 0)
+        library.status_id = done_status.get('id')
+        library.info.update(info)
+        db.commit()
+
+        # Titles
+        [db.delete(title) for title in library.titles]
+        db.commit()
+
+        titles = info.get('titles', [])
+        log.debug('Adding titles: %s', titles)
+        counter = 0
+        for title in titles:
+            if not title:
+                continue
+            title = toUnicode(title)
+            t = LibraryTitle(
+                title = title,
+                simple_title = self.simplifyTitle(title),
+                default = (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title)
+            )
+            library.titles.append(t)
+            counter += 1
 
-            # Titles
-            [db.delete(title) for title in library.titles]
-            db.commit()
+        db.commit()
 
-            titles = info.get('titles', [])
-            log.debug('Adding titles: %s', titles)
-            counter = 0
-            for title in titles:
-                if not title:
+        # Files
+        images = info.get('images', [])
+        for image_type in ['poster']:
+            for image in images.get(image_type, []):
+                if not isinstance(image, (str, unicode)):
                     continue
-                title = toUnicode(title)
-                t = LibraryTitle(
-                    title = title,
-                    simple_title = self.simplifyTitle(title),
-                    default = (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title)
-                )
-                library.titles.append(t)
-                counter += 1
-
-            db.commit()
-
-            # Files
-            images = info.get('images', [])
-            for image_type in ['poster']:
-                for image in images.get(image_type, []):
-                    if not isinstance(image, (str, unicode)):
-                        continue
 
-                    file_path = fireEvent('file.download', url = image, single = True)
-                    if file_path:
-                        file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', image_type), single = True)
-                        try:
-                            file_obj = db.query(File).filter_by(id = file_obj.get('id')).one()
-                            library.files.append(file_obj)
-                            db.commit()
+                file_path = fireEvent('file.download', url = image, single = True)
+                if file_path:
+                    file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', image_type), single = True)
+                    try:
+                        file_obj = db.query(File).filter_by(id = file_obj.get('id')).one()
+                        library.files.append(file_obj)
+                        db.commit()
 
-                            break
-                        except:
-                            log.debug('Failed to attach to library: %s', traceback.format_exc())
+                        break
+                    except:
+                        log.debug('Failed to attach to library: %s', traceback.format_exc())
 
-            library_dict = library.to_dict(self.default_dict)
+        library_dict = library.to_dict(self.default_dict)
 
         db.expire_all()
         return library_dict
@@ -147,14 +140,14 @@ class MovieLibraryPlugin(LibraryBase):
         library = db.query(Library).filter_by(identifier = identifier).first()
 
         if not library.info:
-            library_dict = self.update(identifier, force = True)
+            library_dict = self.update(identifier)
             dates = library_dict.get('info', {}).get('release_date')
         else:
             dates = library.info.get('release_date')
 
         if dates and (dates.get('expires', 0) < time.time() or dates.get('expires', 0) > time.time() + (604800 * 4)) or not dates:
             dates = fireEvent('movie.release_date', identifier = identifier, merge = True)
-            library.info.update({'release_date': dates })
+            library.info.update({'release_date': dates})
             db.commit()
 
         db.expire_all()

couchpotato/core/media/movie/searcher/main.py

@@ -73,11 +73,24 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
 
         db = get_session()
 
-        movies = db.query(Media).filter(
+        movies_raw = db.query(Media).filter(
             Media.status.has(identifier = 'active')
         ).all()
+
         random.shuffle(movies)
 
+        movies = []
+        for m in movies_raw:
+            movies.append(m.to_dict({
+                'category': {},
+                'profile': {'types': {'quality': {}}},
+                'releases': {'status': {}, 'quality': {}},
+                'library': {'titles': {}, 'files': {}},
+                'files': {},
+            }))
+
+        db.expire_all()
+
         self.in_progress = {
             'total': len(movies),
             'to_go': len(movies),
@@ -87,21 +100,14 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
             search_protocols = fireEvent('searcher.protocols', single = True)
 
             for movie in movies:
-                movie_dict = movie.to_dict({
-                    'category': {},
-                    'profile': {'types': {'quality': {}}},
-                    'releases': {'status': {}, 'quality': {}},
-                    'library': {'titles': {}, 'files': {}},
-                    'files': {},
-                })
 
                 try:
-                    self.single(movie_dict, search_protocols)
+                    self.single(movie, search_protocols)
                 except IndexError:
-                    log.error('Forcing library update for %s, if you see this often, please report: %s', (movie_dict['library']['identifier'], traceback.format_exc()))
-                    fireEvent('library.update.movie', movie_dict['library']['identifier'], force = True)
+                    log.error('Forcing library update for %s, if you see this often, please report: %s', (movie['library']['identifier'], traceback.format_exc()))
+                    fireEvent('library.update.movie', movie['library']['identifier'])
                 except:
-                    log.error('Search failed for %s: %s', (movie_dict['library']['identifier'], traceback.format_exc()))
+                    log.error('Search failed for %s: %s', (movie['library']['identifier'], traceback.format_exc()))
 
                 self.in_progress['to_go'] -= 1
 
@@ -133,8 +139,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
             log.debug('Movie doesn\'t have a profile or already done, assuming in manage tab.')
             return
 
-        db = get_session()
-
         pre_releases = fireEvent('quality.pre_releases', single = True)
         release_dates = fireEvent('library.update.movie.release_date', identifier = movie['library']['identifier'], merge = True)
         available_status, ignored_status, failed_status = fireEvent('status.get', ['available', 'ignored', 'failed'], single = True)
@@ -150,6 +154,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
 
         fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'id': movie['id']}, message = 'Searching for "%s"' % default_title)
 
+        db = get_session()
 
         ret = False
         for quality_type in movie['profile']['types']:
@@ -204,6 +209,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
 
         fireEvent('notify.frontend', type = 'movie.searcher.ended', data = {'id': movie['id']})
 
+        db.expire_all()
         return ret
 
     def correctRelease(self, nzb = None, media = None, quality = None, **kwargs):
@@ -336,6 +342,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
             for rel in rels:
                 rel.status_id = ignored_status.get('id')
             db.commit()
+            db.expire_all()
 
             movie_dict = fireEvent('media.get', media_id = media_id, single = True)
             log.info('Trying next release for: %s', getTitle(movie_dict['library']))

couchpotato/core/media/movie/suggestion/main.py

@@ -33,6 +33,7 @@ class Suggestion(Plugin):
                     .options(joinedload_all('library')) \
                     .filter(or_(*[Media.status.has(identifier = s) for s in ['active', 'done']])).all()
                 movies = [x.library.identifier for x in active_movies]
+                db.expire_all()
 
             if not ignored or len(ignored) == 0:
                 ignored = splitString(Env.prop('suggest_ignore', default = ''))
@@ -97,6 +98,7 @@ class Suggestion(Plugin):
                 .filter(Media.status_id.in_([active_status.get('id'), done_status.get('id')])).all()
             movies = [x[0] for x in active_movies]
             movies.extend(seen)
+            db.expire_all()
 
             ignored.extend([x.get('imdb') for x in cached_suggestion])
             suggestions = fireEvent('movie.suggest', movies = movies, ignore = removeDuplicate(ignored), single = True)

couchpotato/core/plugins/category/main.py

@@ -119,3 +119,5 @@ class CategoryPlugin(Plugin):
             for movie in movies:
                 movie.category_id = None
                 db.commit()
+
+        db.expire_all()

couchpotato/core/plugins/dashboard/main.py

@@ -119,6 +119,7 @@ class Dashboard(Plugin):
                         'files': {},
                     }))
 
+        db.expire_all()
         return {
             'success': True,
             'empty': len(movies) == 0,

couchpotato/core/plugins/manage/main.py

@@ -1,7 +1,7 @@
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, addEvent, fireEventAsync
 from couchpotato.core.helpers.encoding import sp
-from couchpotato.core.helpers.variable import splitString, getTitle
+from couchpotato.core.helpers.variable import splitString, getTitle, tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.environment import Env
@@ -59,6 +59,7 @@ class Manage(Plugin):
         fireEventAsync('manage.update', full = True if full == '1' else False)
 
         return {
+            'progress': self.in_progress,
             'success': True
         }
 
@@ -84,15 +85,17 @@ class Manage(Plugin):
             added_identifiers = []
 
             # Add some progress
-            self.in_progress = {}
             for directory in directories:
                 self.in_progress[os.path.normpath(directory)] = {
+                    'started': False,
+                    'eta': -1,
                     'total': None,
                     'to_go': None,
                 }
 
             for directory in directories:
                 folder = os.path.normpath(directory)
+                self.in_progress[os.path.normpath(directory)]['started'] = tryInt(time.time())
 
                 if not os.path.isdir(folder):
                     if len(directory) > 0:
@@ -102,6 +105,7 @@ class Manage(Plugin):
                 log.info('Updating manage library: %s', folder)
                 fireEvent('notify.frontend', type = 'manage.update', data = True, message = 'Scanning for movies in "%s"' % folder)
 
+
                 onFound = self.createAddToLibrary(folder, added_identifiers)
                 fireEvent('scanner.scan', folder = folder, simple = True, newer_than = last_update if not full else 0, on_found = onFound, single = True)
 
@@ -175,10 +179,10 @@ class Manage(Plugin):
 
         def addToLibrary(group, total_found, to_go):
             if self.in_progress[folder]['total'] is None:
-                self.in_progress[folder] = {
+                self.in_progress[folder].update({
                     'total': total_found,
                     'to_go': total_found,
-                }
+                })
 
             if group['library'] and group['library'].get('identifier'):
                 identifier = group['library'].get('identifier')
@@ -186,9 +190,9 @@ class Manage(Plugin):
 
                 # Add it to release and update the info
                 fireEvent('release.add', group = group)
-                fireEventAsync('library.update.movie', identifier = identifier, on_complete = self.createAfterUpdate(folder, identifier))
+                fireEvent('library.update.movie', identifier = identifier, on_complete = self.createAfterUpdate(folder, identifier))
             else:
-                self.in_progress[folder]['to_go'] -= 1
+                self.updateProgress(folder)
 
         return addToLibrary
 
@@ -199,7 +203,7 @@ class Manage(Plugin):
             if not self.in_progress or self.shuttingDown():
                 return
 
-            self.in_progress[folder]['to_go'] -= 1
+            self.updateProgress(folder)
             total = self.in_progress[folder]['total']
             movie_dict = fireEvent('media.get', identifier, single = True)
 
@@ -207,6 +211,15 @@ class Manage(Plugin):
 
         return afterUpdate
 
+    def updateProgress(self, folder):
+
+        pr = self.in_progress[folder]
+        pr['to_go'] -= 1
+
+        avg = (time.time() - pr['started'])/(pr['total'] - pr['to_go'])
+        pr['eta'] = tryInt(avg * pr['to_go'])
+
+
     def directories(self):
         try:
             if self.conf('library', default = '').strip():

couchpotato/core/plugins/profile/main.py

@@ -46,6 +46,8 @@ class ProfilePlugin(Plugin):
                 movie.profile_id = default_profile.get('id')
                 db.commit()
 
+        db.expire_all()
+
     def allView(self, **kwargs):
 
         return {

couchpotato/core/plugins/release/main.py

@@ -143,6 +143,7 @@ class Release(Plugin):
 
         fireEvent('media.restatus', media.id)
 
+        db.expire_all()
         return True
 
     def saveFile(self, filepath, type = 'unknown', include_media_info = False):
@@ -338,6 +339,8 @@ class Release(Plugin):
         except:
             log.error('Failed storing download status: %s', traceback.format_exc())
             return False
+        finally:
+            db.expire_all()
 
         return True
 
@@ -466,4 +469,5 @@ class Release(Plugin):
             #Update all movie info as there is no release update function
             fireEvent('notify.frontend', type = 'release.update_status', data = rel.to_dict())
 
+        db.expire_all()
         return True

couchpotato/core/plugins/scanner/main.py

@@ -134,6 +134,8 @@ class Scanner(Plugin):
 
             except:
                 log.error('Failed getting files from %s: %s', (folder, traceback.format_exc()))
+
+            log.debug('Found %s files to scan and group in %s', (len(files), folder))
         else:
             check_file_date = False
             files = [sp(x) for x in files]
@@ -421,6 +423,7 @@ class Scanner(Plugin):
             else:
                 movie = db.query(Media).filter_by(library_id = group['library']['id']).first()
                 group['movie_id'] = None if not movie else movie.id
+                db.expire_all()
 
             processed_movies[identifier] = group
 
@@ -604,6 +607,7 @@ class Scanner(Plugin):
                     break
                 except:
                     pass
+            db.expire_all()
 
         # Search based on identifiers
         if not imdb_id:
@@ -637,7 +641,7 @@ class Scanner(Plugin):
         try:
             m = re.search(self.cp_imdb, string.lower())
             id = m.group('id')
-            if id:  return id
+            if id: return id
         except AttributeError:
             pass
 

couchpotato/core/providers/metadata/base.py

@@ -25,7 +25,7 @@ class MetaDataBase(Plugin):
 
         # Update library to get latest info
         try:
-            updated_library = fireEvent('library.update.movie', group['library']['identifier'], force = True, single = True)
+            updated_library = fireEvent('library.update.movie', group['library']['identifier'], extended = True, single = True)
             group['library'] = mergeDicts(group['library'], updated_library)
         except:
             log.error('Failed to update movie, before creating metadata: %s', traceback.format_exc())

couchpotato/environment.py

@@ -67,7 +67,7 @@ class Env(object):
 
     @staticmethod
     def getEngine():
-        return create_engine(Env.get('db_path'), echo = False, pool_recycle = 30)
+        return create_engine(Env.get('db_path'), echo = False)
 
     @staticmethod
     def setting(attr, section = 'core', value = None, default = '', type = None):

couchpotato/static/scripts/page/manage.js

@@ -119,7 +119,9 @@ Page.Manage = new Class({
 						sorted_table.each(function(folder){
 							var folder_progress = progress[folder]
 							new Element('div').adopt(
-								new Element('span.folder', {'text': folder}),
+								new Element('span.folder', {'text': folder +
+									(folder_progress.eta > 0 ? ', ' + new Date ().increment('second', folder_progress.eta).timeDiffInWords().replace('from now', 'to go') : '')
+								}),
 								new Element('span.percentage', {'text': folder_progress.total ? (((folder_progress.total-folder_progress.to_go)/folder_progress.total)*100).round() + '%' : '0%'})
 							).inject(self.progress_container)
 						});