Improved folder scanning

couchpotato/core/plugins/manage/main.py

@@ -14,8 +14,6 @@ log = CPLog(__name__)
 
 class Manage(Plugin):
 
-    last_update = 0
-
     def __init__(self):
 
         fireEvent('scheduler.interval', identifier = 'manage.update_library', handle = self.updateLibrary, hours = 2)
@@ -42,12 +40,14 @@ class Manage(Plugin):
         })
 
 
-    def updateLibrary(self, full = False):
+    def updateLibrary(self, full = True):
+        last_update = float(Env.prop('manage.last_update'))
 
-        if self.isDisabled() or (self.last_update > time.time() - 20):
+        if self.isDisabled() or (last_update > time.time() - 20):
             return
 
         directories = self.directories()
+        added_identifiers = []
 
         for directory in directories:
 
@@ -57,21 +57,24 @@ class Manage(Plugin):
                 continue
 
             log.info('Updating manage library: %s' % directory)
-            fireEvent('scanner.folder', folder = directory)
+            identifiers = fireEvent('scanner.folder', folder = directory, newer_than = last_update, single = True)
-
+            added_identifiers.extend(identifiers)
-            # If cleanup option is enabled, remove offline files from database
-            if self.conf('cleanup'):
-                db = get_session()
-                files_in_path = db.query(File).filter(File.path.like(directory + '%%')).filter_by(available = 0).all()
-                [db.delete(x) for x in files_in_path]
-                db.commit()
-                db.remove()
 
             # Break if CP wants to shut down
             if self.shuttingDown():
                 break
 
-        self.last_update = time.time()
+        # If cleanup option is enabled, remove offline files from database
+        if self.conf('cleanup') and full and not self.shuttingDown():
+
+            # Get movies with done status
+            done_movies = fireEvent('movie.list', status = 'done', single = True)
+
+            for done_movie in done_movies:
+                if done_movie['library']['identifier'] not in added_identifiers:
+                    fireEvent('movie.delete', movie_id = done_movie['id'])
+
+        Env.prop('manage.last_update', time.time())
 
     def directories(self):
         try:

couchpotato/core/plugins/scanner/main.py

@@ -7,6 +7,7 @@ from couchpotato.core.plugins.base import Plugin
 from couchpotato.core.settings.model import File
 from couchpotato.environment import Env
 from enzyme.exceptions import NoParserError, ParseError
+from guessit import guess_movie_info
 from subliminal.videos import scan
 import enzyme
 import logging
@@ -96,6 +97,10 @@ class Scanner(Plugin):
 
         addEvent('rename.after', after_rename)
 
+        # Disable lib logging
+        logging.getLogger('guessit').setLevel(logging.ERROR)
+        logging.getLogger('subliminal').setLevel(logging.ERROR)
+
     def scanFilesToLibrary(self, folder = None, files = None):
 
         groups = self.scan(folder = folder, files = files)
@@ -104,21 +109,14 @@ class Scanner(Plugin):
             if group['library']:
                 fireEvent('release.add', group = group)
 
-    def scanFolderToLibrary(self, folder = None, newer_as = None):
+    def scanFolderToLibrary(self, folder = None, newer_than = None):
 
         if not os.path.isdir(folder):
             return
 
         groups = self.scan(folder = folder)
 
-        # Open up the db
+        added_identifier = []
-        db = get_session()
-
-        # Mark all files as "offline" before a adding them to the database (again)
-        files_in_path = db.query(File).filter(File.path.like(toUnicode(folder) + u'%%'))
-        files_in_path.update({'available': 0}, synchronize_session = False)
-        db.commit()
-
         while True and not self.shuttingDown():
             try:
                 identifier, group = groups.popitem()
@@ -130,9 +128,10 @@ class Scanner(Plugin):
 
                 # Add release
                 fireEvent('release.add', group = group)
-                fireEvent('library.update', identifier = group['library'].get('identifier'))
+                library_item = fireEvent('library.update', identifier = group['library'].get('identifier'), single = True)
+                added_identifier.append(library_item['identifier'])
 
-        db.remove()
+        return added_identifier
 
 
     def scan(self, folder = None, files = []):
@@ -179,7 +178,7 @@ class Scanner(Plugin):
                 identifier = self.createStringIdentifier(file_path, folder, exclude_filename = is_dvd_file)
 
                 # Identifier with quality
-                quality = fireEvent('quality.guess', [file_path], single = True)
+                quality = fireEvent('quality.guess', [file_path], single = True) if not is_dvd_file else {'identifier':'dvdr'}
                 identifier_with_quality = '%s %s' % (identifier, quality.get('identifier', ''))
 
                 if not movie_files.get(identifier):
@@ -393,6 +392,7 @@ class Scanner(Plugin):
             paths = group['files']['movie']
             scan_result = []
             for p in paths:
+                if not group['is_dvd']:
                     scan_result.extend(scan(p))
 
             for video, detected_subtitles in scan_result:
@@ -400,7 +400,7 @@ class Scanner(Plugin):
                     if s.language and s.path not in paths:
                         detected_languages[s.path] = [s.language]
         except:
-            log.error('Failed parsing subtitle languages for %s: %s' % (paths, traceback.format_exc()))
+            log.debug('Failed parsing subtitle languages for %s: %s' % (paths, traceback.format_exc()))
 
         # IDX
         for extra in group['files']['subtitle_extra']:
@@ -472,7 +472,9 @@ class Scanner(Plugin):
             for identifier in group['identifiers']:
 
                 if len(identifier) > 2:
-                    movie = fireEvent('movie.search', q = '%(name)s %(year)s' % self.getReleaseNameYear(identifier), merge = True, limit = 1)
+                    try: filename = list(group['files'].get('movie'))[0]
+                    except: filename = None
+                    movie = fireEvent('movie.search', q = '%(name)s %(year)s' % self.getReleaseNameYear(identifier, file_name = filename), merge = True, limit = 1)
 
                     if len(movie) > 0:
                         imdb_id = movie[0]['imdb']
@@ -690,7 +692,18 @@ class Scanner(Plugin):
 
         return ''
 
-    def getReleaseNameYear(self, release_name):
+    def getReleaseNameYear(self, release_name, file_name = None):
+
+        # Use guessit first
+        if file_name:
+            guess = guess_movie_info(file_name)
+            if guess.get('title') and guess.get('year'):
+                return {
+                    'name': guess.get('title'),
+                    'year': guess.get('year'),
+                }
+
+        # Backup to simple
         cleaned = ' '.join(re.split('\W+', simplifyString(release_name)))
         cleaned = re.sub(self.clean, ' ', cleaned)
         year = self.findYear(cleaned)