Progress bars for manage updates

contribute.md

@@ -0,0 +1 @@
+So you feel like

couchpotato/core/plugins/manage/main.py

@@ -2,22 +2,33 @@ from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, addEvent, fireEventAsync
 from couchpotato.core.helpers.encoding import ss
 from couchpotato.core.helpers.request import jsonified, getParam
+from couchpotato.core.helpers.variable import getTitle
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.environment import Env
 import os
 import time
+import traceback
 
 
 log = CPLog(__name__)
 
 class Manage(Plugin):
 
+    in_progress = False
+
     def __init__(self):
 
         fireEvent('scheduler.interval', identifier = 'manage.update_library', handle = self.updateLibrary, hours = 2)
 
         addEvent('manage.update', self.updateLibrary)
+        addEvent('manage.scan_files', self.scanFilesToLibrary)
+
+        # Add files after renaming
+        def after_rename(group):
+            return self.scanFilesToLibrary(folder = group['destination_dir'], files = group['renamed_files'])
+        addEvent('renamer.after', after_rename)
+
         addApiView('manage.update', self.updateLibraryView, docs = {
             'desc': 'Update the library by scanning for new movies',
             'params': {
@@ -25,11 +36,23 @@ class Manage(Plugin):
             }
         })
 
+        addApiView('manage.progress', self.getProgress, docs = {
+            'desc': 'Get the progress of current manage update',
+            'return': {'type': 'object', 'example': """{
+    'progress': False || object, total & to_go,
+}"""},
+        })
+
         if not Env.get('dev'):
             def updateLibrary():
                 self.updateLibrary(full = False)
             addEvent('app.load', updateLibrary)
 
+    def getProgress(self):
+        return jsonified({
+            'progress': self.in_progress
+        })
+
     def updateLibraryView(self):
 
         full = getParam('full', default = 1)
@@ -43,49 +66,127 @@ class Manage(Plugin):
     def updateLibrary(self, full = True):
         last_update = float(Env.prop('manage.last_update', default = 0))
 
-        if self.isDisabled() or (last_update > time.time() - 20):
+        if self.in_progress:
+            log.info('Already updating library: %s', self.in_progress)
+            return
+        elif self.isDisabled() or (last_update > time.time() - 20):
             return
 
-        directories = self.directories()
+        self.in_progress = {}
-        added_identifiers = []
+        fireEvent('notify.frontend', type = 'manage.updating', data = True)
 
-        for directory in directories:
+        try:
+
+            directories = self.directories()
+            added_identifiers = []
+
+            # Add some progress
+            self.in_progress = {}
+            for directory in directories:
+                self.in_progress[os.path.normpath(directory)] = {
+                    'total': None,
+                    'to_go': None,
+                }
+
+            for directory in directories:
+                folder = os.path.normpath(directory)
+
+                if not os.path.isdir(folder):
+                    if len(directory) > 0:
+                        log.error('Directory doesn\'t exist: %s', folder)
+                    continue
+
+                log.info('Updating manage library: %s', folder)
+                fireEvent('notify.frontend', type = 'manage.update', data = True, message = 'Scanning for movies in "%s"' % folder)
+
+                onFound = self.createAddToLibrary(folder, added_identifiers)
+                fireEvent('scanner.scan', folder = folder, simple = True, newer_than = last_update if not full else 0, on_found = onFound, single = True)
+
+                # Break if CP wants to shut down
+                if self.shuttingDown():
+                    break
+
+            # If cleanup option is enabled, remove offline files from database
+            if self.conf('cleanup') and full and not self.shuttingDown():
+
+                # Get movies with done status
+                total_movies, done_movies = fireEvent('movie.list', status = 'done', single = True)
+
+                for done_movie in done_movies:
+                    if done_movie['library']['identifier'] not in added_identifiers:
+                        fireEvent('movie.delete', movie_id = done_movie['id'], delete_from = 'all')
+                    else:
+                        for release in done_movie.get('releases', []):
+                            for release_file in release.get('files', []):
+                                # Remove release not available anymore
+                                if not os.path.isfile(ss(release_file['path'])):
+                                    fireEvent('release.clean', release['id'])
+                                    break
+
+            Env.prop('manage.last_update', time.time())
+        except:
+            log.error('Failed updating library: %s', (traceback.format_exc()))
 
-            if not os.path.isdir(directory):
+        while True and not self.shuttingDown():
-                if len(directory) > 0:
-                    log.error('Directory doesn\'t exist: %s', directory)
-                continue
 
-            log.info('Updating manage library: %s', directory)
+            delete_me = {}
-            identifiers = fireEvent('scanner.folder', folder = directory, newer_than = last_update if not full else 0, single = True)
-            if identifiers:
-                added_identifiers.extend(identifiers)
 
-            # Break if CP wants to shut down
+            for folder in self.in_progress:
-            if self.shuttingDown():
+                if self.in_progress[folder]['to_go'] <= 0:
+                    delete_me[folder] = True
+
+            for delete in delete_me:
+                del self.in_progress[delete]
+
+            if len(self.in_progress) == 0:
                 break
 
-        # If cleanup option is enabled, remove offline files from database
+            time.sleep(1)
-        if self.conf('cleanup') and full and not self.shuttingDown():
+
+        fireEvent('notify.frontend', type = 'manage.updating', data = False)
+        self.in_progress = False
+
+    def createAddToLibrary(self, folder, added_identifiers = []):
+        def addToLibrary(group, total_found, to_go):
+            if self.in_progress[folder]['total'] is None:
+                self.in_progress[folder] = {
+                    'total': total_found,
+                    'to_go': total_found,
+                }
+
+            if group['library']:
+                identifier = group['library'].get('identifier')
+                added_identifiers.append(identifier)
+
+                # Add it to release and update the info
+                fireEvent('release.add', group = group)
+                fireEventAsync('library.update', identifier = identifier, on_complete = self.createAfterUpdate(folder, identifier))
 
-            # Get movies with done status
+        return addToLibrary
-            total_movies, done_movies = fireEvent('movie.list', status = 'done', single = True)
 
-            for done_movie in done_movies:
+    def createAfterUpdate(self, folder, identifier):
-                if done_movie['library']['identifier'] not in added_identifiers:
-                    fireEvent('movie.delete', movie_id = done_movie['id'], delete_from = 'all')
-                else:
-                    for release in done_movie.get('releases', []):
-                        for release_file in release.get('files', []):
-                            # Remove release not available anymore
-                            if not os.path.isfile(ss(release_file['path'])):
-                                fireEvent('release.clean', release['id'])
-                                break
 
-        Env.prop('manage.last_update', time.time())
+        # Notify frontend
+        def afterUpdate():
+            self.in_progress[folder]['to_go'] = self.in_progress[folder]['to_go'] - 1
+            total = self.in_progress[folder]['total']
+            movie_dict = fireEvent('movie.get', identifier, single = True)
+            fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict['library']))
+
+        return afterUpdate
 
     def directories(self):
         try:
             return [x.strip() for x in self.conf('library', default = '').split('::')]
         except:
             return []
+
+    def scanFilesToLibrary(self, folder = None, files = None):
+
+        folder = os.path.normpath(folder)
+
+        groups = fireEvent('scanner.scan', folder = folder, files = files, single = True)
+
+        for group in groups.itervalues():
+            if group['library']:
+                fireEvent('release.add', group = group)

couchpotato/core/plugins/movie/main.py

@@ -107,13 +107,18 @@ class MoviePlugin(Plugin):
     def get(self, movie_id):
 
         db = get_session()
-        m = db.query(Movie).filter_by(id = movie_id).first()
+
+        imdb_id = getImdb(str(movie_id))
+
+        if(imdb_id):
+            m = db.query(Movie).filter(Movie.library.has(identifier = imdb_id)).first()
+        else:
+            m = db.query(Movie).filter_by(id = movie_id).first()
 
         results = None
         if m:
             results = m.to_dict(self.default_dict)
 
-        #db.close()
         return results
 
     def list(self, status = ['active'], limit_offset = None, starts_with = None, search = None):

couchpotato/core/plugins/movie/static/list.js

@@ -33,17 +33,21 @@ var MovieList = new Class({
 		);
 		self.getMovies();
 
-		if(options.add_new)
+		App.addEvent('movie.added', self.movieAdded.bind(self))
-			App.addEvent('movie.added', self.movieAdded.bind(self))
-
 		App.addEvent('movie.deleted', self.movieDeleted.bind(self))
 	},
 
 	movieDeleted: function(notification){
 		var self = this;
 
-		if(!self.movies_added[notification.data.id])
+		if(self.movies_added[notification.data.id]){
-			self.movies_added[notification.data.id].destroy();
+			self.movies.each(function(movie){
+				if(movie.get('id') == notification.data.id){
+					movie.destroy();
+					delete self.movies_added[notification.data.id]
+				}
+			})
+		}
 
 		self.checkIfEmpty();
 	},
@@ -52,7 +56,7 @@ var MovieList = new Class({
 		var self = this;
 		window.scroll(0,0);
 
-		if(!self.movies_added[notification.data.id])
+		if(self.options.add_new && !self.movies_added[notification.data.id] && notification.data.status.identifier == self.options.status)
 			self.createMovie(notification.data, 'top');
 
 		self.checkIfEmpty();

couchpotato/core/plugins/movie/static/movie.css

@@ -550,3 +550,51 @@
 	font-size: 25px;
 	line-height: 150%;
 }
+
+	.movies .empty_manage .after_manage {
+		margin-top: 30px;
+		font-size: 16px;
+	}
+
+	.movies .progress {
+		border-radius: 2px;
+		padding: 10px;
+		margin: 5px 0;
+		text-align: left;
+	}
+
+		.movies .progress > div {
+			padding: 5px 10px;
+			font-size: 12px;
+			line-height: 12px;
+			text-align: left;
+			display: inline-block;
+			width: 49%;
+			background: rgba(255, 255, 255, 0.05);
+			margin: 2px 0.5%;
+			border-radius: 3px;
+		}
+
+			.movies .progress > div .folder {
+				display: inline-block;
+				padding: 5px 20px 5px 0;
+			    white-space: nowrap;
+				text-overflow: ellipsis;
+				overflow: hidden;
+				width: 85%;
+				direction: rtl;
+				vertical-align: middle;
+			}
+
+			.movies .progress > div .percentage {
+				font-weight: bold;
+				display: inline-block;
+				text-transform: uppercase;
+				text-shadow: none;
+				font-weight: normal;
+				font-size: 20px;
+				border-left: 1px solid rgba(255, 255, 255, .2);
+				width: 15%;
+				text-align: right;
+				vertical-align: middle;
+			}

couchpotato/core/plugins/scanner/main.py

@@ -34,6 +34,7 @@ class Scanner(Plugin):
         'subtitle_extra': ['idx'],
         'trailer': ['mov', 'mp4', 'flv']
     }
+
     file_types = {
         'subtitle': ('subtitle', 'subtitle'),
         'subtitle_extra': ('subtitle', 'subtitle_extra'),
@@ -42,6 +43,8 @@ class Scanner(Plugin):
         'movie': ('video', 'movie'),
         'movie_extra': ('movie', 'movie_extra'),
         'backdrop': ('image', 'backdrop'),
+        'poster': ('image', 'poster'),
+        'thumbnail': ('image', 'thumbnail'),
         'leftover': ('leftover', 'leftover'),
     }
 
@@ -80,55 +83,10 @@ class Scanner(Plugin):
         addEvent('scanner.remove_cptag', self.removeCPTag)
 
         addEvent('scanner.scan', self.scan)
-        addEvent('scanner.files', self.scanFilesToLibrary)
-        addEvent('scanner.folder', self.scanFolderToLibrary)
         addEvent('scanner.name_year', self.getReleaseNameYear)
         addEvent('scanner.partnumber', self.getPartNumber)
 
-        def after_rename(group):
+    def scan(self, folder = None, files = [], simple = False, newer_than = 0, on_found = None):
-            return self.scanFilesToLibrary(folder = group['destination_dir'], files = group['renamed_files'])
-
-        addEvent('renamer.after', after_rename)
-
-    def scanFilesToLibrary(self, folder = None, files = None):
-
-        folder = os.path.normpath(folder)
-
-        groups = self.scan(folder = folder, files = files)
-
-        for group in groups.itervalues():
-            if group['library']:
-                fireEvent('release.add', group = group)
-
-    def scanFolderToLibrary(self, folder = None, newer_than = 0, simple = True):
-
-        folder = os.path.normpath(folder)
-
-        if not os.path.isdir(folder):
-            return
-
-        groups = self.scan(folder = folder, simple = simple, newer_than = newer_than)
-
-        added_identifier = []
-        while True and not self.shuttingDown():
-            try:
-                identifier, group = groups.popitem()
-            except:
-                break
-
-            # Save to DB
-            if group['library']:
-
-                # Add release
-                fireEvent('release.add', group = group)
-                library_item = fireEvent('library.update', identifier = group['library'].get('identifier'), single = True)
-                if library_item:
-                    added_identifier.append(library_item['identifier'])
-
-        return added_identifier
-
-
-    def scan(self, folder = None, files = [], simple = False, newer_than = 0):
 
         folder = ss(os.path.normpath(folder))
 
@@ -281,6 +239,7 @@ class Scanner(Plugin):
 
         # Determine file types
         processed_movies = {}
+        total_found = len(movie_files)
         while True and not self.shuttingDown():
             try:
                 identifier, group = movie_files.popitem()
@@ -395,9 +354,12 @@ class Scanner(Plugin):
                 movie = db.query(Movie).filter_by(library_id = group['library']['id']).first()
                 group['movie_id'] = None if not movie else movie.id
 
-
             processed_movies[identifier] = group
 
+            # Notify parent & progress on something found
+            if on_found:
+                on_found(group, total_found, total_found - len(processed_movies))
+
         if len(processed_movies) > 0:
             log.info('Found %s movies in the folder %s', (len(processed_movies), folder))
         else:

couchpotato/static/scripts/page/manage.js

@@ -41,10 +41,23 @@ Page.Manage = new Class({
 							'text': 'Settings > Manage',
 							'href': App.createUrl('settings/manage')
 						})
+					),
+					new Element('div.after_manage', {
+						'text': 'When you\'ve done that, hit this button → '
+					}).adopt(
+						new Element('a.button.green', {
+							'text': 'Hit me, but not to hard',
+							'events':{
+								'click': self.refresh.bind(self, true)
+							}
+						})
 					)
 				)
 			});
 			$(self.list).inject(self.el);
+
+			// Check if search is in progress
+			self.startProgressInterval();
 		}
 
 	},
@@ -52,11 +65,55 @@ Page.Manage = new Class({
 	refresh: function(full){
 		var self = this;
 
-		Api.request('manage.update', {
+		if(!self.update_in_progress){
-			'data': {
+
-				'full': +full
+			Api.request('manage.update', {
-			}
+				'data': {
-		})
+					'full': +full
+				}
+			})
+
+			self.startProgressInterval();
+
+		}
+
+	},
+
+	startProgressInterval: function(){
+		var self = this;
+
+		self.progress_interval = setInterval(function(){
+
+			Api.request('manage.progress', {
+				'onComplete': function(json){
+					self.update_in_progress = true;
+
+					if(!json.progress){
+						clearInterval(self.progress_interval);
+						self.update_in_progress = false;
+						if(self.progress_container){
+							self.progress_container.destroy();
+							self.list.update();
+						}
+					}
+					else {
+						if(!self.progress_container)
+							self.progress_container = new Element('div.progress').inject(self.list.navigation, 'after')
+
+						self.progress_container.empty();
+
+						Object.each(json.progress, function(progress, folder){
+							new Element('div').adopt(
+								new Element('span.folder', {'text': folder}),
+								new Element('span.percentage', {'text': progress.total ? (((progress.total-progress.to_go)/progress.total)*100).round() + '%' : '0%'})
+							).inject(self.progress_container)
+						});
+
+					}
+				}
+			})
+
+		}, 1000);
 
 	}
 

couchpotato/static/scripts/page/wanted.js

@@ -136,7 +136,7 @@ window.addEvent('domready', function(){
 							'text': profile.label ? profile.label : profile.data.label
 						}).inject(self.profile_select);
 
-						if(self.movie.profile && self.movie.profile.data.id == profile_id)
+						if(self.movie.profile && self.movie.profile.data && self.movie.profile.data.id == profile_id)
 							self.profile_select.set('value', profile_id);
 					});