From dc4b4676f6085c32203fe38dc81350d2764126c1 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 8 Sep 2011 11:24:21 +0200 Subject: [PATCH] Searcher and renamer fixes --- couchpotato/core/notifications/base.py | 2 +- couchpotato/core/plugins/base.py | 7 +++- couchpotato/core/plugins/library/main.py | 2 +- couchpotato/core/plugins/movie/main.py | 4 ++ couchpotato/core/plugins/movie/static/search.css | 7 +++- couchpotato/core/plugins/quality/main.py | 45 ++++++++++++++-------- couchpotato/core/plugins/renamer/__init__.py | 15 ++++++-- couchpotato/core/plugins/renamer/main.py | 49 +++++++++++++----------- couchpotato/core/plugins/scanner/main.py | 41 ++++++++++++++------ 9 files changed, 112 insertions(+), 60 deletions(-) diff --git a/couchpotato/core/notifications/base.py b/couchpotato/core/notifications/base.py index bc2e6a3..8feea71 100644 --- a/couchpotato/core/notifications/base.py +++ b/couchpotato/core/notifications/base.py @@ -39,7 +39,7 @@ class Notification(Plugin): data = {} ) - #return jsonified({'success': success}) + return jsonified({'success': success}) def testNotifyName(self): return 'notify.%s.test' % self.getName().lower() diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 0fcc0b3..e844a53 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -73,9 +73,12 @@ class Plugin(object): try: if not os.path.isdir(path): os.makedirs(path, Env.getPermission('folder')) + return True except Exception, e: log.error('Unable to create folder "%s": %s' % (path, e)) + return False + # http request def urlopen(self, url, timeout = 10, params = {}, headers = {}): @@ -93,7 +96,7 @@ class Plugin(object): data = urllib2.urlopen(request).read() except IOError, e: log.error('Failed opening url, %s: %s' % (url, e)) - data = '' + data = None self.http_last_use[host] = time.time() @@ -111,7 +114,7 @@ class Plugin(object): time.sleep(last_use - now + self.http_time_between_calls) def beforeCall(self, handler): - log.debug('Calling %s.%s' % (self.getName(), handler.__name__)) + #log.debug('Calling %s.%s' % (self.getName(), handler.__name__)) self.isRunning('%s.%s' % (self.getName(), handler.__name__)) def afterCall(self, handler): diff --git a/couchpotato/core/plugins/library/main.py b/couchpotato/core/plugins/library/main.py index d9876f1..5d2e24c 100644 --- a/couchpotato/core/plugins/library/main.py +++ b/couchpotato/core/plugins/library/main.py @@ -62,7 +62,7 @@ class LibraryPlugin(Plugin): info = fireEvent('provider.movie.info', merge = True, identifier = identifier) if not info or len(info) == 0: log.error('Could not update, no movie info to work with: %s' % identifier) - do_update = False + return False # Main info if do_update: diff --git a/couchpotato/core/plugins/movie/main.py b/couchpotato/core/plugins/movie/main.py index 6bf3e3f..dcf1231 100644 --- a/couchpotato/core/plugins/movie/main.py +++ b/couchpotato/core/plugins/movie/main.py @@ -118,11 +118,15 @@ class MoviePlugin(Plugin): if release.status_id == status_snatched.get('id'): release.delete() + m.profile_id = params.get('profile_id') + m.status_id = status_active.get('id') db.commit() movie_dict = m.to_dict(self.default_dict) + fireEventAsync('searcher.single', movie_dict) + return jsonified({ 'success': True, 'added': True, diff --git a/couchpotato/core/plugins/movie/static/search.css b/couchpotato/core/plugins/movie/static/search.css index e94a449..f65235f 100644 --- a/couchpotato/core/plugins/movie/static/search.css +++ b/couchpotato/core/plugins/movie/static/search.css @@ -1,4 +1,7 @@ -/* @override http://localhost:5000/static/movie_plugin/search.css */ +/* @override + http://localhost:5000/static/movie_plugin/search.css + http://192.168.1.20:5000/static/movie_plugin/search.css +*/ .search_form { display: inline-block; @@ -86,7 +89,7 @@ margin-right: 10px; } .search_form .results .movie .options select[name=title] { width: 180px; } - .search_form .results .movie .options select[name=quality] { width: 90px; } + .search_form .results .movie .options select[name=profile] { width: 90px; } .search_form .results .movie .options .button { vertical-align: middle; diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 665636b..b102f3a 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -18,7 +18,7 @@ class QualityPlugin(Plugin): {'identifier': '720p', 'size': (3500, 10000), 'label': '720P', 'width': 1280, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts'], 'tags': ['x264', 'h264', 'bluray']}, {'identifier': 'brrip', 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p'], 'ext':['avi']}, {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': [], 'allow': [], 'ext':['iso', 'img'], 'tags': ['pal', 'ntsc']}, - {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'alternative': [], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, + {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'alternative': ['dvdrip'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['dvdscr', 'ppvrip'], 'allow': ['dvdr'], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': [], 'allow': ['dvdr'], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, @@ -113,12 +113,15 @@ class QualityPlugin(Plugin): return True - def guess(self, files, extra = {}): + def guess(self, files, extra = {}, loose = False): found = False + print files, extra + for file in files: size = (os.path.getsize(file) / 1024 / 1024) words = re.split('\W+', file.lower()) + for quality in self.all(): correctSize = False @@ -126,27 +129,37 @@ class QualityPlugin(Plugin): correctSize = True # Check tags - if type in words: + if quality['identifier'] in words: + log.debug('Found via identifier "%s" in %s' % (quality['identifier'], file)) found = True - for alt in quality.get('alternative'): - if alt in words: - found = True + if list(set(quality.get('alternative', [])) & set(words)): + log.debug('Found via alt %s in %s' % (quality.get('alternative'), file)) + found = True - for tag in quality.get('tags', []): - if tag in words: - found = True + if list(set(quality.get('tags', [])) & set(words)): + log.debug('Found via tag %s in %s' % (quality.get('tags'), file)) + found = True - # Check extension + filesize - for ext in quality.get('ext'): - if ext in words and correctSize: + # Check on unreliable stuff + if loose: + # Check extension + filesize + if list(set(quality.get('ext', [])) & set(words)) and correctSize: + log.debug('Found via ext %s in %s' % (quality.get('ext'), words)) found = True - # Last check on resolution only - if quality.get('width', 480) == extra.get('resolution_width', 0): - found = True + # Last check on resolution only + if quality.get('width', 480) == extra.get('resolution_width', 0): + log.debug('Found via resoludtion_width: %s == %s' % (quality.get('width', 480), extra.get('resolution_width', 0))) + found = True if found: return quality - return '' + # Try again with loose testing + quality = self.guess(files, extra = extra, loose = True) + if quality: + return quality + + log.error('Could not identify quality for: %s' % files) + return {} diff --git a/couchpotato/core/plugins/renamer/__init__.py b/couchpotato/core/plugins/renamer/__init__.py index 07ad234..290b1fc 100644 --- a/couchpotato/core/plugins/renamer/__init__.py +++ b/couchpotato/core/plugins/renamer/__init__.py @@ -61,14 +61,21 @@ config = [{ 'advanced': True, 'options': [ { - 'name': 'trailer_name', - 'label': 'Trailer naming', - 'default': '-trailer.', + 'name': 'rename_nfo', + 'label': 'Rename .NFO', + 'description': 'Rename original .nfo file', + 'type': 'bool', + 'default': True, }, { 'name': 'nfo_name', 'label': 'NFO naming', - 'default': '.', + 'default': '.-orig', + }, + { + 'name': 'trailer_name', + 'label': 'Trailer naming', + 'default': '-trailer.', }, { 'name': 'backdrop_name', diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 9ec86e3..a3c1859 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -1,5 +1,5 @@ from couchpotato import get_session -from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode from couchpotato.core.helpers.variable import getExt from couchpotato.core.logger import CPLog @@ -40,7 +40,7 @@ class Renamer(Plugin): group = groups[group_identifier] rename_files = {} - # Add _UNKNOWN_ if no library is connected + # Add _UNKNOWN_ if no library item is connected if not group['library']: if group['dirname']: rename_files[group['parentdir']] = group['parentdir'].replace(group['dirname'], '_UNKNOWN_%s' % group['dirname']) @@ -53,6 +53,10 @@ class Renamer(Plugin): # Rename the files using the library data else: group['library'] = fireEvent('library.update', identifier = group['library']['identifier'], single = True) + if not group['library']: + log.error('Could not rename, no library item to work with: %s' % group_identifier) + continue + library = group['library'] # Find subtitle for renaming @@ -85,12 +89,9 @@ class Renamer(Plugin): for file_type in group['files']: - # Move DVD files (no renaming) - if group['is_dvd'] and file_type is 'movie': - continue - # Move nfo depending on settings if file_type is 'nfo' and not self.conf('rename_nfo'): + log.debug('Skipping, renaming of %s disabled' % file_type) continue # Subtitle extra @@ -98,7 +99,7 @@ class Renamer(Plugin): continue # Move other files - multiple = len(group['files']['movie']) > 1 + multiple = len(group['files']['movie']) > 1 and not group['is_dvd'] cd = 1 if multiple else 0 for file in sorted(list(group['files'][file_type])): @@ -123,7 +124,7 @@ class Renamer(Plugin): if file_type is 'trailer': final_file_name = self.doReplace(trailer_name, replacements) elif file_type is 'nfo': - final_file_name = self.doReplace(nfo_name, replacements) + '-orig' + final_file_name = self.doReplace(nfo_name, replacements) elif file_type is 'backdrop': final_file_name = self.doReplace(backdrop_name, replacements) @@ -131,8 +132,13 @@ class Renamer(Plugin): if separator: final_file_name = final_file_name.replace(' ', separator) - # Main file - rename_files[file] = os.path.join(destination, final_folder_name, final_file_name) + # Move DVD files (no structure renaming) + if group['is_dvd'] and file_type is 'movie': + structure_dir = file.split(group['dirname'])[-1].lstrip(os.path.sep) + rename_files[file] = os.path.join(destination, final_folder_name, structure_dir) + # Do rename others + else: + rename_files[file] = os.path.join(destination, final_folder_name, final_file_name) # Check for extra subtitle files if file_type is 'subtitle': @@ -154,17 +160,13 @@ class Renamer(Plugin): if multiple: cd += 1 - # Notify on download - download_message = 'Download of %s (%s) successful.' % (group['library']['titles'][0]['title'], replacements['quality']) - fireEvent('movie.downloaded', message = download_message, data = group) - # Before renaming, remove the lower quality files db = get_session() library = db.query(Library).filter_by(identifier = group['library']['identifier']).first() done_status = fireEvent('status.get', 'done', single = True) for movie in library.movies: for release in movie.releases: - if release.quality.order < group['meta_data']['quality']['order']: + if release.quality.order > group['meta_data']['quality']['order']: log.info('Removing older release for %s, with quality %s' % (movie.library.titles[0].title, release.quality.label)) elif release.status_id is done_status.get('id'): if release.quality.order is group['meta_data']['quality']['order']: @@ -200,21 +202,24 @@ class Renamer(Plugin): log.info('Renaming "%s" to "%s"' % (src, dst)) path = os.path.dirname(dst) - try: - if not os.path.isdir(path): os.makedirs(path) - except: - log.error('Failed creating dir %s: %s' % (path, traceback.format_exc())) - continue + + # Create dir + self.makeDir(path) try: - shutil.move(src, dst) + pass + #shutil.move(src, dst) except: log.error('Failed moving the file "%s" : %s' % (os.path.basename(src), traceback.format_exc())) #print rename_me, rename_files[rename_me] # Search for trailers etc - fireEvent('renamer.after', group) + fireEventAsync('renamer.after', group) + + # Notify on download + download_message = 'Download of %s (%s) successful.' % (group['library']['titles'][0]['title'], replacements['quality']) + fireEventAsync('movie.downloaded', message = download_message, data = group) # Break if CP wants to shut down if self.shuttingDown(): diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index ec1fc67..537f7b9 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -4,13 +4,13 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString from couchpotato.core.helpers.variable import getExt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin -from couchpotato.core.settings.model import File, Release, Movie +from couchpotato.core.settings.model import File from couchpotato.environment import Env from flask.helpers import json -from sqlalchemy.sql.expression import and_, or_ import os import re import subprocess +import time import traceback log = CPLog(__name__) @@ -23,7 +23,7 @@ class Scanner(Plugin): 'trailer': 1048576, # 1MB } ignored_in_path = ['_unpack', '_failed_', '_unknown_', '_exists_', '.appledouble', '.appledb', '.appledesktop', os.path.sep + '._', '.ds_store', 'cp.cpnfo'] #unpacking, smb-crap, hidden files - ignore_names = ['extract', 'extracting', 'extracted', 'movie', 'movies', 'film', 'films', 'download', 'downloads'] + ignore_names = ['extract', 'extracting', 'extracted', 'movie', 'movies', 'film', 'films', 'download', 'downloads', 'video_ts', 'audio_ts'] extensions = { 'movie': ['mkv', 'wmv', 'avi', 'mpg', 'mpeg', 'mp4', 'm2ts', 'iso', 'img'], 'dvd': ['vts_*', 'vob'], @@ -172,9 +172,21 @@ class Scanner(Plugin): # Determine file types + delete_identifier = [] for identifier in movie_files: group = movie_files[identifier] + # Check if movie is fresh and maybe still unpacking, ignore files new then 1 minute + file_too_new = False + for file in group['unsorted_files']: + if os.path.getmtime(file) > time.time() - 60: + file_too_new = True + + if file_too_new: + log.info('Files seem to be still unpacking or just unpacked, ignoring for now: %s' % identifier) + delete_identifier.append(identifier) + continue + # Group extra (and easy) files first images = self.getImages(group['unsorted_files']) group['files'] = { @@ -199,10 +211,11 @@ class Scanner(Plugin): group['dirname'] = None folders = group['parentdir'].replace(folder, '').split(os.path.sep) + folders.reverse() - # Try and get a proper dirname, so no "A", "Movie", "Download" + # Try and get a proper dirname, so no "A", "Movie", "Download" etc for folder in folders: - if folder.lower() in self.ignore_names or len(folder) < 2: + if folder.lower() not in self.ignore_names and len(folder) > 2: group['dirname'] = folder break @@ -220,12 +233,16 @@ class Scanner(Plugin): if not group['library']: log.error('Unable to determin movie: %s' % group['identifiers']) + # Delete still (asuming) unpacking files + for identifier in delete_identifier: + del movie_files[identifier] + return movie_files def getMetaData(self, group): data = {} - files = group['files']['movie'] + files = list(group['files']['movie']) for file in files: if os.path.getsize(file) < self.minimal_filesize['media']: continue # Ignore smaller files @@ -246,10 +263,11 @@ class Scanner(Plugin): if not data['quality']: data['quality'] = fireEvent('quality.single', 'dvdr' if group['is_dvd'] else 'dvdrip', single = True) - data['quality_type'] = 'HD' if data.get('resolution_width', 0) >= 720 else 'SD' + data['quality_type'] = 'HD' if data.get('resolution_width', 0) >= 1280 else 'SD' - data['group'] = self.getGroup(file[0]) - data['source'] = self.getSourceMedia(file[0]) + file = re.sub('(.cp\(tt[0-9{7}]+\))', '', files[0]) + data['group'] = self.getGroup(file) + data['source'] = self.getSourceMedia(file) return data @@ -365,7 +383,6 @@ class Scanner(Plugin): return set(filter(test, files)) def getDVDFiles(self, files): - def test(s): return self.isDVDFile(s) @@ -510,8 +527,8 @@ class Scanner(Plugin): def getGroup(self, file): try: - group = re.search('-(?P[A-Z0-9]+)$', file, re.I) - return group.group('group') or '' + match = re.search('-(?P[A-Z0-9]+).', file, re.I) + return match.group('group') or '' except: return ''