From 6a18e546ca1232b3880612aeed583a4d7b4d5701 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Mon, 25 Mar 2013 23:35:23 +0100 Subject: [PATCH] Add and make use of renamer.scanfolder in downloaders This is the next step in closing the loop between the downloaders and CPS. The download_id and folder from the downloader are used to find the downloaded files and start the renamer. This is done by adding an additional API call: renamer.scanfolder. I tested this for SabNZBd only (!) and everything works as expected. I also added transmission with thanks @manusfreedom for setting this up in f1cf0d91da. @manusfreedom, please check if this works as expected. Note that transmission now has a feature which is not in the other torrent providers: it waits until the seed ratio is met and then removes the torrent. I opened a topic in the forum to discuss how we want to deal with torrents: https://couchpota.to/forum/thread-1704.html --- couchpotato/core/downloaders/base.py | 1 + couchpotato/core/downloaders/nzbget/main.py | 1 + couchpotato/core/downloaders/nzbvortex/main.py | 3 +- couchpotato/core/downloaders/sabnzbd/main.py | 12 +-- .../core/downloaders/transmission/__init__.py | 11 +- couchpotato/core/downloaders/transmission/main.py | 115 ++++++++++++++++++++- couchpotato/core/downloaders/utorrent/main.py | 4 +- couchpotato/core/plugins/renamer/main.py | 69 ++++++++++++- couchpotato/core/plugins/scanner/main.py | 55 +++++++--- 9 files changed, 235 insertions(+), 36 deletions(-) diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index 6944924..8ccc136 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -150,6 +150,7 @@ class StatusList(list): 'id': 0, 'status': 'busy', 'downloader': self.provider.getName(), + 'folder': '', } return mergeDicts(defaults, result) diff --git a/couchpotato/core/downloaders/nzbget/main.py b/couchpotato/core/downloaders/nzbget/main.py index 1bc54fd..d0533d5 100644 --- a/couchpotato/core/downloaders/nzbget/main.py +++ b/couchpotato/core/downloaders/nzbget/main.py @@ -120,6 +120,7 @@ class NZBGet(Downloader): 'status': 'completed' if item['ParStatus'] == 'SUCCESS' and item['ScriptStatus'] == 'SUCCESS' else 'failed', 'original_status': item['ParStatus'] + ', ' + item['ScriptStatus'], 'timeleft': str(timedelta(seconds = 0)), + 'folder': item['DestDir'] }) return statuses diff --git a/couchpotato/core/downloaders/nzbvortex/main.py b/couchpotato/core/downloaders/nzbvortex/main.py index e6cbd02..f1f8acc 100644 --- a/couchpotato/core/downloaders/nzbvortex/main.py +++ b/couchpotato/core/downloaders/nzbvortex/main.py @@ -55,7 +55,8 @@ class NZBVortex(Downloader): 'name': item['uiTitle'], 'status': status, 'original_status': item['state'], - 'timeleft':-1, + 'timeleft': -1, + 'folder': item['destinationPath'], }) return statuses diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index 8b37469..f2f217a 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -3,6 +3,7 @@ from couchpotato.core.helpers.encoding import tryUrlencode, ss from couchpotato.core.helpers.variable import cleanHost, mergeDicts from couchpotato.core.logger import CPLog from couchpotato.environment import Env +from datetime import timedelta from urllib2 import URLError import json import traceback @@ -46,19 +47,15 @@ class Sabnzbd(Downloader): log.error('Failed sending release, use API key, NOT the NZB key: %s', traceback.format_exc(0)) return False - if sab_data.get('error'): - log.error('Error getting data from SABNZBd: %s', sab_data.get('error')) - return False - log.debug('Result from SAB: %s', sab_data) - if sab_data.get('status'): + if sab_data.get('status') and not sab_data.get('error'): log.info('NZB sent to SAB successfully.') if filedata: return self.downloadReturnId(sab_data.get('nzo_ids')[0]) else: return True else: - log.error(sab_data) + log.error('Error getting data from SABNZBd: %s', sab_data) return False def getAllDownloadStatus(self): @@ -109,7 +106,8 @@ class Sabnzbd(Downloader): 'name': item['name'], 'status': status, 'original_status': item['status'], - 'timeleft': 0, + 'timeleft': str(timedelta(seconds = 0)), + 'folder': item['storage'], }) return statuses diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index 210a0d9..0fd783b 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -41,16 +41,23 @@ config = [{ { 'name': 'directory', 'type': 'directory', - 'description': 'Where should Transmission saved the downloaded files?', + 'description': 'Where should Transmission save the downloaded files?', }, { 'name': 'ratio', 'default': 10, - 'type': 'int', + 'type': 'float', 'advanced': True, 'description': 'Stop transfer when reaching ratio', }, { + 'name': 'ratiomode', + 'default': 0, + 'type': 'int', + 'advanced': True, + 'description': '0 = Use session limit, 1 = Use transfer limit, 2 = Disable limit.', + }, + { 'name': 'manual', 'default': 0, 'type': 'bool', diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 6c4607f..24f7649 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -1,12 +1,15 @@ from base64 import b64encode -from couchpotato.core.downloaders.base import Downloader +from couchpotato.core.downloaders.base import Downloader, StatusList from couchpotato.core.helpers.encoding import isInt from couchpotato.core.logger import CPLog +from couchpotato.environment import Env +from datetime import timedelta import httplib import json import os.path import re import urllib2 +import shutil log = CPLog(__name__) @@ -18,7 +21,7 @@ class Transmission(Downloader): def download(self, data, movie, filedata = None): - log.debug('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type'))) + log.info('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type'))) # Load host from config and split out port. host = self.conf('host').split(':') @@ -30,7 +33,7 @@ class Transmission(Downloader): folder_name = self.createFileName(data, filedata, movie)[:-len(data.get('type')) - 1] folder_path = os.path.join(self.conf('directory', default = ''), folder_name).rstrip(os.path.sep) - # Create the empty folder to download too + # Create the empty folder to download to self.makeDir(folder_path) params = { @@ -42,7 +45,7 @@ class Transmission(Downloader): if self.conf('ratio'): torrent_params = { 'seedRatioLimit': self.conf('ratio'), - 'seedRatioMode': self.conf('ratio') + 'seedRatioMode': self.conf('ratiomode') } if not filedata and data.get('type') == 'torrent': @@ -62,11 +65,99 @@ class Transmission(Downloader): if torrent_params: trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params) + log.info('Torrent sent to Transmission successfully.') return self.downloadReturnId(remote_torrent['torrent-added']['hashString']) except Exception, err: log.error('Failed to change settings for transfer: %s', err) return False + def getAllDownloadStatus(self): + + log.debug('Checking Transmission download status.') + + # Load host from config and split out port. + host = self.conf('host').split(':') + if not isInt(host[1]): + log.error('Config properties are not filled in correctly, port is missing.') + return False + + # Go through Queue + try: + trpc = TransmissionRPC(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) + return_params = { + 'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isFinished', 'downloadDir', 'uploadRatio'] + } + queue = trpc.get_alltorrents(return_params) + + except Exception, err: + log.error('Failed getting queue: %s', err) + return False + + statuses = StatusList(self) + + # Get torrents status + # CouchPotato Status + #status = 'busy' + #status = 'failed' + #status = 'completed' + # Transmission Status + #status = 0 => "Torrent is stopped" + #status = 1 => "Queued to check files" + #status = 2 => "Checking files" + #status = 3 => "Queued to download" + #status = 4 => "Downloading" + #status = 4 => "Queued to seed" + #status = 6 => "Seeding" + #To do : + # add checking file + # manage no peer in a range time => fail + + for item in queue['torrents']: + log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / confRatio=%s / isFinished=%s', (item['name'], item['id'], item['downloadDir'], item['hashString'], item['percentDone'], item['status'], item['eta'], item['uploadRatio'], self.conf('ratio'), item['isFinished'] )) + + if not os.path.isdir(Env.setting('from', 'renamer')): + log.debug('Renamer folder has to exist.') + return + + if (item['percentDone'] * 100) >= 100 and (item['status'] == 6 or item['status'] == 0) and item['uploadRatio'] > self.conf('ratio'): + try: + trpc.stop_torrent(item['hashString'], {}) + + if not os.path.isdir(item['downloadDir']): + raise Exception('Missing folder: %s' % item['downloadDir']) + + else: + log.info('Moving folder from "%s" to "%s"', (item['downloadDir'], Env.setting('from', 'renamer'))) + shutil.move(item['downloadDir'], Env.setting('from', 'renamer')) + + statuses.append({ + 'id': item['hashString'], + 'name': item['downloadDir'], + 'status': 'completed', + 'original_status': item['status'], + 'timeleft': str(timedelta(seconds = 0)), + 'folder': os.path.join(Env.setting('from', 'renamer'), os.path.basename(item['downloadDir'].rstrip(os.path.sep))), + }) + trpc.remove_torrent(item['hashString'], True, {}) + except Exception, err: + log.error('Failed to stop and remove torrent "%s" with error: %s', (item['name'], err)) + statuses.append({ + 'id': item['hashString'], + 'name': item['downloadDir'], + 'status': 'failed', + 'original_status': item['status'], + 'timeleft': str(timedelta(seconds = 0)), + }) + else: + statuses.append({ + 'id': item['hashString'], + 'name': item['downloadDir'], + 'status': 'busy', + 'original_status': item['status'], + 'timeleft': str(timedelta(seconds = item['eta'])), # Is ETA in seconds?? + }) + + return statuses class TransmissionRPC(object): @@ -97,6 +188,7 @@ class TransmissionRPC(object): try: open_request = urllib2.urlopen(request) response = json.loads(open_request.read()) + log.debug('request: %s', json.dumps(ojson)) log.debug('response: %s', json.dumps(response)) if response['result'] == 'success': log.debug('Transmission action successfull') @@ -146,3 +238,18 @@ class TransmissionRPC(object): arguments['ids'] = torrent_id post_data = {'arguments': arguments, 'method': 'torrent-set', 'tag': self.tag} return self._request(post_data) + + def get_alltorrents(self, arguments): + post_data = {'arguments': arguments, 'method': 'torrent-get', 'tag': self.tag} + return self._request(post_data) + + def stop_torrent(self, torrent_id, arguments): + arguments['ids'] = torrent_id + post_data = {'arguments': arguments, 'method': 'torrent-stop', 'tag': self.tag} + return self._request(post_data) + + def remove_torrent(self, torrent_id, remove_local_data, arguments): + arguments['ids'] = torrent_id + arguments['delete-local-data'] = remove_local_data + post_data = {'arguments': arguments, 'method': 'torrent-remove', 'tag': self.tag} + return self._request(post_data) diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index c64db13..ca0a0da 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -5,6 +5,7 @@ from couchpotato.core.helpers.encoding import isInt, ss from couchpotato.core.logger import CPLog from hashlib import sha1 from multipartpost import MultipartPostHandler +from datetime import timedelta import cookielib import httplib import json @@ -118,7 +119,8 @@ class uTorrent(Downloader): 'name': item[2], 'status': status, 'original_status': item[1], - 'timeleft': item[10], + 'timeleft': str(timedelta(seconds = item[10])), + 'folder': '', #no fucntion to get folder, but can be deduced with getSettings function. }) return statuses diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index aba2072..ad96157 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -2,7 +2,7 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, ss -from couchpotato.core.helpers.request import jsonified +from couchpotato.core.helpers.request import getParams, jsonified, getParam from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ getImdb from couchpotato.core.logger import CPLog @@ -31,6 +31,17 @@ class Renamer(Plugin): }) addEvent('renamer.scan', self.scan) + + addApiView('renamer.scanfolder', self.scanfolderView, docs = { + 'desc': 'For the renamer to check for new files to rename in a specified folder', + 'params': { + 'movie_folder': {'desc': 'The folder of the movie to scan'}, + 'downloader' : {'desc': 'Optional: The downloader this movie has been downloaded with'}, + 'download_id': {'desc': 'Optional: The downloader\'s nzb/torrent ID'}, + }, + }) + + addEvent('renamer.scanfolder', self.scanfolder) addEvent('renamer.check_snatched', self.checkSnatched) addEvent('app.load', self.scan) @@ -51,6 +62,26 @@ class Renamer(Plugin): }) def scan(self): + self.scanfolder() + + def scanfolderView(self): + + params = getParams() + movie_folder = params.get('movie_folder', None) + downloader = params.get('downloader', None) + download_id = params.get('download_id', None) + + fireEventAsync('renamer.scanfolder', + movie_folder = movie_folder, + downloader = downloader, + download_id = download_id + ) + + return jsonified({ + 'success': True + }) + + def scanfolder(self, movie_folder = None, downloader = None, download_id = None): if self.isDisabled(): return @@ -59,18 +90,43 @@ class Renamer(Plugin): log.info('Renamer is already running, if you see this often, check the logs above for errors.') return + self.renaming_started = True + # Check to see if the "to" folder is inside the "from" folder. - if not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')): + if movie_folder and not os.path.isdir(movie_folder): # or not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')): log.debug('"To" and "From" have to exist.') return elif self.conf('from') in self.conf('to'): log.error('The "to" can\'t be inside of the "from" folder. You\'ll get an infinite loop.') return + elif (movie_folder and movie_folder in [self.conf('to'), self.conf('from')]): + log.error('The "to" and "from" folders can\'t be inside of or the same as the provided movie folder.') + return - groups = fireEvent('scanner.scan', folder = self.conf('from'), single = True) + # make sure the movie folder name is included in the search + folder = None + movie_files = [] + if movie_folder: + log.info('Scanning movie folder %s...', movie_folder) + movie_folder = movie_folder.rstrip(os.path.sep) + folder = os.path.dirname(movie_folder) - self.renaming_started = True + # Get all files from the specified folder + try: + for root, folders, names in os.walk(movie_folder): + movie_files.extend([os.path.join(root, name) for name in names]) + except: + log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc())) + groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = movie_files, downloader = downloader, download_id = download_id, single = True) + + # Make sure only one movie was found if a download ID is provided + if downloader and download_id and not len(groups) == 1: + log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (download_id, len(groups))) + downloader = None + download_id = None + groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = movie_files, single = True) + destination = self.conf('to') folder_name = self.conf('folder_name') file_name = self.conf('file_name') @@ -597,7 +653,10 @@ class Renamer(Plugin): db.commit() elif item['status'] == 'completed': log.info('Download of %s completed!', item['name']) - scan_required = True + if item['id'] and item['downloader'] and item['folder']: + fireEventAsync('renamer.scanfolder', movie_folder = item['folder'], downloader = item['downloader'], download_id = item['id']) + else: + scan_required = True found = True break diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index b2ac938..6e9a9b6 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -4,7 +4,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString, ss from couchpotato.core.helpers.variable import getExt, getImdb, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin -from couchpotato.core.settings.model import File, Movie +from couchpotato.core.settings.model import File, Movie, Release, ReleaseInfo from enzyme.exceptions import NoParserError, ParseError from guessit import guess_movie_info from subliminal.videos import Video @@ -101,7 +101,7 @@ class Scanner(Plugin): addEvent('scanner.name_year', self.getReleaseNameYear) addEvent('scanner.partnumber', self.getPartNumber) - def scan(self, folder = None, files = None, simple = False, newer_than = 0, on_found = None): + def scan(self, folder = None, files = None, downloader = None, download_id = None, simple = False, newer_than = 0, on_found = None): folder = ss(os.path.normpath(folder)) @@ -119,8 +119,7 @@ class Scanner(Plugin): try: files = [] for root, dirs, walk_files in os.walk(folder): - for filename in walk_files: - files.append(os.path.join(root, filename)) + files.extend(os.path.join(root, filename) for filename in walk_files) except: log.error('Failed getting files from %s: %s', (folder, traceback.format_exc())) else: @@ -129,6 +128,24 @@ class Scanner(Plugin): db = get_session() + # Get the release with the downloader ID that was downloded by the downloader + download_quality = None + download_imdb_id = None + if downloader and download_id: + # NOTE TO RUUD: Don't really know how to do this better... but there must be a way...? + rlsnfo_dwnlds = db.query(ReleaseInfo).filter_by(identifier = 'download_downloader', value = downloader) + rlsnfo_ids = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = download_id) + for rlsnfo_dwnld in rlsnfo_dwnlds: + for rlsnfo_id in rlsnfo_ids: + if rlsnfo_id.release == rlsnfo_dwnld.release: + rls = rlsnfo_id.release + + if rls: + download_imdb_id = rls.movie.library.identifier + download_quality = rls.quality.identifier + else: + log.error('Download ID %s from downloader %s not found in releases', (download_id, downloader)) + for file_path in files: if not os.path.exists(file_path): @@ -346,7 +363,7 @@ class Scanner(Plugin): continue log.debug('Getting metadata for %s', identifier) - group['meta_data'] = self.getMetaData(group, folder = folder) + group['meta_data'] = self.getMetaData(group, folder = folder, download_quality = download_quality) # Subtitle meta group['subtitle_language'] = self.getSubtitleLanguage(group) if not simple else {} @@ -376,7 +393,7 @@ class Scanner(Plugin): del group['unsorted_files'] # Determine movie - group['library'] = self.determineMovie(group) + group['library'] = self.determineMovie(group, download_imdb_id = download_imdb_id) if not group['library']: log.error('Unable to determine movie: %s', group['identifiers']) else: @@ -401,7 +418,7 @@ class Scanner(Plugin): return processed_movies - def getMetaData(self, group, folder = ''): + def getMetaData(self, group, folder = '', download_quality = None): data = {} files = list(group['files']['movie']) @@ -423,10 +440,14 @@ class Scanner(Plugin): if data.get('audio'): break + # Use the quality guess first, if that failes use the quality we wanted to download data['quality'] = fireEvent('quality.guess', files = files, extra = data, single = True) if not data['quality']: - data['quality'] = fireEvent('quality.single', 'dvdr' if group['is_dvd'] else 'dvdrip', single = True) - + if download_quality: + data['quality'] = fireEvent('quality.single', download_quality, single = True) + else: + data['quality'] = fireEvent('quality.single', 'dvdr' if group['is_dvd'] else 'dvdrip', single = True) + data['quality_type'] = 'HD' if data.get('resolution_width', 0) >= 1280 or data['quality'].get('hd') else 'SD' filename = re.sub('(.cp\(tt[0-9{7}]+\))', '', files[0]) @@ -501,17 +522,19 @@ class Scanner(Plugin): return detected_languages - def determineMovie(self, group): - imdb_id = None + def determineMovie(self, group, download_imdb_id = None): + # Get imdb id from downloader + imdb_id = download_imdb_id files = group['files'] # Check for CP(imdb_id) string in the file paths - for cur_file in files['movie']: - imdb_id = self.getCPImdb(cur_file) - if imdb_id: - log.debug('Found movie via CP tag: %s', cur_file) - break + if not imdb_id: + for cur_file in files['movie']: + imdb_id = self.getCPImdb(cur_file) + if imdb_id: + log.debug('Found movie via CP tag: %s', cur_file) + break # Check and see if nfo contains the imdb-id if not imdb_id: