From a9dc443370e29bf0c8f69b6a21420d907d34e3fa Mon Sep 17 00:00:00 2001 From: Viktor Elofsson Date: Thu, 9 Jul 2015 15:44:52 +0200 Subject: [PATCH 1/4] Updated Hadouken downloader to support both v4 and v5. --- couchpotato/core/downloaders/hadouken.py | 447 +++++++++++++++++++++---------- 1 file changed, 303 insertions(+), 144 deletions(-) diff --git a/couchpotato/core/downloaders/hadouken.py b/couchpotato/core/downloaders/hadouken.py index c89ed0e..8bb0326 100644 --- a/couchpotato/core/downloaders/hadouken.py +++ b/couchpotato/core/downloaders/hadouken.py @@ -31,13 +31,33 @@ class Hadouken(DownloaderBase): log.error('Config properties are not filled in correctly, port is missing.') return False - if not self.conf('api_key'): - log.error('Config properties are not filled in correctly, API key is missing.') - return False + # This is where v4 and v5 begin to differ + if(self.conf('version') == 'v4'): + if not self.conf('api_key'): + log.error('Config properties are not filled in correctly, API key is missing.') + return False + + url = 'http://' + str(host[0]) + ':' + str(host[1]) + '/jsonrpc' + client = JsonRpcClient(url, 'Token ' + self.conf('api_key')) + self.hadouken_api = HadoukenAPIv4(client) - self.hadouken_api = HadoukenAPI(host[0], port = host[1], api_key = self.conf('api_key')) + return True + else: + auth_type = self.conf('auth_type') + header = None - return True + if auth_type == 'api_key': + header = 'Token ' + self.conf('api_key') + elif auth_type == 'user_pass': + header = 'Basic ' + b64encode(self.conf('auth_user') + ':' + self.conf('auth_pass')) + + url = 'http://' + str(host[0]) + ':' + str(host[1]) + '/api' + client = JsonRpcClient(url, header) + self.hadouken_api = HadoukenAPIv5(client) + + return True + + return False def download(self, data = None, media = None, filedata = None): """ Send a torrent/nzb file to the downloader @@ -66,6 +86,8 @@ class Hadouken(DownloaderBase): if self.conf('label'): torrent_params['label'] = self.conf('label') + # Set the tags array since that is what v5 expects. + torrent_params['tags'] = [self.conf('label')] torrent_filename = self.createFileName(data, filedata, media) @@ -132,71 +154,25 @@ class Hadouken(DownloaderBase): if torrent is None: continue - torrent_filelist = self.hadouken_api.get_files_by_hash(torrent['InfoHash']) + torrent_filelist = self.hadouken_api.get_files_by_hash(torrent.info_hash) torrent_files = [] - save_path = torrent['SavePath'] - - # The 'Path' key for each file_item contains - # the full path to the single file relative to the - # torrents save path. - - # For a single file torrent the result would be, - # - Save path: "C:\Downloads" - # - file_item['Path'] = "file1.iso" - # Resulting path: "C:\Downloads\file1.iso" - - # For a multi file torrent the result would be, - # - Save path: "C:\Downloads" - # - file_item['Path'] = "dirname/file1.iso" - # Resulting path: "C:\Downloads\dirname/file1.iso" - for file_item in torrent_filelist: - torrent_files.append(sp(os.path.join(save_path, file_item['Path']))) + torrent_files.append(sp(os.path.join(torrent.save_path, file_item))) release_downloads.append({ - 'id': torrent['InfoHash'].upper(), - 'name': torrent['Name'], - 'status': self.get_torrent_status(torrent), - 'seed_ratio': self.get_seed_ratio(torrent), - 'original_status': torrent['State'], + 'id': torrent.info_hash.upper(), + 'name': torrent.name, + 'status': torrent.get_status(), + 'seed_ratio': torrent.get_seed_ratio(), + 'original_status': torrent.state, 'timeleft': -1, - 'folder': sp(save_path if len(torrent_files == 1) else os.path.join(save_path, torrent['Name'])), + 'folder': sp(torrent.save_path if len(torrent_files == 1) else os.path.join(torrent.save_path, torrent.name)), 'files': torrent_files }) return release_downloads - def get_seed_ratio(self, torrent): - """ Returns the seed ratio for a given torrent. - - Keyword arguments: - torrent -- The torrent to calculate seed ratio for. - """ - - up = torrent['TotalUploadedBytes'] - down = torrent['TotalDownloadedBytes'] - - if up > 0 and down > 0: - return up / down - - return 0 - - def get_torrent_status(self, torrent): - """ Returns the CouchPotato status for a given torrent. - - Keyword arguments: - torrent -- The torrent to translate status for. - """ - - if torrent['IsSeeding'] and torrent['IsFinished'] and torrent['Paused']: - return 'completed' - - if torrent['IsSeeding']: - return 'seeding' - - return 'busy' - def pause(self, release_download, pause = True): """ Pauses or resumes the torrent specified by the ID field in release_download. @@ -243,45 +219,85 @@ class Hadouken(DownloaderBase): return self.hadouken_api.remove(release_download['id'], remove_data = delete_files) -class HadoukenAPI(object): - def __init__(self, host = 'localhost', port = 7890, api_key = None): - self.url = 'http://' + str(host) + ':' + str(port) - self.api_key = api_key - self.requestId = 0; +class JsonRpcClient(object): + def __init__(self, url, auth_header = None): + self.url = url + self.requestId = 0 self.opener = urllib2.build_opener() - self.opener.addheaders = [('User-agent', 'couchpotato-hadouken-client/1.0'), ('Accept', 'application/json')] + self.opener.addheaders = [ + ('User-Agent', 'couchpotato-hadouken-client/1.0'), + ('Accept', 'application/json'), + ('Content-Type', 'application/json') + ] + + if auth_header: + self.opener.addheaders.append(('Authorization', auth_header)) + + def invoke(self, method, params): + self.requestId += 1 - if not api_key: - log.error('API key missing.') + data = { + 'jsonrpc': '2.0', + 'id': self.requestId, + 'method': method, + 'params': params + } + + request = urllib2.Request(self.url, data = json.dumps(data)) + + try: + f = self.opener.open(request) + response = f.read() + f.close() + + obj = json.loads(response) + + if 'error' in obj.keys(): + log.error('JSONRPC error, %s: %s', obj['error']['code'], obj['error']['message']) + return False - def add_file(self, filedata, torrent_params): + if 'result' in obj.keys(): + return obj['result'] + + return True + except httplib.InvalidURL as err: + log.error('Invalid Hadouken host, check your config %s', err) + except urllib2.HTTPError as err: + if err.code == 401: + log.error('Could not authenticate, check your config') + else: + log.error('Hadouken HTTPError: %s', err) + except urllib2.URLError as err: + log.error('Unable to connect to Hadouken %s', err) + + return False + + +class HadoukenAPI(object): + def __init__(self, rpc_client): + self.rpc = rpc_client + + if not rpc_client: + log.error('No JSONRPC client specified.') + + def add_file(self, data, params): """ Add a file to Hadouken with the specified parameters. Keyword arguments: filedata -- The binary torrent data. torrent_params -- Additional parameters for the file. """ - data = { - 'method': 'torrents.addFile', - 'params': [b64encode(filedata), torrent_params] - } - - return self._request(data) + pass - def add_magnet_link(self, magnetLink, torrent_params): + def add_magnet_link(self, link, params): """ Add a magnet link to Hadouken with the specified parameters. Keyword arguments: magnetLink -- The magnet link to send. torrent_params -- Additional parameters for the magnet link. """ - data = { - 'method': 'torrents.addUrl', - 'params': [magnetLink, torrent_params] - } - - return self._request(data) + pass def get_by_hash_list(self, infoHashList): """ Gets a list of torrents filtered by the given info hash list. @@ -289,12 +305,7 @@ class HadoukenAPI(object): Keyword arguments: infoHashList -- A list of info hashes. """ - data = { - 'method': 'torrents.getByInfoHashList', - 'params': [infoHashList] - } - - return self._request(data) + pass def get_files_by_hash(self, infoHash): """ Gets a list of files for the torrent identified by the @@ -303,26 +314,11 @@ class HadoukenAPI(object): Keyword arguments: infoHash -- The info hash of the torrent to return files for. """ - data = { - 'method': 'torrents.getFiles', - 'params': [infoHash] - } - - return self._request(data) + pass def get_version(self): """ Gets the version, commitish and build date of Hadouken. """ - data = { - 'method': 'core.getVersion', - 'params': None - } - - result = self._request(data) - - if not result: - return False - - return result['Version'] + pass def pause(self, infoHash, pause): """ Pauses/unpauses the torrent identified by the given info hash. @@ -331,15 +327,7 @@ class HadoukenAPI(object): infoHash -- The info hash of the torrent to operate on. pause -- If true, pauses the torrent. Otherwise resumes. """ - data = { - 'method': 'torrents.pause', - 'params': [infoHash] - } - - if not pause: - data['method'] = 'torrents.resume' - - return self._request(data) + pass def remove(self, infoHash, remove_data = False): """ Removes the torrent identified by the given info hash and @@ -349,46 +337,190 @@ class HadoukenAPI(object): infoHash -- The info hash of the torrent to remove. remove_data -- If true, removes the data associated with the torrent. """ - data = { - 'method': 'torrents.remove', - 'params': [infoHash, remove_data] - } + pass - return self._request(data) +class TorrentItem(object): + @property + def info_hash(self): + pass - def _request(self, data): - self.requestId += 1 + @property + def save_path(self): + pass - data['jsonrpc'] = '2.0' - data['id'] = self.requestId + @property + def name(self): + pass - request = urllib2.Request(self.url + '/jsonrpc', data = json.dumps(data)) - request.add_header('Authorization', 'Token ' + self.api_key) - request.add_header('Content-Type', 'application/json') + @property + def state(self): + pass - try: - f = self.opener.open(request) - response = f.read() - f.close() + def get_status(self): + """ Returns the CouchPotato status for a given torrent.""" + pass - obj = json.loads(response) + def get_seed_ratio(self): + """ Returns the seed ratio for a given torrent.""" + pass - if not 'error' in obj.keys(): - return obj['result'] - log.error('JSONRPC error, %s: %s', obj['error']['code'], obj['error']['message']) - except httplib.InvalidURL as err: - log.error('Invalid Hadouken host, check your config %s', err) - except urllib2.HTTPError as err: - if err.code == 401: - log.error('Invalid Hadouken API key, check your config') - else: - log.error('Hadouken HTTPError: %s', err) - except urllib2.URLError as err: - log.error('Unable to connect to Hadouken %s', err) +class TorrentItemv5(TorrentItem): + def __init__(self, obj): + self.obj = obj + + def info_hash(self): + return self.obj['infoHash'] + + def save_path(self): + return self.obj['savePath'] + + def name(self): + return self.obj['name'] + + def state(self): + return self.obj['state'] + + def get_status(self): + if self.obj['isSeeding'] and self.obj['isFinished'] and self.obj['isPaused']: + return 'completed' + + if self.obj['isSeeding']: + return 'seeding' + + return 'busy' + + def get_seed_ratio(self): + up = self.obj['uploadedBytesTotal'] + down = self.obj['downloadedBytesTotal'] + + if up > 0 and down > 0: + return up / down + + return 0 + + +class HadoukenAPIv5(HadoukenAPI): + def add_file(self, data, params): + return self.rpc.invoke('session.addTorrentFile', [b64encode(data), params]) + + def add_magnet_link(self, link, params): + return self.rpc.invoke('session.addTorrentUri', [link, params]) + + def get_by_hash_list(self, infoHashList): + torrents = self.rpc.invoke('session.getTorrents') + result = [] + + for torrent in torrents.values(): + if torrent['infoHash'] in infoHashList: + result.append(TorrentItemv5(torrent)) + + return result + + def get_files_by_hash(self, infoHash): + files = self.rpc.invoke('torrent.getFiles', [infoHash]) + result = [] + + for file in files: + result.append(file['path']) + + return result + + def get_version(self): + result = self.rpc.invoke('core.getSystemInfo', None) + + if not result: + return False + + return result['versions']['hadouken'] + + def pause(self, infoHash, pause): + if pause: + return self.rpc.invoke('torrent.pause', [infoHash]) + + return self.rpc.invoke('torrent.resume', [infoHash]) + + def remove(self, infoHash, remove_data = False): + return self.rpc.invoke('session.removeTorrent', [infoHash, remove_data]) - return False + +class TorrentItemv4(TorrentItem): + def __init__(self, obj): + self.obj = obj + + def info_hash(self): + return self.obj['InfoHash'] + + def save_path(self): + return self.obj['SavePath'] + + def name(self): + return self.obj['Name'] + + def state(self): + return self.obj['State'] + + def get_status(self): + if self.obj['IsSeeding'] and self.obj['IsFinished'] and self.obj['Paused']: + return 'completed' + + if self.obj['IsSeeding']: + return 'seeding' + + return 'busy' + + def get_seed_ratio(self): + up = self.obj['TotalUploadedBytes'] + down = self.obj['TotalDownloadedBytes'] + + if up > 0 and down > 0: + return up / down + + return 0 + + +class HadoukenAPIv4(object): + def add_file(self, data, params): + return self.rpc.invoke('torrents.addFile', [b64encode(data), params]) + + def add_magnet_link(self, link, params): + return self.rpc.invoke('torrents.addUrl', [link, params]) + + def get_by_hash_list(self, infoHashList): + torrents = self.rpc.invoke('torrents.getByInfoHashList', [infoHashList]) + result = [] + + for torrent in torrents: + result.append(TorrentItemv4(torrent)) + + return result + + def get_files_by_hash(self, infoHash): + files = self.rpc.invoke('torrents.getFiles', [infoHash]) + result = [] + + for file in files: + result.append(file['Path']) + + return result + + def get_version(self): + result = self.rpc.invoke('core.getVersion', None) + + if not result: + return False + + return result['Version'] + + def pause(self, infoHash, pause): + if pause: + return self.rpc.invoke('torrents.pause', [infoHash]) + + return self.rpc.invoke('torrents.resume', [infoHash]) + + def remove(self, infoHash, remove_data = False): + return self.rpc.invoke('torrents.remove', [infoHash, remove_data]) config = [{ @@ -409,15 +541,42 @@ config = [{ 'radio_group': 'torrent' }, { + 'name': 'version', + 'label': 'Version', + 'type': 'dropdown', + 'default': 'v4', + 'values': [('v4.x', 'v4'), ('v5.x', 'v5')], + 'description': 'Hadouken version.', + }, + { 'name': 'host', 'default': 'localhost:7890' }, { + 'name': 'auth_type', + 'label': 'Auth. type', + 'type': 'dropdown', + 'default': 'api_key', + 'values': [('None', 'none'), ('API key/Token', 'api_key'), ('Username/Password', 'user_pass')], + 'description': 'Type of authentication', + }, + { 'name': 'api_key', - 'label': 'API key', + 'label': 'API key (v4)/Token (v5)', 'type': 'password' }, { + 'name': 'auth_user', + 'label': 'Username', + 'description': '(only for v5)' + }, + { + 'name': 'auth_pass', + 'label': 'Password', + 'type': 'password', + 'description': '(only for v5)' + }, + { 'name': 'label', 'description': 'Label to add torrent as.' } From 27b8c98b09aa9971724ed76608ad02ed98c01737 Mon Sep 17 00:00:00 2001 From: h3llrais3r Date: Thu, 16 Jul 2015 09:25:45 +0200 Subject: [PATCH 2/4] Fix scanning of DVD (VIDEO_TS) files This fixes issue https://github.com/RuudBurger/CouchPotatoServer/issues/3609 Thanks to @rhodespa --- couchpotato/core/plugins/scanner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/scanner.py b/couchpotato/core/plugins/scanner.py index 8ba5dae..1088ddd 100644 --- a/couchpotato/core/plugins/scanner.py +++ b/couchpotato/core/plugins/scanner.py @@ -797,6 +797,10 @@ class Scanner(Plugin): identifier = file_path.replace(folder, '').lstrip(os.path.sep) # root folder identifier = os.path.splitext(identifier)[0] # ext + # Exclude file name path if needed (f.e. for DVD files) + if exclude_filename: + identifier = identifier[:len(identifier) - len(os.path.split(identifier)[-1])] + # Make sure the identifier is lower case as all regex is with lower case tags identifier = identifier.lower() @@ -805,9 +809,6 @@ class Scanner(Plugin): identifier = path_split[-2] if len(path_split) > 1 and len(path_split[-2]) > len(path_split[-1]) else path_split[-1] # Only get filename except: pass - if exclude_filename: - identifier = identifier[:len(identifier) - len(os.path.split(identifier)[-1])] - # multipart identifier = self.removeMultipart(identifier) From a2544965311ab9780aad9baf3e5a6a7715a4da75 Mon Sep 17 00:00:00 2001 From: mescon Date: Sat, 8 Aug 2015 17:51:32 +0200 Subject: [PATCH 3/4] Add support for HTTP proxies Closes [#5195](https://github.com/RuudBurger/CouchPotatoServer/issues/5195). --- couchpotato/core/_base/_core.py | 19 +++++++++++++++++++ couchpotato/core/plugins/base.py | 19 ++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/_base/_core.py b/couchpotato/core/_base/_core.py index f455340..81f5fa5 100644 --- a/couchpotato/core/_base/_core.py +++ b/couchpotato/core/_base/_core.py @@ -276,6 +276,25 @@ config = [{ 'description': 'Let 3rd party app do stuff. Docs', }, { + 'name': 'use_proxy', + 'default': 0, + 'type': 'bool', + 'description': 'Route outbound connections via proxy. Currently, only HTTP(S) proxies are supported. ', + }, + { + 'name': 'proxy_server', + 'description': 'Override system default proxy server. Currently, only HTTP(S) proxies are supported. Ex. \"127.0.0.1:8080\". Keep empty to use system default proxy server.', + }, + { + 'name': 'proxy_username', + 'description': 'Only HTTP Basic Auth is supported. Leave blank to disable authentication.', + }, + { + 'name': 'proxy_password', + 'type': 'password', + 'description': 'Leave blank for no password.', + }, + { 'name': 'debug', 'default': 0, 'type': 'bool', diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 57a31e2..d82473d 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -1,5 +1,5 @@ import threading -from urllib import quote +from urllib import quote, getproxies from urlparse import urlparse import glob import inspect @@ -200,6 +200,22 @@ class Plugin(object): headers['Connection'] = headers.get('Connection', 'keep-alive') headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0') + use_proxy = Env.setting('use_proxy') + proxy_server = Env.setting('proxy_server') + proxy_username = Env.setting('proxy_username') + proxy_password = Env.setting('proxy_password') + proxy_url = None + + if use_proxy: + if proxy_server: + loc = "{0}:{1}@{2}".format(proxy_username, proxy_password, proxy_server) if proxy_username else proxy_server + proxy_url = { + "http": "http://"+loc, + "https": "https://"+loc, + } + else: + proxy_url = getproxies() + r = Env.get('http_opener') # Don't try for failed requests @@ -225,6 +241,7 @@ class Plugin(object): 'files': files, 'verify': False, #verify_ssl, Disable for now as to many wrongly implemented certificates.. 'stream': stream, + 'proxies': proxy_url, } method = 'post' if len(data) > 0 or files else 'get' From 58a90c0ffede87b1cf2ce5a862a457103ee91a60 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 10 Aug 2015 14:04:45 +0200 Subject: [PATCH 4/4] Only get proxy settings when enabled --- couchpotato/core/plugins/base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index d82473d..8b76e11 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -201,12 +201,13 @@ class Plugin(object): headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0') use_proxy = Env.setting('use_proxy') - proxy_server = Env.setting('proxy_server') - proxy_username = Env.setting('proxy_username') - proxy_password = Env.setting('proxy_password') proxy_url = None if use_proxy: + proxy_server = Env.setting('proxy_server') + proxy_username = Env.setting('proxy_username') + proxy_password = Env.setting('proxy_password') + if proxy_server: loc = "{0}:{1}@{2}".format(proxy_username, proxy_password, proxy_server) if proxy_username else proxy_server proxy_url = {