Merge branch 'develop' of github.com:RuudBurger/CouchPotatoServer into develop

couchpotato/core/downloaders/deluge/main.py

@@ -2,7 +2,7 @@ from base64 import b64encode, b16encode, b32decode
 from bencode import bencode as benc, bdecode
 from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList
 from couchpotato.core.helpers.encoding import isInt, sp
-from couchpotato.core.helpers.variable import tryFloat
+from couchpotato.core.helpers.variable import tryFloat, cleanHost
 from couchpotato.core.logger import CPLog
 from datetime import timedelta
 from hashlib import sha1
@@ -22,7 +22,7 @@ class Deluge(Downloader):
 
     def connect(self):
         # Load host from config and split out port.
-        host = self.conf('host').split(':')
+        host = cleanHost(self.conf('host'), protocol = False).split(':')
         if not isInt(host[1]):
             log.error('Config properties are not filled in correctly, port is missing.')
             return False
@@ -103,7 +103,12 @@ class Deluge(Downloader):
 
         for torrent_id in queue:
             torrent = queue[torrent_id]
-            log.debug('name=%s / id=%s / save_path=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (torrent['name'], torrent['hash'], torrent['save_path'], torrent['move_completed_path'], torrent['hash'], torrent['progress'], torrent['state'], torrent['eta'], torrent['ratio'], torrent['stop_ratio'], torrent['is_seed'], torrent['is_finished'], torrent['paused']))
+
+            if not 'hash' in torrent:
+                # When given a list of ids, deluge will return an empty item for a non-existant torrent.
+                continue
+
+            log.debug('name=%s / id=%s / save_path=%s / move_on_completed=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (torrent['name'], torrent['hash'], torrent['save_path'], torrent['move_on_completed'], torrent['move_completed_path'], torrent['hash'], torrent['progress'], torrent['state'], torrent['eta'], torrent['ratio'], torrent['stop_ratio'], torrent['is_seed'], torrent['is_finished'], torrent['paused']))
     
             # Deluge has no easy way to work out if a torrent is stalled or failing.
             #status = 'failed'
@@ -212,7 +217,7 @@ class DelugeRPC(object):
         ret = False
         try:
             self.connect()
-            ret = self.client.core.get_torrents_status({'id': ids}, {}).get()
+            ret = self.client.core.get_torrents_status({'id': ids}, ('name', 'hash', 'save_path', 'move_completed_path', 'progress', 'state', 'eta', 'ratio', 'stop_ratio', 'is_seed', 'is_finished', 'paused', 'move_on_completed', 'files')).get()
         except Exception, err:
             log.error('Failed to get all torrents: %s %s', (err, traceback.format_exc()))
         finally:

couchpotato/core/downloaders/nzbget/main.py

@@ -1,7 +1,7 @@
 from base64 import standard_b64encode
 from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList
 from couchpotato.core.helpers.encoding import ss, sp
-from couchpotato.core.helpers.variable import tryInt, md5
+from couchpotato.core.helpers.variable import tryInt, md5, cleanHost
 from couchpotato.core.logger import CPLog
 from datetime import timedelta
 import re
@@ -17,7 +17,7 @@ class NZBGet(Downloader):
 
     protocol = ['nzb']
 
-    url = '%(protocol)s://%(username)s:%(password)s@%(host)s/xmlrpc'
+    rpc = 'xmlrpc'
 
     def download(self, data = None, media = None, filedata = None):
         if not media: media = {}
@@ -29,10 +29,11 @@ class NZBGet(Downloader):
 
         log.info('Sending "%s" to NZBGet.', data.get('name'))
 
-        url = self.url % {'protocol': 'https' if self.conf('ssl') else 'http', 'host': self.conf('host'), 'username': self.conf('username'), 'password': self.conf('password')}
         nzb_name = ss('%s.nzb' % self.createNzbName(data, media))
 
+        url = cleanHost(host = self.conf('host'), ssl = self.conf('ssl'), username = self.conf('username'), password = self.conf('password')) + self.rpc
         rpc = xmlrpclib.ServerProxy(url)
+
         try:
             if rpc.writelog('INFO', 'CouchPotato connected to drop off %s.' % nzb_name):
                 log.debug('Successfully connected to NZBGet')
@@ -71,9 +72,9 @@ class NZBGet(Downloader):
 
         log.debug('Checking NZBGet download status.')
 
-        url = self.url % {'protocol': 'https' if self.conf('ssl') else 'http', 'host': self.conf('host'), 'username': self.conf('username'), 'password': self.conf('password')}
-
+        url = cleanHost(host = self.conf('host'), ssl = self.conf('ssl'), username = self.conf('username'), password = self.conf('password')) + self.rpc
         rpc = xmlrpclib.ServerProxy(url)
+
         try:
             if rpc.writelog('INFO', 'CouchPotato connected to check status'):
                 log.debug('Successfully connected to NZBGet')
@@ -157,9 +158,9 @@ class NZBGet(Downloader):
 
         log.info('%s failed downloading, deleting...', release_download['name'])
 
-        url = self.url % {'host': self.conf('host'), 'username': self.conf('username'), 'password': self.conf('password')}
-
+        url = cleanHost(host = self.conf('host'), ssl = self.conf('ssl'), username = self.conf('username'), password = self.conf('password')) + self.rpc
         rpc = xmlrpclib.ServerProxy(url)
+
         try:
             if rpc.writelog('INFO', 'CouchPotato connected to delete some history'):
                 log.debug('Successfully connected to NZBGet')

couchpotato/core/downloaders/nzbvortex/__init__.py

@@ -22,7 +22,15 @@ config = [{
                 },
                 {
                     'name': 'host',
-                    'default': 'https://localhost:4321',
+                    'default': 'localhost:4321',
+                    'description': 'Hostname with port. Usually <strong>localhost:4321</strong>',
+                },
+                {
+                    'name': 'ssl',
+                    'default': 1,
+                    'type': 'bool',
+                    'advanced': True,
+                    'description': 'Use HyperText Transfer Protocol Secure, or <strong>https</strong>',
                 },
                 {
                     'name': 'api_key',

couchpotato/core/downloaders/nzbvortex/main.py

@@ -116,7 +116,7 @@ class NZBVortex(Downloader):
 
         params = tryUrlencode(parameters)
 
-        url = cleanHost(self.conf('host')) + 'api/' + call
+        url = cleanHost(self.conf('host'), ssl = self.conf('ssl')) + 'api/' + call
 
         try:
             data = self.urlopen('%s?%s' % (url, params), *args, **kwargs)

couchpotato/core/downloaders/rtorrent/__init__.py

@@ -20,11 +20,32 @@ config = [{
                     'type': 'enabler',
                     'radio_group': 'torrent',
                 },
+# @RuudBurger: How do I migrate this?
+#                {
+#                    'name': 'url',
+#                    'default': 'http://localhost:80/RPC2',
+#                    'description': 'XML-RPC Endpoint URI. Usually <strong>scgi://localhost:5000</strong> '
+#                                   'or <strong>http://localhost:80/RPC2</strong>'
+#                },
                 {
-                    'name': 'url',
-                    'default': 'http://localhost:80/RPC2',
-                    'description': 'XML-RPC Endpoint URI. Usually <strong>scgi://localhost:5000</strong> '
-                                   'or <strong>http://localhost:80/RPC2</strong>'
+                    'name': 'host',
+                    'default': 'localhost:80',
+                    'description': 'Hostname with port or XML-RPC Endpoint URI. Usually <strong>scgi://localhost:5000</strong> '
+                                   'or <strong>localhost:80</strong>'
+                },
+                {
+                    'name': 'ssl',
+                    'default': 0,
+                    'type': 'bool',
+                    'advanced': True,
+                    'description': 'Use HyperText Transfer Protocol Secure, or <strong>https</strong>',
+                },
+                {
+                    'name': 'rpc_url',
+                    'type': 'string',
+                    'default': 'RPC2',
+                    'advanced': True,
+                    'description': 'Change if you don\'t run rTorrent RPC at the default url.',
                 },
                 {
                     'name': 'username',

couchpotato/core/downloaders/rtorrent/main.py

@@ -1,7 +1,9 @@
 from base64 import b16encode, b32decode
 from bencode import bencode, bdecode
 from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList
+from couchpotato.core.event import fireEvent, addEvent
 from couchpotato.core.helpers.encoding import sp
+from couchpotato.core.helpers.variable import cleanHost, splitString
 from couchpotato.core.logger import CPLog
 from datetime import timedelta
 from hashlib import sha1
@@ -17,24 +19,39 @@ class rTorrent(Downloader):
     protocol = ['torrent', 'torrent_magnet']
     rt = None
 
+    # Migration url to host options
+    def __init__(self):
+        super(rTorrent, self).__init__()
+
+        addEvent('app.load', self.migrate)
+
+    def migrate(self):
+
+        url = self.conf('url')
+        if url:
+            host_split = splitString(url.split('://')[-1], split_on = '/')
+
+            self.conf('ssl', value = url.startswith('https'))
+            self.conf('host', value = host_split[0].strip())
+            self.conf('rpc_url', value = '/'.join(host_split[1:]))
+
+            self.deleteConf('url')
+
     def connect(self):
         # Already connected?
         if self.rt is not None:
             return self.rt
 
-        # Ensure url is set
-        if not self.conf('url'):
-            log.error('Config properties are not filled in correctly, url is missing.')
-            return False
+        url = cleanHost(self.conf('host'), protocol = True, ssl = self.conf('ssl')) + '/' + self.conf('rpc_url').strip('/ ') + '/'
 
         if self.conf('username') and self.conf('password'):
             self.rt = RTorrent(
-                self.conf('url'),
+                url,
                 self.conf('username'),
                 self.conf('password')
             )
         else:
-            self.rt = RTorrent(self.conf('url'))
+            self.rt = RTorrent(url)
 
         return self.rt
 
@@ -159,14 +176,14 @@ class rTorrent(Downloader):
                     torrent_files = []
                     for file_item in torrent.get_files():
                         torrent_files.append(sp(os.path.join(torrent.directory, file_item.path)))
-    
+
                     status = 'busy'
                     if torrent.complete:
                         if torrent.active:
                             status = 'seeding'
                         else:
                             status = 'completed'
-    
+
                     release_downloads.append({
                         'id': torrent.info_hash,
                         'name': torrent.name,

couchpotato/core/downloaders/sabnzbd/__init__.py

@@ -25,6 +25,13 @@ config = [{
                     'default': 'localhost:8080',
                 },
                 {
+                    'name': 'ssl',
+                    'default': 0,
+                    'type': 'bool',
+                    'advanced': True,
+                    'description': 'Use HyperText Transfer Protocol Secure, or <strong>https</strong>',
+                },
+                {
                     'name': 'api_key',
                     'label': 'Api Key',
                     'description': 'Used for all calls to Sabnzbd.',

couchpotato/core/downloaders/sabnzbd/main.py

@@ -165,7 +165,7 @@ class Sabnzbd(Downloader):
 
     def call(self, request_params, use_json = True, **kwargs):
 
-        url = cleanHost(self.conf('host')) + 'api?' + tryUrlencode(mergeDicts(request_params, {
+        url = cleanHost(self.conf('host'), ssl = self.conf('ssl')) + 'api?' + tryUrlencode(mergeDicts(request_params, {
            'apikey': self.conf('api_key'),
            'output': 'json'
         }))

couchpotato/core/downloaders/synology/main.py

@@ -1,5 +1,6 @@
 from couchpotato.core.downloaders.base import Downloader
 from couchpotato.core.helpers.encoding import isInt
+from couchpotato.core.helpers.variable import cleanHost
 from couchpotato.core.logger import CPLog
 import json
 import requests
@@ -21,7 +22,7 @@ class Synology(Downloader):
         log.error('Sending "%s" (%s) to Synology.', (data['name'], data['protocol']))
 
         # Load host from config and split out port.
-        host = self.conf('host').split(':')
+        host = cleanHost(self.conf('host'), protocol = False).split(':')
         if not isInt(host[1]):
             log.error('Config properties are not filled in correctly, port is missing.')
             return False

couchpotato/core/downloaders/transmission/main.py

@@ -1,7 +1,7 @@
 from base64 import b64encode
 from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList
 from couchpotato.core.helpers.encoding import isInt, sp
-from couchpotato.core.helpers.variable import tryInt, tryFloat
+from couchpotato.core.helpers.variable import tryInt, tryFloat, cleanHost
 from couchpotato.core.logger import CPLog
 from datetime import timedelta
 import httplib
@@ -21,13 +21,13 @@ class Transmission(Downloader):
 
     def connect(self):
         # Load host from config and split out port.
-        host = self.conf('host').split(':')
+        host = cleanHost(self.conf('host'), protocol = False).split(':')
         if not isInt(host[1]):
             log.error('Config properties are not filled in correctly, port is missing.')
             return False
 
         if not self.trpc:
-            self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url'), username = self.conf('username'), password = self.conf('password'))
+            self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url').strip('/ '), username = self.conf('username'), password = self.conf('password'))
 
         return self.trpc
 

couchpotato/core/downloaders/utorrent/main.py

@@ -2,7 +2,7 @@ from base64 import b16encode, b32decode
 from bencode import bencode as benc, bdecode
 from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList
 from couchpotato.core.helpers.encoding import isInt, ss, sp
-from couchpotato.core.helpers.variable import tryInt, tryFloat
+from couchpotato.core.helpers.variable import tryInt, tryFloat, cleanHost
 from couchpotato.core.logger import CPLog
 from datetime import timedelta
 from hashlib import sha1
@@ -37,7 +37,7 @@ class uTorrent(Downloader):
 
     def connect(self):
         # Load host from config and split out port.
-        host = self.conf('host').split(':')
+        host = cleanHost(self.conf('host'), protocol = False).split(':')
         if not isInt(host[1]):
             log.error('Config properties are not filled in correctly, port is missing.')
             return False

couchpotato/core/helpers/variable.py

@@ -118,12 +118,22 @@ def isLocalIP(ip):
 def getExt(filename):
     return os.path.splitext(filename)[1][1:]
 
-def cleanHost(host):
-    if not host.startswith(('http://', 'https://')):
-        host = 'http://' + host
+def cleanHost(host, protocol = True, ssl = False, username = None, password = None):
 
-    host = host.rstrip('/')
-    host += '/'
+    if not '://' in host and protocol:
+        host = 'https://' if ssl else 'http://' + host
+
+    if not protocol:
+        host = host.split('://', 1)[-1]
+
+    if protocol and username and password:
+        login = '%s:%s@' % (username, password)
+        if not login in host:
+            host.replace('://', '://' + login, 1) 
+
+    host = host.rstrip('/ ')
+    if protocol:
+        host += '/'
 
     return host
 

couchpotato/core/notifications/xbmc/main.py

@@ -8,6 +8,7 @@ import socket
 import traceback
 import urllib
 import requests
+from requests.packages.urllib3.exceptions import MaxRetryError
 
 log = CPLog(__name__)
 
@@ -168,7 +169,7 @@ class XBMC(Notification):
                 # manually fake expected response array
                 return [{'result': 'Error'}]
 
-        except requests.exceptions.Timeout:
+        except (MaxRetryError, requests.exceptions.Timeout):
             log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off')
             return [{'result': 'Error'}]
         except:
@@ -203,7 +204,7 @@ class XBMC(Notification):
             log.debug('Returned from request %s: %s', (host, response))
 
             return response
-        except requests.exceptions.Timeout:
+        except (MaxRetryError, requests.exceptions.Timeout):
             log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off')
             return []
         except:

couchpotato/core/plugins/base.py

@@ -5,6 +5,8 @@ from couchpotato.core.helpers.variable import getExt, md5, isLocalIP
 from couchpotato.core.logger import CPLog
 from couchpotato.environment import Env
 import requests
+from requests.packages.urllib3 import Timeout
+from requests.packages.urllib3.exceptions import MaxRetryError
 from tornado import template
 from tornado.web import StaticFileHandler
 from urlparse import urlparse
@@ -52,8 +54,11 @@ class Plugin(object):
             self.registerStatic(inspect.getfile(self.__class__))
 
     def conf(self, attr, value = None, default = None, section = None):
-        class_name = self.getName().lower().split(':')
-        return Env.setting(attr, section = section if section else class_name[0].lower(), value = value, default = default)
+        class_name = self.getName().lower().split(':')[0].lower()
+        return Env.setting(attr, section = section if section else class_name, value = value, default = default)
+
+    def deleteConf(self, attr):
+        return Env._settings.delete(attr, section = self.getName().lower().split(':')[0].lower())
 
     def getName(self):
         return self._class_name or self.__class__.__name__
@@ -170,9 +175,9 @@ class Plugin(object):
             data = response.content if return_raw else response.text
 
             self.http_failed_request[host] = 0
-        except IOError:
+        except (IOError, MaxRetryError, Timeout):
             if show_error:
-                log.error('Failed opening url in %s: %s %s', (self.getName(), url, traceback.format_exc(1)))
+                log.error('Failed opening url in %s: %s %s', (self.getName(), url, traceback.format_exc(0)))
 
             # Save failed requests by hosts
             try:
@@ -262,7 +267,7 @@ class Plugin(object):
                 if not kwargs.get('show_error', True):
                     raise
 
-                log.error('Failed getting cache: %s', (traceback.format_exc()))
+                log.debug('Failed getting cache: %s', (traceback.format_exc(0)))
                 return ''
 
     def setCache(self, cache_key, value, timeout = 300):

couchpotato/core/providers/base.py

@@ -164,7 +164,7 @@ class YarrProvider(Provider):
         try:
             if not self.login():
                 log.error('Failed downloading from %s', self.getName())
-            return self.urlopen(url)
+            return self.urlopen(url, return_raw = True)
         except:
             log.error('Failed downloading from %s: %s', (self.getName(), traceback.format_exc()))
 
@@ -173,7 +173,7 @@ class YarrProvider(Provider):
 
     def download(self, url = '', nzb_id = ''):
         try:
-            return self.urlopen(url, headers = {'User-Agent': Env.getIdentifier()}, show_error = False)
+            return self.urlopen(url, headers = {'User-Agent': Env.getIdentifier()}, show_error = False, return_raw = True)
         except:
             log.error('Failed getting nzb from %s: %s', (self.getName(), traceback.format_exc()))
 

couchpotato/core/settings/__init__.py

@@ -110,6 +110,10 @@ class Settings(object):
         except:
             return default
 
+    def delete(self, option = '', section = 'core'):
+        self.p.remove_option(section, option)
+        self.save()
+
     def getEnabler(self, section, option):
         return self.getBool(section, option)
 

couchpotato/runner.py

@@ -168,7 +168,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
         logger.addHandler(hdlr)
 
     # To file
-    hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10)
+    hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = Env.get('encoding'))
     hdlr2.setFormatter(formatter)
     logger.addHandler(hdlr2)