diff --git a/couchpotato/api.py b/couchpotato/api.py index 15ef2b4..718527c 100644 --- a/couchpotato/api.py +++ b/couchpotato/api.py @@ -1,6 +1,5 @@ from flask.blueprints import Blueprint from flask.helpers import url_for -from tornado.ioloop import IOLoop from tornado.web import RequestHandler, asynchronous from werkzeug.utils import redirect @@ -11,7 +10,11 @@ api_nonblock = {} class NonBlockHandler(RequestHandler): - stoppers = [] + + def __init__(self, application, request, **kwargs): + cls = NonBlockHandler + cls.stoppers = [] + super(NonBlockHandler, self).__init__(application, request, **kwargs) @asynchronous def get(self, route): diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index 98ab73b..7a695e2 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -1,16 +1,17 @@ from base64 import b32decode, b16encode from couchpotato.core.event import addEvent from couchpotato.core.logger import CPLog -from couchpotato.core.plugins.base import Plugin +from couchpotato.core.providers.base import Provider import random import re log = CPLog(__name__) -class Downloader(Plugin): +class Downloader(Provider): type = [] + http_time_between_calls = 0 torrent_sources = [ 'http://torrage.com/torrent/%s.torrent', diff --git a/couchpotato/core/downloaders/nzbget/__init__.py b/couchpotato/core/downloaders/nzbget/__init__.py index 8b68c95..4f5afb3 100644 --- a/couchpotato/core/downloaders/nzbget/__init__.py +++ b/couchpotato/core/downloaders/nzbget/__init__.py @@ -10,7 +10,7 @@ config = [{ 'tab': 'downloaders', 'name': 'nzbget', 'label': 'NZBGet', - 'description': 'Send NZBs to your NZBGet installation.', + 'description': 'Use NZBGet to download NZBs.', 'options': [ { 'name': 'enabled', @@ -25,6 +25,7 @@ config = [{ }, { 'name': 'password', + 'type': 'password', 'description': 'Default NZBGet password is tegbzn6789', }, { diff --git a/couchpotato/core/downloaders/nzbvortex/__init__.py b/couchpotato/core/downloaders/nzbvortex/__init__.py new file mode 100644 index 0000000..e0e7746 --- /dev/null +++ b/couchpotato/core/downloaders/nzbvortex/__init__.py @@ -0,0 +1,46 @@ +from .main import NZBVortex + +def start(): + return NZBVortex() + +config = [{ + 'name': 'nzbvortex', + 'groups': [ + { + 'tab': 'downloaders', + 'name': 'nzbvortex', + 'label': 'NZBVortex', + 'description': 'Use NZBVortex to download NZBs.', + 'wizard': True, + 'options': [ + { + 'name': 'enabled', + 'default': 0, + 'type': 'enabler', + 'radio_group': 'nzb', + }, + { + 'name': 'host', + 'default': 'https://localhost:4321', + }, + { + 'name': 'api_key', + 'label': 'Api Key', + }, + { + 'name': 'manual', + 'default': False, + 'type': 'bool', + 'advanced': True, + 'description': 'Disable this downloader for automated searches, but use it when I manually send a release.', + }, + { + 'name': 'delete_failed', + 'default': True, + 'type': 'bool', + 'description': 'Delete a release after the download has failed.', + }, + ], + } + ], +}] diff --git a/couchpotato/core/downloaders/nzbvortex/main.py b/couchpotato/core/downloaders/nzbvortex/main.py new file mode 100644 index 0000000..32b1e6e --- /dev/null +++ b/couchpotato/core/downloaders/nzbvortex/main.py @@ -0,0 +1,176 @@ +from base64 import b64encode +from couchpotato.core.downloaders.base import Downloader +from couchpotato.core.helpers.encoding import tryUrlencode, ss +from couchpotato.core.helpers.variable import cleanHost +from couchpotato.core.logger import CPLog +from urllib2 import URLError +from uuid import uuid4 +import hashlib +import httplib +import json +import socket +import ssl +import sys +import traceback +import urllib2 + +log = CPLog(__name__) + +class NZBVortex(Downloader): + + type = ['nzb'] + api_level = None + session_id = None + + def download(self, data = {}, movie = {}, manual = False, filedata = None): + + if self.isDisabled(manual) or not self.isCorrectType(data.get('type')) or not self.getApiLevel(): + return + + # Send the nzb + try: + nzb_filename = self.createFileName(data, filedata, movie) + self.call('nzb/add', params = {'file': (ss(nzb_filename), filedata)}, multipart = True) + + return True + except: + log.error('Something went wrong sending the NZB file: %s', traceback.format_exc()) + return False + + def getAllDownloadStatus(self): + + if self.isDisabled(manual = True): + return False + + raw_statuses = self.call('nzb') + + statuses = [] + for item in raw_statuses.get('nzbs', []): + + # Check status + status = 'busy' + if item['state'] == 20: + status = 'completed' + elif item['state'] in [21, 22, 24]: + status = 'failed' + + statuses.append({ + 'id': item['id'], + 'name': item['uiTitle'], + 'status': status, + 'original_status': item['state'], + 'timeleft':-1, + }) + + return statuses + + def removeFailed(self, item): + + if not self.conf('delete_failed', default = True): + return False + + log.info('%s failed downloading, deleting...', item['name']) + + try: + self.call('nzb/%s/cancel' % item['id']) + except: + log.error('Failed deleting: %s', traceback.format_exc(0)) + return False + + return True + + def login(self): + + nonce = self.call('auth/nonce', auth = False).get('authNonce') + cnonce = uuid4().hex + hashed = b64encode(hashlib.sha256('%s:%s:%s' % (nonce, cnonce, self.conf('api_key'))).digest()) + + params = { + 'nonce': nonce, + 'cnonce': cnonce, + 'hash': hashed + } + + login_data = self.call('auth/login', parameters = params, auth = False) + + # Save for later + if login_data.get('loginResult') == 'successful': + self.session_id = login_data.get('sessionID') + return True + + log.error('Login failed, please check you api-key') + return False + + + def call(self, call, parameters = {}, repeat = False, auth = True, *args, **kwargs): + + # Login first + if not self.session_id and auth: + self.login() + + # Always add session id to request + if self.session_id: + parameters['sessionid'] = self.session_id + + params = tryUrlencode(parameters) + + url = cleanHost(self.conf('host')) + 'api/' + call + url_opener = urllib2.build_opener(HTTPSHandler()) + + try: + data = self.urlopen('%s?%s' % (url, params), opener = url_opener, *args, **kwargs) + + if data: + return json.loads(data) + except URLError, e: + if hasattr(e, 'code') and e.code == 403: + # Try login and do again + if not repeat: + self.login() + return self.call(call, parameters = parameters, repeat = True, *args, **kwargs) + + log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) + except: + log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) + + return {} + + def getApiLevel(self): + + if not self.api_level: + + url = cleanHost(self.conf('host')) + 'api/app/apilevel' + url_opener = urllib2.build_opener(HTTPSHandler()) + + try: + data = self.urlopen(url, opener = url_opener, show_error = False) + self.api_level = float(json.loads(data).get('apilevel')) + except URLError, e: + if hasattr(e, 'code') and e.code == 403: + log.error('This version of NZBVortex isn\'t supported. Please update to 2.8.6 or higher') + else: + log.error('NZBVortex doesn\'t seem to be running or maybe the remote option isn\'t enabled yet: %s', traceback.format_exc(1)) + + return self.api_level + + +class HTTPSConnection(httplib.HTTPSConnection): + def __init__(self, *args, **kwargs): + httplib.HTTPSConnection.__init__(self, *args, **kwargs) + + def connect(self): + sock = socket.create_connection((self.host, self.port), self.timeout) + if sys.version_info < (2, 6, 7): + if hasattr(self, '_tunnel_host'): + self.sock = sock + self._tunnel() + else: + if self._tunnel_host: + self.sock = sock + self._tunnel() + + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version = ssl.PROTOCOL_TLSv1) + +class HTTPSHandler(urllib2.HTTPSHandler): + def https_open(self, req): + return self.do_open(HTTPSConnection, req) diff --git a/couchpotato/core/downloaders/pneumatic/__init__.py b/couchpotato/core/downloaders/pneumatic/__init__.py index 004821c..f119cfc 100644 --- a/couchpotato/core/downloaders/pneumatic/__init__.py +++ b/couchpotato/core/downloaders/pneumatic/__init__.py @@ -11,7 +11,7 @@ config = [{ 'tab': 'downloaders', 'name': 'pneumatic', 'label': 'Pneumatic', - 'description': 'Download the .strm file to a specific folder.', + 'description': 'Use Pneumatic to download .strm files.', 'options': [ { 'name': 'enabled', diff --git a/couchpotato/core/downloaders/sabnzbd/__init__.py b/couchpotato/core/downloaders/sabnzbd/__init__.py index 927a9ff..e416250 100644 --- a/couchpotato/core/downloaders/sabnzbd/__init__.py +++ b/couchpotato/core/downloaders/sabnzbd/__init__.py @@ -10,7 +10,7 @@ config = [{ 'tab': 'downloaders', 'name': 'sabnzbd', 'label': 'Sabnzbd', - 'description': 'Send NZBs to your Sabnzbd installation.', + 'description': 'Use SABnzbd to download NZBs.', 'wizard': True, 'options': [ { diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index e9b4c0a..e3848e4 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -1,5 +1,5 @@ from couchpotato.core.downloaders.base import Downloader -from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.helpers.encoding import tryUrlencode, ss from couchpotato.core.helpers.variable import cleanHost, mergeDicts from couchpotato.core.logger import CPLog from urllib2 import URLError @@ -41,7 +41,7 @@ class Sabnzbd(Downloader): try: if params.get('mode') is 'addfile': - sab = self.urlopen(url, timeout = 60, params = {'nzbfile': (nzb_filename, filedata)}, multipart = True, show_error = False) + sab = self.urlopen(url, timeout = 60, params = {'nzbfile': (ss(nzb_filename), filedata)}, multipart = True, show_error = False) else: sab = self.urlopen(url, timeout = 60, show_error = False) except URLError: @@ -65,7 +65,7 @@ class Sabnzbd(Downloader): return False def getAllDownloadStatus(self): - if self.isDisabled(manual = False): + if self.isDisabled(manual = True): return False log.debug('Checking SABnzbd download status.') diff --git a/couchpotato/core/downloaders/synology/__init__.py b/couchpotato/core/downloaders/synology/__init__.py index 2b4757a..2b7e861 100644 --- a/couchpotato/core/downloaders/synology/__init__.py +++ b/couchpotato/core/downloaders/synology/__init__.py @@ -10,7 +10,7 @@ config = [{ 'tab': 'downloaders', 'name': 'synology', 'label': 'Synology', - 'description': 'Send torrents to Synology\'s Download Station.', + 'description': 'Use Synology Download Station to download.', 'wizard': True, 'options': [ { diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index 189fcd9..0fe1184 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -10,7 +10,7 @@ config = [{ 'tab': 'downloaders', 'name': 'transmission', 'label': 'Transmission', - 'description': 'Send torrents to Transmission.', + 'description': 'Use Transmission to download torrents.', 'wizard': True, 'options': [ { diff --git a/couchpotato/core/downloaders/utorrent/__init__.py b/couchpotato/core/downloaders/utorrent/__init__.py index 88ceaf7..09a82a1 100644 --- a/couchpotato/core/downloaders/utorrent/__init__.py +++ b/couchpotato/core/downloaders/utorrent/__init__.py @@ -10,7 +10,7 @@ config = [{ 'tab': 'downloaders', 'name': 'utorrent', 'label': 'uTorrent', - 'description': 'Send torrents to uTorrent.', + 'description': 'Use uTorrent to download torrents.', 'wizard': True, 'options': [ { diff --git a/couchpotato/core/notifications/growl/main.py b/couchpotato/core/notifications/growl/main.py index 4aa1c31..7f1398d 100644 --- a/couchpotato/core/notifications/growl/main.py +++ b/couchpotato/core/notifications/growl/main.py @@ -37,8 +37,11 @@ class Growl(Notification): ) self.growl.register() self.registered = True - except: - log.error('Failed register of growl: %s', traceback.format_exc()) + except Exception, e: + if 'timed out' in str(e): + self.registered = True + else: + log.error('Failed register of growl: %s', traceback.format_exc()) def notify(self, message = '', data = {}, listener = None): if self.isDisabled(): return diff --git a/couchpotato/core/notifications/toasty/__init__.py b/couchpotato/core/notifications/toasty/__init__.py new file mode 100644 index 0000000..25d27ec --- /dev/null +++ b/couchpotato/core/notifications/toasty/__init__.py @@ -0,0 +1,32 @@ +from .main import Toasty + +def start(): + return Toasty() + +config = [{ + 'name': 'toasty', + 'groups': [ + { + 'tab': 'notifications', + 'name': 'toasty', + 'options': [ + { + 'name': 'enabled', + 'default': 0, + 'type': 'enabler', + }, + { + 'name': 'api_key', + 'label': 'Device ID', + }, + { + 'name': 'on_snatch', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Also send message when movie is snatched.', + }, + ], + } + ], +}] diff --git a/couchpotato/core/notifications/toasty/main.py b/couchpotato/core/notifications/toasty/main.py new file mode 100644 index 0000000..638c75d --- /dev/null +++ b/couchpotato/core/notifications/toasty/main.py @@ -0,0 +1,30 @@ +from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode +from couchpotato.core.logger import CPLog +from couchpotato.core.notifications.base import Notification +import traceback + +log = CPLog(__name__) + +class Toasty(Notification): + + urls = { + 'api': 'http://api.supertoasty.com/notify/%s?%s' + } + + def notify(self, message = '', data = {}, listener = None): + if self.isDisabled(): return + + data = { + 'title': self.default_title, + 'text': toUnicode(message), + 'sender': toUnicode("CouchPotato"), + 'image': 'https://raw.github.com/RuudBurger/CouchPotatoServer/master/couchpotato/static/images/homescreen.png', + } + + try: + self.urlopen(self.urls['api'] % (self.conf('api_key'), tryUrlencode(data)), show_error = False) + return True + except: + log.error('Toasty failed: %s', traceback.format_exc()) + + return False diff --git a/couchpotato/core/notifications/xbmc/__init__.py b/couchpotato/core/notifications/xbmc/__init__.py index 588c5e2..ee2b4cc 100644 --- a/couchpotato/core/notifications/xbmc/__init__.py +++ b/couchpotato/core/notifications/xbmc/__init__.py @@ -10,6 +10,7 @@ config = [{ 'tab': 'notifications', 'name': 'xbmc', 'label': 'XBMC', + 'description': 'v11 (Eden) and v12 (Frodo)', 'options': [ { 'name': 'enabled', diff --git a/couchpotato/core/notifications/xbmc/main.py b/couchpotato/core/notifications/xbmc/main.py index 96bb2cf..eef9469 100755 --- a/couchpotato/core/notifications/xbmc/main.py +++ b/couchpotato/core/notifications/xbmc/main.py @@ -4,6 +4,7 @@ from couchpotato.core.notifications.base import Notification from flask.helpers import json import base64 import traceback +import urllib log = CPLog(__name__) @@ -11,27 +12,147 @@ log = CPLog(__name__) class XBMC(Notification): listen_to = ['renamer.after'] + use_json_notifications = {} def notify(self, message = '', data = {}, listener = None): if self.isDisabled(): return hosts = splitString(self.conf('host')) + successful = 0 for host in hosts: - response = self.request(host, [ - ('GUI.ShowNotification', {"title":"CouchPotato", "message":message}), - ('VideoLibrary.Scan', {}), - ]) + + if self.use_json_notifications.get(host) is None: + self.getXBMCJSONversion(host, message = message) + + if self.use_json_notifications.get(host): + response = self.request(host, [ + ('GUI.ShowNotification', {'title':self.default_title, 'message':message}), + ('VideoLibrary.Scan', {}), + ]) + else: + response = self.notifyXBMCnoJSON(host, {'title':self.default_title, 'message':message}) + response += self.request(host, [('VideoLibrary.Scan', {})]) try: for result in response: - if result['result'] == "OK": + if (result.get('result') and result['result'] == 'OK'): successful += 1 + elif (result.get('error')): + log.error('XBMC error; %s: %s (%s)', (result['id'], result['error']['message'], result['error']['code'])) + except: log.error('Failed parsing results: %s', traceback.format_exc()) return successful == len(hosts) * 2 + def getXBMCJSONversion(self, host, message = ''): + + success = False + + # XBMC JSON-RPC version request + response = self.request(host, [ + ('JSONRPC.Version', {}) + ]) + for result in response: + if (result.get('result') and type(result['result']['version']).__name__ == 'int'): + # only v2 and v4 return an int object + # v6 (as of XBMC v12(Frodo)) is required to send notifications + xbmc_rpc_version = str(result['result']['version']) + + log.debug('XBMC JSON-RPC Version: %s ; Notifications by JSON-RPC only supported for v6 [as of XBMC v12(Frodo)]', xbmc_rpc_version) + + # disable JSON use + self.use_json_notifications[host] = False + + # send the text message + resp = self.notifyXBMCnoJSON(host, {'title':self.default_title, 'message':message}) + for result in resp: + if (result.get('result') and result['result'] == 'OK'): + log.debug('Message delivered successfully!') + success = True + break + elif (result.get('error')): + log.error('XBMC error; %s: %s (%s)', (result['id'], result['error']['message'], result['error']['code'])) + break + + elif (result.get('result') and type(result['result']['version']).__name__ == 'dict'): + # XBMC JSON-RPC v6 returns an array object containing + # major, minor and patch number + xbmc_rpc_version = str(result['result']['version']['major']) + xbmc_rpc_version += '.' + str(result['result']['version']['minor']) + xbmc_rpc_version += '.' + str(result['result']['version']['patch']) + + log.debug('XBMC JSON-RPC Version: %s', xbmc_rpc_version) + + # ok, XBMC version is supported + self.use_json_notifications[host] = True + + # send the text message + resp = self.request(host, [('GUI.ShowNotification', {'title':self.default_title, 'message':message})]) + for result in resp: + if (result.get('result') and result['result'] == 'OK'): + log.debug('Message delivered successfully!') + success = True + break + elif (result.get('error')): + log.error('XBMC error; %s: %s (%s)', (result['id'], result['error']['message'], result['error']['code'])) + break + + # error getting version info (we do have contact with XBMC though) + elif (result.get('error')): + log.error('XBMC error; %s: %s (%s)', (result['id'], result['error']['message'], result['error']['code'])) + + log.debug('Use JSON notifications: %s ', self.use_json_notifications) + + return success + + def notifyXBMCnoJSON(self, host, data): + + server = 'http://%s/xbmcCmds/' % host + + # title, message [, timeout , image #can be added!] + cmd = "xbmcHttp?command=ExecBuiltIn(Notification('%s','%s'))" % (urllib.quote(data['title']), urllib.quote(data['message'])) + server += cmd + + # I have no idea what to set to, just tried text/plain and seems to be working :) + headers = { + 'Content-Type': 'text/plain', + } + + # authentication support + if self.conf('password'): + base64string = base64.encodestring('%s:%s' % (self.conf('username'), self.conf('password'))).replace('\n', '') + headers['Authorization'] = 'Basic %s' % base64string + + try: + log.debug('Sending non-JSON-type request to %s: %s', (host, data)) + + # response wil either be 'OK': + # + #
  • OK + # + # + # or 'Error': + # + #
  • Error: + # + # + response = self.urlopen(server, headers = headers) + + if 'OK' in response: + log.debug('Returned from non-JSON-type request %s: %s', (host, response)) + # manually fake expected response array + return [{'result': 'OK'}] + else: + log.error('Returned from non-JSON-type request %s: %s', (host, response)) + # manually fake expected response array + return [{'result': 'Error'}] + + except: + log.error('Failed sending non-JSON-type request to XBMC: %s', traceback.format_exc()) + return [{'result': 'Error'}] + def request(self, host, requests): server = 'http://%s/jsonrpc' % host diff --git a/couchpotato/core/plugins/automation/main.py b/couchpotato/core/plugins/automation/main.py index f4ede40..c216688 100644 --- a/couchpotato/core/plugins/automation/main.py +++ b/couchpotato/core/plugins/automation/main.py @@ -12,7 +12,7 @@ class Automation(Plugin): fireEvent('schedule.interval', 'automation.add_movies', self.addMovies, hours = self.conf('hour', default = 12)) - if not Env.get('dev'): + if Env.get('dev'): addEvent('app.load', self.addMovies) def addMovies(self): diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 2392f25..a3d5628 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -98,6 +98,7 @@ class Plugin(object): # http request def urlopen(self, url, timeout = 30, params = None, headers = None, opener = None, multipart = False, show_error = True): + url = ss(url) if not headers: headers = {} if not params: params = {} @@ -129,8 +130,11 @@ class Plugin(object): log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data')) request = urllib2.Request(url, params, headers) - cookies = cookielib.CookieJar() - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies), MultipartPostHandler) + if opener: + opener.add_handler(MultipartPostHandler()) + else: + cookies = cookielib.CookieJar() + opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies), MultipartPostHandler) response = opener.open(request, timeout = timeout) else: diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 75982ba..91dfb33 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -33,6 +33,7 @@ class Renamer(Plugin): addEvent('renamer.check_snatched', self.checkSnatched) addEvent('app.load', self.scan) + addEvent('app.load', self.checkSnatched) if self.conf('run_every') > 0: fireEvent('schedule.interval', 'renamer.check_snatched', self.checkSnatched, minutes = self.conf('run_every')) diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 2a1d307..684f682 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -89,7 +89,7 @@ class Scanner(Plugin): '()([ab])(\.....?)$' #*a.mkv ] - cp_imdb = '(\.cp\((?Ptt[0-9{7}]+)\))' + cp_imdb = '(.cp.(?Ptt[0-9{7}]+).)' def __init__(self): @@ -341,7 +341,7 @@ class Scanner(Plugin): group['files']['movie'] = self.getMediaFiles(group['unsorted_files']) if len(group['files']['movie']) == 0: - log.error('Couldn\t find any movie files for %s', identifier) + log.error('Couldn\'t find any movie files for %s', identifier) continue log.debug('Getting metadata for %s', identifier) diff --git a/couchpotato/core/providers/automation/base.py b/couchpotato/core/providers/automation/base.py index 9f86ad1..8d08f91 100644 --- a/couchpotato/core/providers/automation/base.py +++ b/couchpotato/core/providers/automation/base.py @@ -1,13 +1,13 @@ from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.logger import CPLog -from couchpotato.core.plugins.base import Plugin +from couchpotato.core.providers.base import Provider from couchpotato.environment import Env import time log = CPLog(__name__) -class Automation(Plugin): +class Automation(Provider): enabled_option = 'automation_enabled' @@ -19,6 +19,9 @@ class Automation(Plugin): def _getMovies(self): + if self.isDisabled(): + return + if not self.canCheck(): log.debug('Just checked, skipping %s', self.getName()) return [] diff --git a/couchpotato/core/providers/automation/bluray/main.py b/couchpotato/core/providers/automation/bluray/main.py index ac28152..235a1e5 100644 --- a/couchpotato/core/providers/automation/bluray/main.py +++ b/couchpotato/core/providers/automation/bluray/main.py @@ -1,8 +1,7 @@ from couchpotato.core.helpers.rss import RSS -from couchpotato.core.helpers.variable import md5, tryInt +from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.automation.base import Automation -import xml.etree.ElementTree as XMLTree log = CPLog(__name__) @@ -14,32 +13,24 @@ class Bluray(Automation, RSS): def getIMDBids(self): - if self.isDisabled(): - return - movies = [] - cache_key = 'bluray.%s' % md5(self.rss_url) - rss_data = self.getCache(cache_key, self.rss_url) - data = XMLTree.fromstring(rss_data) - - if data is not None: - rss_movies = self.getElements(data, 'channel/item') + rss_movies = self.getRSSData(self.rss_url) - for movie in rss_movies: - name = self.getTextElement(movie, "title").lower().split("blu-ray")[0].strip("(").rstrip() - year = self.getTextElement(movie, "description").split("|")[1].strip("(").strip() + for movie in rss_movies: + name = self.getTextElement(movie, 'title').lower().split('blu-ray')[0].strip('(').rstrip() + year = self.getTextElement(movie, 'description').split('|')[1].strip('(').strip() - if not name.find("/") == -1: # make sure it is not a double movie release - continue + if not name.find('/') == -1: # make sure it is not a double movie release + continue - if tryInt(year) < self.getMinimal('year'): - continue + if tryInt(year) < self.getMinimal('year'): + continue - imdb = self.search(name, year) + imdb = self.search(name, year) - if imdb: - if self.isMinimalMovie(imdb): - movies.append(imdb['imdb']) + if imdb: + if self.isMinimalMovie(imdb): + movies.append(imdb['imdb']) return movies diff --git a/couchpotato/core/providers/automation/cp/main.py b/couchpotato/core/providers/automation/cp/main.py index 24d6909..22b7942 100644 --- a/couchpotato/core/providers/automation/cp/main.py +++ b/couchpotato/core/providers/automation/cp/main.py @@ -8,7 +8,4 @@ class CP(Automation): def getMovies(self): - if self.isDisabled(): - return - return [] diff --git a/couchpotato/core/providers/automation/imdb/main.py b/couchpotato/core/providers/automation/imdb/main.py index c232485..428a8b2 100644 --- a/couchpotato/core/providers/automation/imdb/main.py +++ b/couchpotato/core/providers/automation/imdb/main.py @@ -1,5 +1,5 @@ from couchpotato.core.helpers.rss import RSS -from couchpotato.core.helpers.variable import md5, getImdb, splitString, tryInt +from couchpotato.core.helpers.variable import getImdb, splitString, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.automation.base import Automation import traceback @@ -13,9 +13,6 @@ class IMDB(Automation, RSS): def getIMDBids(self): - if self.isDisabled(): - return - movies = [] enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))] @@ -29,8 +26,7 @@ class IMDB(Automation, RSS): continue try: - cache_key = 'imdb.rss.%s' % md5(url) - rss_data = self.getCache(cache_key, url) + rss_data = self.getHTMLData(url) imdbs = getImdb(rss_data, multiple = True) for imdb in imdbs: diff --git a/couchpotato/core/providers/automation/itunes/__init__.py b/couchpotato/core/providers/automation/itunes/__init__.py new file mode 100644 index 0000000..368c1e4 --- /dev/null +++ b/couchpotato/core/providers/automation/itunes/__init__.py @@ -0,0 +1,35 @@ +from .main import ITunes + +def start(): + return ITunes() + +config = [{ + 'name': 'itunes', + 'groups': [ + { + 'tab': 'automation', + 'name': 'itunes_automation', + 'label': 'iTunes', + 'description': 'From any iTunes Store feed. Url should be the RSS link. (uses minimal requirements)', + 'options': [ + { + 'name': 'automation_enabled', + 'default': False, + 'type': 'enabler', + }, + { + 'name': 'automation_urls_use', + 'label': 'Use', + 'default': ',', + }, + { + 'name': 'automation_urls', + 'label': 'url', + 'type': 'combined', + 'combine': ['automation_urls_use', 'automation_urls'], + 'default': 'https://itunes.apple.com/rss/topmovies/limit=25/xml,', + }, + ], + }, + ], +}] diff --git a/couchpotato/core/providers/automation/itunes/main.py b/couchpotato/core/providers/automation/itunes/main.py new file mode 100644 index 0000000..14ca2a8 --- /dev/null +++ b/couchpotato/core/providers/automation/itunes/main.py @@ -0,0 +1,63 @@ +from couchpotato.core.helpers.rss import RSS +from couchpotato.core.helpers.variable import md5, splitString, tryInt +from couchpotato.core.logger import CPLog +from couchpotato.core.providers.automation.base import Automation +from xml.etree.ElementTree import QName +import datetime +import traceback +import xml.etree.ElementTree as XMLTree + +log = CPLog(__name__) + + +class ITunes(Automation, RSS): + + interval = 1800 + + def getIMDBids(self): + + if self.isDisabled(): + return + + movies = [] + + enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))] + urls = splitString(self.conf('automation_urls')) + + namespace = 'http://www.w3.org/2005/Atom' + namespaceIM = 'http://itunes.apple.com/rss' + + index = -1 + for url in urls: + + index += 1 + if not enablers[index]: + continue + + try: + cache_key = 'itunes.rss.%s' % md5(url) + rss_data = self.getCache(cache_key, url) + + data = XMLTree.fromstring(rss_data) + + if data is not None: + entry_tag = str(QName(namespace, 'entry')) + rss_movies = self.getElements(data, entry_tag) + + for movie in rss_movies: + name_tag = str(QName(namespaceIM, 'name')) + name = self.getTextElement(movie, name_tag) + + releaseDate_tag = str(QName(namespaceIM, 'releaseDate')) + releaseDateText = self.getTextElement(movie, releaseDate_tag) + year = datetime.datetime.strptime(releaseDateText, '%Y-%m-%dT00:00:00-07:00').strftime("%Y") + + imdb = self.search(name, year) + + if imdb and self.isMinimalMovie(imdb): + movies.append(imdb['imdb']) + + except: + log.error('Failed loading iTunes rss feed: %s %s', (url, traceback.format_exc())) + + return movies diff --git a/couchpotato/core/providers/automation/kinepolis/main.py b/couchpotato/core/providers/automation/kinepolis/main.py index f6633af..4158d48 100644 --- a/couchpotato/core/providers/automation/kinepolis/main.py +++ b/couchpotato/core/providers/automation/kinepolis/main.py @@ -1,9 +1,7 @@ from couchpotato.core.helpers.rss import RSS -from couchpotato.core.helpers.variable import md5 from couchpotato.core.logger import CPLog from couchpotato.core.providers.automation.base import Automation import datetime -import xml.etree.ElementTree as XMLTree log = CPLog(__name__) @@ -15,25 +13,17 @@ class Kinepolis(Automation, RSS): def getIMDBids(self): - if self.isDisabled(): - return - movies = [] - cache_key = 'kinepolis.%s' % md5(self.rss_url) - rss_data = self.getCache(cache_key, self.rss_url) - data = XMLTree.fromstring(rss_data) - - if data is not None: - rss_movies = self.getElements(data, 'channel/item') + rss_movies = self.getRSSData(self.rss_url) - for movie in rss_movies: - name = self.getTextElement(movie, "title") - year = datetime.datetime.now().strftime("%Y") + for movie in rss_movies: + name = self.getTextElement(movie, 'title') + year = datetime.datetime.now().strftime('%Y') - imdb = self.search(name, year) + imdb = self.search(name, year) - if imdb and self.isMinimalMovie(imdb): - movies.append(imdb['imdb']) + if imdb and self.isMinimalMovie(imdb): + movies.append(imdb['imdb']) return movies diff --git a/couchpotato/core/providers/automation/moviemeter/__init__.py b/couchpotato/core/providers/automation/moviemeter/__init__.py new file mode 100644 index 0000000..8ea7c06 --- /dev/null +++ b/couchpotato/core/providers/automation/moviemeter/__init__.py @@ -0,0 +1,23 @@ +from .main import Moviemeter + +def start(): + return Moviemeter() + +config = [{ + 'name': 'moviemeter', + 'groups': [ + { + 'tab': 'automation', + 'name': 'moviemeter_automation', + 'label': 'Moviemeter', + 'description': 'Imports movies from the current top 10 of moviemeter.nl. (uses minimal requirements)', + 'options': [ + { + 'name': 'automation_enabled', + 'default': False, + 'type': 'enabler', + }, + ], + }, + ], +}] diff --git a/couchpotato/core/providers/automation/moviemeter/main.py b/couchpotato/core/providers/automation/moviemeter/main.py new file mode 100644 index 0000000..dae764b --- /dev/null +++ b/couchpotato/core/providers/automation/moviemeter/main.py @@ -0,0 +1,28 @@ +from couchpotato.core.event import fireEvent +from couchpotato.core.helpers.rss import RSS +from couchpotato.core.logger import CPLog +from couchpotato.core.providers.automation.base import Automation + +log = CPLog(__name__) + + +class Moviemeter(Automation, RSS): + + interval = 1800 + rss_url = 'http://www.moviemeter.nl/rss/cinema' + + def getIMDBids(self): + + movies = [] + + rss_movies = self.getRSSData(self.rss_url) + + for movie in rss_movies: + + name_year = fireEvent('scanner.name_year', self.getTextElement(movie, 'title'), single = True) + imdb = self.search(name_year.get('name'), name_year.get('year')) + + if imdb and self.isMinimalMovie(imdb): + movies.append(imdb['imdb']) + + return movies diff --git a/couchpotato/core/providers/automation/movies_io/main.py b/couchpotato/core/providers/automation/movies_io/main.py index 5875a4d..0737e2e 100644 --- a/couchpotato/core/providers/automation/movies_io/main.py +++ b/couchpotato/core/providers/automation/movies_io/main.py @@ -1,11 +1,8 @@ from couchpotato.core.event import fireEvent from couchpotato.core.helpers.rss import RSS -from couchpotato.core.helpers.variable import md5 +from couchpotato.core.helpers.variable import tryInt, splitString from couchpotato.core.logger import CPLog from couchpotato.core.providers.automation.base import Automation -from xml.etree.ElementTree import ParseError -import traceback -import xml.etree.ElementTree as XMLTree log = CPLog(__name__) @@ -16,39 +13,27 @@ class MoviesIO(Automation, RSS): def getIMDBids(self): - if self.isDisabled(): - return - movies = [] - enablers = self.conf('automation_urls_use').split(',') + enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))] index = -1 - for rss_url in self.conf('automation_urls').split(','): + for rss_url in splitString(self.conf('automation_urls')): index += 1 if not enablers[index]: continue - try: - cache_key = 'imdb.rss.%s' % md5(rss_url) - - rss_data = self.getCache(cache_key, rss_url, headers = {'Referer': ''}) - data = XMLTree.fromstring(rss_data) - rss_movies = self.getElements(data, 'channel/item') + rss_movies = self.getRSSData(rss_url, headers = {'Referer': ''}) - for movie in rss_movies: + for movie in rss_movies: - nameyear = fireEvent('scanner.name_year', self.getTextElement(movie, "title"), single = True) - imdb = self.search(nameyear.get('name'), nameyear.get('year'), imdb_only = True) + nameyear = fireEvent('scanner.name_year', self.getTextElement(movie, 'title'), single = True) + imdb = self.search(nameyear.get('name'), nameyear.get('year'), imdb_only = True) - if not imdb: - continue + if not imdb: + continue - movies.append(imdb) - except ParseError: - log.debug('Failed loading Movies.io watchlist, probably empty: %s', (rss_url)) - except: - log.error('Failed loading Movies.io watchlist: %s %s', (rss_url, traceback.format_exc())) + movies.append(imdb) return movies diff --git a/couchpotato/core/providers/automation/trakt/main.py b/couchpotato/core/providers/automation/trakt/main.py index 764f6cf..0109daf 100644 --- a/couchpotato/core/providers/automation/trakt/main.py +++ b/couchpotato/core/providers/automation/trakt/main.py @@ -1,9 +1,8 @@ from couchpotato.core.event import addEvent -from couchpotato.core.helpers.variable import md5, sha1 +from couchpotato.core.helpers.variable import sha1 from couchpotato.core.logger import CPLog from couchpotato.core.providers.automation.base import Automation import base64 -import json log = CPLog(__name__) @@ -25,9 +24,6 @@ class Trakt(Automation): def getIMDBids(self): - if self.isDisabled(): - return - movies = [] for movie in self.getWatchlist(): movies.append(movie.get('imdb_id')) @@ -38,22 +34,11 @@ class Trakt(Automation): method = (self.urls['watchlist'] % self.conf('automation_api_key')) + self.conf('automation_username') return self.call(method) - def call(self, method_url): - try: - if self.conf('automation_password'): - headers = { - 'Authorization': 'Basic %s' % base64.encodestring('%s:%s' % (self.conf('automation_username'), self.conf('automation_password')))[:-1] - } - else: - headers = {} - - cache_key = 'trakt.%s' % md5(method_url) - json_string = self.getCache(cache_key, self.urls['base'] + method_url, headers = headers) - if json_string: - return json.loads(json_string) - except: - log.error('Failed to get data from trakt, check your login.') - - return [] + headers = {} + if self.conf('automation_password'): + headers['Authorization'] = 'Basic %s' % base64.encodestring('%s:%s' % (self.conf('automation_username'), self.conf('automation_password')))[:-1] + + data = self.getJsonData(self.urls['base'] + method_url, headers = headers) + return data if data else [] diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index f16cf6e..9c143d8 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -44,6 +44,34 @@ class Provider(Plugin): return self.is_available.get(host, False) + def getJsonData(self, url, **kwargs): + + data = self.getCache(md5(url), url, **kwargs) + + if data: + try: + return json.loads(data) + except: + log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) + + return [] + + def getRSSData(self, url, **kwargs): + + data = self.getCache(md5(url), url, **kwargs) + + if data: + try: + data = XMLTree.fromstring(data) + return self.getElements(data, 'channel/item') + except: + log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) + + return [] + + def getHTMLData(self, url, **kwargs): + return self.getCache(md5(url), url, **kwargs) + class YarrProvider(Provider): @@ -67,15 +95,20 @@ class YarrProvider(Provider): urllib2.install_opener(opener) log.info2('Logging into %s', self.urls['login']) f = opener.open(self.urls['login'], self.getLoginParams()) - f.read() + output = f.read() f.close() - self.login_opener = opener - return True + + if self.loginSuccess(output): + self.login_opener = opener + return True except: log.error('Failed to login %s: %s', (self.getName(), traceback.format_exc())) return False + def loginSuccess(self, output): + return True + def loginDownload(self, url = '', nzb_id = ''): try: if not self.login_opener and not self.login(): @@ -106,11 +139,11 @@ class YarrProvider(Provider): return [] # Create result container - imdb_result = hasattr(self, '_search') - results = ResultList(self, movie, quality, imdb_result = imdb_result) + imdb_results = hasattr(self, '_search') + results = ResultList(self, movie, quality, imdb_results = imdb_results) # Do search based on imdb id - if imdb_result: + if imdb_results: self._search(movie, quality, results) # Search possible titles else: @@ -165,34 +198,6 @@ class YarrProvider(Provider): return [self.cat_backup_id] - def getJsonData(self, url, **kwargs): - - data = self.getCache(md5(url), url, **kwargs) - - if data: - try: - return json.loads(data) - except: - log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) - - return [] - - def getRSSData(self, url, **kwargs): - - data = self.getCache(md5(url), url, **kwargs) - - if data: - try: - data = XMLTree.fromstring(data) - return self.getElements(data, 'channel/item') - except: - log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) - - return [] - - def getHTMLData(self, url, **kwargs): - return self.getCache(md5(url), url, **kwargs) - class ResultList(list): diff --git a/couchpotato/core/providers/movie/imdbapi/__init__.py b/couchpotato/core/providers/movie/imdbapi/__init__.py deleted file mode 100644 index dd1202e..0000000 --- a/couchpotato/core/providers/movie/imdbapi/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .main import IMDBAPI - -def start(): - return IMDBAPI() - -config = [] diff --git a/couchpotato/core/providers/movie/omdbapi/__init__.py b/couchpotato/core/providers/movie/omdbapi/__init__.py new file mode 100644 index 0000000..765662e --- /dev/null +++ b/couchpotato/core/providers/movie/omdbapi/__init__.py @@ -0,0 +1,6 @@ +from .main import OMDBAPI + +def start(): + return OMDBAPI() + +config = [] diff --git a/couchpotato/core/providers/movie/imdbapi/main.py b/couchpotato/core/providers/movie/omdbapi/main.py similarity index 94% rename from couchpotato/core/providers/movie/imdbapi/main.py rename to couchpotato/core/providers/movie/omdbapi/main.py index 53f503e..cdfece0 100644 --- a/couchpotato/core/providers/movie/imdbapi/main.py +++ b/couchpotato/core/providers/movie/omdbapi/main.py @@ -10,11 +10,11 @@ import traceback log = CPLog(__name__) -class IMDBAPI(MovieProvider): +class OMDBAPI(MovieProvider): urls = { - 'search': 'http://www.imdbapi.com/?%s', - 'info': 'http://www.imdbapi.com/?i=%s', + 'search': 'http://www.omdbapi.com/?%s', + 'info': 'http://www.omdbapi.com/?i=%s', } http_time_between_calls = 0 @@ -32,7 +32,7 @@ class IMDBAPI(MovieProvider): 'name': q } - cache_key = 'imdbapi.cache.%s' % q + cache_key = 'omdbapi.cache.%s' % q cached = self.getCache(cache_key, self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')}), timeout = 3) if cached: @@ -50,7 +50,7 @@ class IMDBAPI(MovieProvider): if not identifier: return {} - cache_key = 'imdbapi.cache.%s' % identifier + cache_key = 'omdbapi.cache.%s' % identifier cached = self.getCache(cache_key, self.urls['info'] % identifier, timeout = 3) if cached: diff --git a/couchpotato/core/providers/nzb/ftdworld/main.py b/couchpotato/core/providers/nzb/ftdworld/main.py index 81ac8ed..c5a0665 100644 --- a/couchpotato/core/providers/nzb/ftdworld/main.py +++ b/couchpotato/core/providers/nzb/ftdworld/main.py @@ -1,12 +1,10 @@ -from bs4 import BeautifulSoup from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.nzb.base import NZBProvider from couchpotato.environment import Env from dateutil.parser import parse -import re -import time +import traceback log = CPLog(__name__) @@ -14,7 +12,7 @@ log = CPLog(__name__) class FTDWorld(NZBProvider): urls = { - 'search': 'http://ftdworld.net/category.php?%s', + 'search': 'http://ftdworld.net/api/index.php?%s', 'detail': 'http://ftdworld.net/spotinfo.php?id=%s', 'download': 'http://ftdworld.net/cgi-bin/nzbdown.pl?fileID=%s', 'login': 'http://ftdworld.net/index.php', @@ -25,7 +23,7 @@ class FTDWorld(NZBProvider): cat_ids = [ ([4, 11], ['dvdr']), ([1], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr', 'brrip']), - ([10, 13, 14], ['bd50', '720p', '1080p']), + ([7, 10, 13, 14], ['bd50', '720p', '1080p']), ] cat_backup_id = 1 @@ -43,38 +41,29 @@ class FTDWorld(NZBProvider): 'ctype': ','.join([str(x) for x in self.getCatId(quality['identifier'])]), }) - data = self.getHTMLData(self.urls['search'] % params, opener = self.login_opener) + data = self.getJsonData(self.urls['search'] % params, opener = self.login_opener) if data: try: - html = BeautifulSoup(data) - main_table = html.find('table', attrs = {'id':'ftdresult'}) - - if not main_table: + if data.get('numRes') == 0: return - items = main_table.find_all('tr', attrs = {'class': re.compile('tcontent')}) - - for item in items: - tds = item.find_all('td') - nzb_id = tryInt(item.attrs['data-spot']) - - up = item.find('img', attrs = {'src': re.compile('up.png')}) - down = item.find('img', attrs = {'src': re.compile('down.png')}) + for item in data.get('data'): + nzb_id = tryInt(item.get('id')) results.append({ 'id': nzb_id, - 'name': toUnicode(item.find('a', attrs = {'href': re.compile('./spotinfo')}).text.strip()), - 'age': self.calculateAge(int(time.mktime(parse(tds[2].text).timetuple()))), + 'name': toUnicode(item.get('Title')), + 'age': self.calculateAge(tryInt(item.get('Created'))), 'url': self.urls['download'] % nzb_id, 'download': self.loginDownload, 'detail_url': self.urls['detail'] % nzb_id, - 'score': (tryInt(up.attrs['title'].split(' ')[0]) * 3) - (tryInt(down.attrs['title'].split(' ')[0]) * 3) if up else 0, + 'score': (tryInt(item.get('webPlus', 0)) - tryInt(item.get('webMin', 0))) * 3, }) except: - log.error('Failed to parse HTML response from FTDWorld') + log.error('Failed to parse HTML response from FTDWorld: %s', traceback.format_exc()) def getLoginParams(self): return tryUrlencode({ @@ -82,3 +71,6 @@ class FTDWorld(NZBProvider): 'passlogin': self.conf('password'), 'submit': 'Log In', }) + + def loginSuccess(self, output): + return 'password is incorrect' not in output diff --git a/couchpotato/core/providers/nzb/newznab/__init__.py b/couchpotato/core/providers/nzb/newznab/__init__.py index c9507f9..1e76d1c 100644 --- a/couchpotato/core/providers/nzb/newznab/__init__.py +++ b/couchpotato/core/providers/nzb/newznab/__init__.py @@ -13,7 +13,7 @@ config = [{ 'order': 10, 'description': 'Enable NewzNab providers such as NZB.su, \ NZBs.org, DOGnzb.cr, \ - Spotweb', + Spotweb or NZBGeek', 'wizard': True, 'options': [ { @@ -22,16 +22,16 @@ config = [{ }, { 'name': 'use', - 'default': '0,0,0' + 'default': '0,0,0,0' }, { 'name': 'host', - 'default': 'nzb.su,dognzb.cr,nzbs.org', + 'default': 'nzb.su,dognzb.cr,nzbs.org,https://index.nzbgeek.info', 'description': 'The hostname of your newznab provider', }, { 'name': 'api_key', - 'default': ',,', + 'default': ',,,', 'label': 'Api Key', 'description': 'Can be found on your profile page', 'type': 'combined', diff --git a/couchpotato/core/providers/nzb/newznab/main.py b/couchpotato/core/providers/nzb/newznab/main.py index c18724a..f1f0d48 100644 --- a/couchpotato/core/providers/nzb/newznab/main.py +++ b/couchpotato/core/providers/nzb/newznab/main.py @@ -29,7 +29,7 @@ class Newznab(NZBProvider, RSS): def search(self, movie, quality): hosts = self.getHosts() - results = ResultList(self, movie, quality, imdb_result = True) + results = ResultList(self, movie, quality, imdb_results = True) for host in hosts: if self.isDisabled(host): @@ -136,6 +136,6 @@ class Newznab(NZBProvider, RSS): self.limits_reached[host] = time.time() return 'try_next' - log.error('Failed download from %s', (host, traceback.format_exc())) + log.error('Failed download from %s: %s', (host, traceback.format_exc())) return 'try_next' diff --git a/couchpotato/core/providers/nzb/nzbx/__init__.py b/couchpotato/core/providers/nzb/nzbx/__init__.py index 129fba0..f1c7d58 100644 --- a/couchpotato/core/providers/nzb/nzbx/__init__.py +++ b/couchpotato/core/providers/nzb/nzbx/__init__.py @@ -10,7 +10,7 @@ config = [{ 'tab': 'searcher', 'subtab': 'nzb_providers', 'name': 'nzbX', - 'description': 'Free provider, less accurate. See nzbX', + 'description': 'Free provider. See nzbX', 'options': [ { 'name': 'enabled', diff --git a/couchpotato/core/providers/nzb/nzbx/main.py b/couchpotato/core/providers/nzb/nzbx/main.py index 97ee9cd..ec7fbfe 100644 --- a/couchpotato/core/providers/nzb/nzbx/main.py +++ b/couchpotato/core/providers/nzb/nzbx/main.py @@ -2,6 +2,7 @@ from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.nzb.base import NZBProvider +from couchpotato.environment import Env log = CPLog(__name__) @@ -22,7 +23,7 @@ class Nzbx(NZBProvider): 'q': movie['library']['identifier'].replace('tt', ''), 'sf': quality.get('size_min'), }) - nzbs = self.getJsonData(self.urls['search'] % arguments) + nzbs = self.getJsonData(self.urls['search'] % arguments, headers = {'User-Agent': Env.getIdentifier()}) for nzb in nzbs: diff --git a/couchpotato/core/providers/nzb/omgwtfnzbs/main.py b/couchpotato/core/providers/nzb/omgwtfnzbs/main.py index f3de767..0a18b8f 100644 --- a/couchpotato/core/providers/nzb/omgwtfnzbs/main.py +++ b/couchpotato/core/providers/nzb/omgwtfnzbs/main.py @@ -14,7 +14,7 @@ log = CPLog(__name__) class OMGWTFNZBs(NZBProvider, RSS): urls = { - 'search': 'http://rss.omgwtfnzbs.com/rss-search.php?%s', + 'search': 'http://rss.omgwtfnzbs.org/rss-search.php?%s', } http_time_between_calls = 1 #seconds diff --git a/couchpotato/runner.py b/couchpotato/runner.py index f55c65b..c0b7eb8 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -4,7 +4,6 @@ from couchpotato.api import api, NonBlockHandler from couchpotato.core.event import fireEventAsync, fireEvent from couchpotato.core.helpers.variable import getDataDir, tryInt from logging import handlers -from tornado.ioloop import IOLoop from tornado.web import Application, FallbackHandler from tornado.wsgi import WSGIContainer from werkzeug.contrib.cache import FileSystemCache @@ -231,6 +230,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En fireEventAsync('app.load') # Go go go! + from tornado.ioloop import IOLoop web_container = WSGIContainer(app) web_container._log = _log loop = IOLoop.instance() diff --git a/couchpotato/static/scripts/api.js b/couchpotato/static/scripts/api.js index f14eb14..5e507bc 100644 --- a/couchpotato/static/scripts/api.js +++ b/couchpotato/static/scripts/api.js @@ -13,7 +13,7 @@ var ApiClass = new Class({ return new Request[r_type](Object.merge({ 'callbackKey': 'callback_func', 'method': 'get', - 'url': self.createUrl(type), + 'url': self.createUrl(type, {'t': randomString()}), }, options)).send() }, diff --git a/init/freebsd b/init/freebsd index eeba51d..d389933 100644 --- a/init/freebsd +++ b/init/freebsd @@ -1,89 +1,49 @@ #!/bin/sh # # PROVIDE: couchpotato -# REQUIRE: sabnzbd +# REQUIRE: DAEMON # KEYWORD: shutdown -# -# Add the following lines to /etc/rc.conf.local or /etc/rc.conf -# to enable this service: -# -# couchpotato_enable (bool): Set to NO by default. -# Set it to YES to enable it. -# couchpotato_user: The user account CouchPotato daemon runs as what -# you want it to be. It uses '_sabnzbd' user by -# default. Do not sets it as empty or it will run -# as root. -# couchpotato_dir: Directory where CouchPotato lives. -# Default: /usr/local/couchpotato -# couchpotato_chdir: Change to this directory before running CouchPotato. -# Default is same as couchpotato_dir. -# couchpotato_pid: The name of the pidfile to create. -# Default is couchpotato.pid in couchpotato_dir. + +# Add the following lines to /etc/rc.conf to enable couchpotato: +# couchpotato_enable: Set to NO by default. Set it to YES to enable it. +# couchpotato_user: The user account CouchPotato daemon runs as what +# you want it to be. +# couchpotato_dir: Directory where CouchPotato lives. +# Default: /usr/local/CouchPotatoServer +# couchpotato_datadir: Directory where CouchPotato user data lives. +# Default: $couchpotato_dir/data +# couchpotato_conf: Directory where CouchPotato user data lives. +# Default: $couchpotato_datadir/settings.conf +# couchpotato_pid: Full path to PID file. +# Default: $couchpotato_datadir/couchpotato.pid +# couchpotato_flags: Set additonal flags as needed. . /etc/rc.subr name="couchpotato" -rcvar=${name}_enable +rcvar=couchpotato_enable load_rc_config ${name} -: ${couchpotato_enable:="NO"} -: ${couchpotato_user:="_sabnzbd"} -: ${couchpotato_dir:="/usr/local/couchpotato"} -: ${couchpotato_chdir:="${couchpotato_dir}"} -: ${couchpotato_pid:="${couchpotato_dir}/couchpotato.pid"} -: ${couchpotato_conf:="${couchpotato_dir}/data/settings.conf"} - -WGET="/usr/local/bin/wget" # You need wget for this script to safely shutdown CouchPotato. -if [ -e "${couchpotato_conf}" ]; then - HOST=`grep -A14 "\[core\]" "${couchpotato_conf}"|egrep "^host"|perl -wple 's/^host = (.*)$/$1/'` - PORT=`grep -A14 "\[core\]" "${couchpotato_conf}"|egrep "^port"|perl -wple 's/^port = (.*)$/$1/'` - CPAPI=`grep -A14 "\[core\]" "${couchpotato_conf}"|egrep "^api_key"|perl -wple 's/^api_key = (.*)$/$1/'` +: ${couchpotato_enable:=NO} +: ${couchpotato_user:=} #default is root +: ${couchpotato_dir:=/usr/local/CouchPotatoServer} +: ${couchpotato_datadir:=${couchpotato_dir}/data} +: ${couchpotato_conf:=} #default is datadir/settings.conf +: ${couchpotato_pid:=} #default is datadir/couchpotato.pid +: ${couchpotato_flags:=} + +command="${couchpotato_dir}/CouchPotato.py" +command_interpreter="/usr/local/bin/python" +command_args="--daemon --data_dir ${couchpotato_datadir}" + +# append optional flags +if [ -n "${couchpotato_pid}" ]; then + pidfile=${couchpotato_pid} + couchpotato_flags="${couchpotato_flags} --pid_file ${couchpotato_pid}" fi - -status_cmd="${name}_status" -stop_cmd="${name}_stop" - -command="/usr/sbin/daemon" -command_args="-f -p ${couchpotato_pid} python ${couchpotato_dir}/CouchPotato.py ${couchpotato_flags}" - -# Check for wget and refuse to start without it. -if [ ! -x "${WGET}" ]; then - warn "couchpotato not started: You need wget to safely shut down CouchPotato." - exit 1 +if [ -n "${couchpotato_conf}" ]; then + couchpotato_flags="${couchpotato_flags} --config_file ${couchpotato_conf}" fi -# Ensure user is root when running this script. -if [ `id -u` != "0" ]; then - echo "Oops, you should be root before running this!" - exit 1 -fi - -verify_couchpotato_pid() { - # Make sure the pid corresponds to the CouchPotato process. - pid=`cat ${couchpotato_pid} 2>/dev/null` - ps -p ${pid} | grep -q "python ${couchpotato_dir}/CouchPotato.py" - return $? -} - -# Try to stop CouchPotato cleanly by calling shutdown over http. -couchpotato_stop() { - if [ ! -e "${couchpotato_conf}" ]; then - echo "CouchPotato's settings file does not exist. Try starting CouchPotato, as this should create the file." - exit 1 - fi - echo "Stopping $name" - verify_couchpotato_pid - ${WGET} -O - -q "http://${HOST}:${PORT}/api/${CPAPI}/app.shutdown/" >/dev/null - if [ -n "${pid}" ]; then - wait_for_pids ${pid} - echo "Stopped" - fi -} - -couchpotato_status() { - verify_couchpotato_pid && echo "$name is running as ${pid}" || echo "$name is not running" -} - run_rc_command "$1" - diff --git a/libs/gntp/__init__.py b/libs/gntp/__init__.py index 7e0d60f..eabbfa4 100755 --- a/libs/gntp/__init__.py +++ b/libs/gntp/__init__.py @@ -1,8 +1,9 @@ -import hashlib import re +import hashlib import time +import StringIO -__version__ = '0.6' +__version__ = '0.8' #GNTP/ [:][ :.] GNTP_INFO_LINE = re.compile( @@ -19,7 +20,7 @@ GNTP_INFO_LINE_SHORT = re.compile( GNTP_HEADER = re.compile('([\w-]+):(.+)') -GNTP_EOL = u'\r\n' +GNTP_EOL = '\r\n' class BaseError(Exception): @@ -43,6 +44,14 @@ class UnsupportedError(BaseError): errordesc = 'Currently unsupported by gntp.py' +class _GNTPBuffer(StringIO.StringIO): + """GNTP Buffer class""" + def writefmt(self, message = "", *args): + """Shortcut function for writing GNTP Headers""" + self.write((message % args).encode('utf8', 'replace')) + self.write(GNTP_EOL) + + class _GNTPBase(object): """Base initilization @@ -206,8 +215,8 @@ class _GNTPBase(object): if not match: continue - key = match.group(1).strip() - val = match.group(2).strip() + key = unicode(match.group(1).strip(), 'utf8', 'replace') + val = unicode(match.group(2).strip(), 'utf8', 'replace') dict[key] = val return dict @@ -217,6 +226,15 @@ class _GNTPBase(object): else: self.headers[key] = unicode('%s' % value, 'utf8', 'replace') + def add_resource(self, data): + """Add binary resource + + :param string data: Binary Data + """ + identifier = hashlib.md5(data).hexdigest() + self.resources[identifier] = data + return 'x-growl-resource://%s' % identifier + def decode(self, data, password = None): """Decode GNTP Message @@ -229,19 +247,30 @@ class _GNTPBase(object): self.headers = self._parse_dict(parts[0]) def encode(self): - """Encode a GNTP Message + """Encode a generic GNTP Message - :return string: Encoded GNTP Message ready to be sent + :return string: GNTP Message ready to be sent """ - self.validate() - message = self._format_info() + GNTP_EOL + buffer = _GNTPBuffer() + + buffer.writefmt(self._format_info()) + #Headers for k, v in self.headers.iteritems(): - message += u'%s: %s%s' % (k, v, GNTP_EOL) + buffer.writefmt('%s: %s', k, v) + buffer.writefmt() - message += GNTP_EOL - return message + #Resources + for resource, data in self.resources.iteritems(): + buffer.writefmt('Identifier: %s', resource) + buffer.writefmt('Length: %d', len(data)) + buffer.writefmt() + buffer.write(data) + buffer.writefmt() + buffer.writefmt() + + return buffer.getvalue() class GNTPRegister(_GNTPBase): @@ -290,7 +319,7 @@ class GNTPRegister(_GNTPBase): for i, part in enumerate(parts): if i == 0: - continue # Skip Header + continue # Skip Header if part.strip() == '': continue notice = self._parse_dict(part) @@ -319,22 +348,33 @@ class GNTPRegister(_GNTPBase): :return string: Encoded GNTP Registration message """ - self.validate() - message = self._format_info() + GNTP_EOL + buffer = _GNTPBuffer() + + buffer.writefmt(self._format_info()) + #Headers for k, v in self.headers.iteritems(): - message += u'%s: %s%s' % (k, v, GNTP_EOL) + buffer.writefmt('%s: %s', k, v) + buffer.writefmt() #Notifications if len(self.notifications) > 0: for notice in self.notifications: - message += GNTP_EOL for k, v in notice.iteritems(): - message += u'%s: %s%s' % (k, v, GNTP_EOL) + buffer.writefmt('%s: %s', k, v) + buffer.writefmt() + + #Resources + for resource, data in self.resources.iteritems(): + buffer.writefmt('Identifier: %s', resource) + buffer.writefmt('Length: %d', len(data)) + buffer.writefmt() + buffer.write(data) + buffer.writefmt() + buffer.writefmt() - message += GNTP_EOL - return message + return buffer.getvalue() class GNTPNotice(_GNTPBase): @@ -379,7 +419,7 @@ class GNTPNotice(_GNTPBase): for i, part in enumerate(parts): if i == 0: - continue # Skip Header + continue # Skip Header if part.strip() == '': continue notice = self._parse_dict(part) @@ -388,21 +428,6 @@ class GNTPNotice(_GNTPBase): #open('notice.png','wblol').write(notice['Data']) self.resources[notice.get('Identifier')] = notice - def encode(self): - """Encode a GNTP Notification Message - - :return string: GNTP Notification Message ready to be sent - """ - self.validate() - - message = self._format_info() + GNTP_EOL - #Headers - for k, v in self.headers.iteritems(): - message += u'%s: %s%s' % (k, v, GNTP_EOL) - - message += GNTP_EOL - return message - class GNTPSubscribe(_GNTPBase): """Represents a GNTP Subscribe Command @@ -457,7 +482,8 @@ class GNTPError(_GNTPBase): self.add_header('Error-Description', errordesc) def error(self): - return self.headers['Error-Code'], self.headers['Error-Description'] + return (self.headers.get('Error-Code', None), + self.headers.get('Error-Description', None)) def parse_gntp(data, password = None): diff --git a/libs/gntp/notifier.py b/libs/gntp/notifier.py index 300e4a6..539dae2 100755 --- a/libs/gntp/notifier.py +++ b/libs/gntp/notifier.py @@ -22,43 +22,6 @@ __all__ = [ logger = logging.getLogger(__name__) -def mini(description, applicationName = 'PythonMini', noteType = "Message", - title = "Mini Message", applicationIcon = None, hostname = 'localhost', - password = None, port = 23053, sticky = False, priority = None, - callback = None): - """Single notification function - - Simple notification function in one line. Has only one required parameter - and attempts to use reasonable defaults for everything else - :param string description: Notification message - - .. warning:: - For now, only URL callbacks are supported. In the future, the - callback argument will also support a function - """ - growl = GrowlNotifier( - applicationName = applicationName, - notifications = [noteType], - defaultNotifications = [noteType], - hostname = hostname, - password = password, - port = port, - ) - result = growl.register() - if result is not True: - return result - - return growl.notify( - noteType = noteType, - title = title, - description = description, - icon = applicationIcon, - sticky = sticky, - priority = priority, - callback = callback, - ) - - class GrowlNotifier(object): """Helper class to simplfy sending Growl messages @@ -93,10 +56,12 @@ class GrowlNotifier(object): def _checkIcon(self, data): ''' Check the icon to see if it's valid - @param data: - @todo Consider checking for a valid URL + + If it's a simple URL icon, then we return True. If it's a data icon + then we return False ''' - return data + logger.info('Checking icon') + return data.startswith('http') def register(self): """Send GNTP Registration @@ -112,7 +77,11 @@ class GrowlNotifier(object): enabled = notification in self.defaultNotifications register.add_notification(notification, enabled) if self.applicationIcon: - register.add_header('Application-Icon', self.applicationIcon) + if self._checkIcon(self.applicationIcon): + register.add_header('Application-Icon', self.applicationIcon) + else: + id = register.add_resource(self.applicationIcon) + register.add_header('Application-Icon', id) if self.password: register.set_password(self.password, self.passwordHash) self.add_origin_info(register) @@ -120,7 +89,7 @@ class GrowlNotifier(object): return self._send('register', register) def notify(self, noteType, title, description, icon = None, sticky = False, - priority = None, callback = None): + priority = None, callback = None, identifier = None): """Send a GNTP notifications .. warning:: @@ -151,11 +120,18 @@ class GrowlNotifier(object): if priority: notice.add_header('Notification-Priority', priority) if icon: - notice.add_header('Notification-Icon', self._checkIcon(icon)) + if self._checkIcon(icon): + notice.add_header('Notification-Icon', icon) + else: + id = notice.add_resource(icon) + notice.add_header('Notification-Icon', id) + if description: notice.add_header('Notification-Text', description) if callback: notice.add_header('Notification-Callback-Target', callback) + if identifier: + notice.add_header('Notification-Coalescing-ID', identifier) self.add_origin_info(notice) self.notify_hook(notice) @@ -193,9 +169,10 @@ class GrowlNotifier(object): def subscribe_hook(self, packet): pass - def _send(self, type, packet): + def _send(self, messagetype, packet): """Send the GNTP Packet""" + packet.validate() data = packet.encode() logger.debug('To : %s:%s <%s>\n%s', self.hostname, self.port, packet.__class__, data) @@ -203,7 +180,7 @@ class GrowlNotifier(object): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(self.socketTimeout) s.connect((self.hostname, self.port)) - s.send(data.encode('utf8', 'replace')) + s.send(data) recv_data = s.recv(1024) while not recv_data.endswith("\r\n\r\n"): recv_data += s.recv(1024) @@ -212,11 +189,51 @@ class GrowlNotifier(object): logger.debug('From : %s:%s <%s>\n%s', self.hostname, self.port, response.__class__, response) - if response.info['messagetype'] == '-OK': + if type(response) == gntp.GNTPOK: return True logger.error('Invalid response: %s', response.error()) return response.error() + +def mini(description, applicationName = 'PythonMini', noteType = "Message", + title = "Mini Message", applicationIcon = None, hostname = 'localhost', + password = None, port = 23053, sticky = False, priority = None, + callback = None, notificationIcon = None, identifier = None, + notifierFactory = GrowlNotifier): + """Single notification function + + Simple notification function in one line. Has only one required parameter + and attempts to use reasonable defaults for everything else + :param string description: Notification message + + .. warning:: + For now, only URL callbacks are supported. In the future, the + callback argument will also support a function + """ + growl = notifierFactory( + applicationName = applicationName, + notifications = [noteType], + defaultNotifications = [noteType], + applicationIcon = applicationIcon, + hostname = hostname, + password = password, + port = port, + ) + result = growl.register() + if result is not True: + return result + + return growl.notify( + noteType = noteType, + title = title, + description = description, + icon = notificationIcon, + sticky = sticky, + priority = priority, + callback = callback, + identifier = identifier, + ) + if __name__ == '__main__': # If we're running this module directly we're likely running it as a test # so extra debugging is useful diff --git a/libs/tornado/curl_httpclient.py b/libs/tornado/curl_httpclient.py index a6c0bb0..52350d2 100755 --- a/libs/tornado/curl_httpclient.py +++ b/libs/tornado/curl_httpclient.py @@ -96,17 +96,18 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): pycurl.POLL_INOUT: ioloop.IOLoop.READ | ioloop.IOLoop.WRITE } if event == pycurl.POLL_REMOVE: - self.io_loop.remove_handler(fd) - del self._fds[fd] + if fd in self._fds: + self.io_loop.remove_handler(fd) + del self._fds[fd] else: ioloop_event = event_map[event] if fd not in self._fds: - self._fds[fd] = ioloop_event self.io_loop.add_handler(fd, self._handle_events, ioloop_event) - else: self._fds[fd] = ioloop_event + else: self.io_loop.update_handler(fd, ioloop_event) + self._fds[fd] = ioloop_event def _set_timeout(self, msecs): """Called by libcurl to schedule a timeout.""" diff --git a/libs/tornado/ioloop.py b/libs/tornado/ioloop.py index 3c9b05b..7b320e5 100755 --- a/libs/tornado/ioloop.py +++ b/libs/tornado/ioloop.py @@ -194,7 +194,7 @@ class IOLoop(Configurable): def initialize(self): pass - def close(self, all_fds=False): + def close(self, all_fds = False): """Closes the IOLoop, freeing any resources used. If ``all_fds`` is true, all file descriptors registered on the @@ -320,7 +320,7 @@ class IOLoop(Configurable): """ raise NotImplementedError() - def add_callback(self, callback): + def add_callback(self, callback, *args, **kwargs): """Calls the given callback on the next I/O loop iteration. It is safe to call this method from any thread at any time, @@ -335,7 +335,7 @@ class IOLoop(Configurable): """ raise NotImplementedError() - def add_callback_from_signal(self, callback): + def add_callback_from_signal(self, callback, *args, **kwargs): """Calls the given callback on the next I/O loop iteration. Safe for use from a Python signal handler; should not be used @@ -359,8 +359,7 @@ class IOLoop(Configurable): assert isinstance(future, IOLoop._FUTURE_TYPES) callback = stack_context.wrap(callback) future.add_done_callback( - lambda future: self.add_callback( - functools.partial(callback, future))) + lambda future: self.add_callback(callback, future)) def _run_callback(self, callback): """Runs a callback with error handling. @@ -382,7 +381,7 @@ class IOLoop(Configurable): The exception itself is not passed explicitly, but is available in sys.exc_info. """ - app_log.error("Exception in callback %r", callback, exc_info=True) + app_log.error("Exception in callback %r", callback, exc_info = True) @@ -393,7 +392,7 @@ class PollIOLoop(IOLoop): (Linux), `tornado.platform.kqueue.KQueueIOLoop` (BSD and Mac), or `tornado.platform.select.SelectIOLoop` (all platforms). """ - def initialize(self, impl, time_func=None): + def initialize(self, impl, time_func = None): super(PollIOLoop, self).initialize() self._impl = impl if hasattr(self._impl, 'fileno'): @@ -417,7 +416,7 @@ class PollIOLoop(IOLoop): lambda fd, events: self._waker.consume(), self.READ) - def close(self, all_fds=False): + def close(self, all_fds = False): with self._callback_lock: self._closing = True self.remove_handler(self._waker.fileno()) @@ -426,7 +425,7 @@ class PollIOLoop(IOLoop): try: os.close(fd) except Exception: - gen_log.debug("error closing fd %s", fd, exc_info=True) + gen_log.debug("error closing fd %s", fd, exc_info = True) self._waker.close() self._impl.close() @@ -442,8 +441,8 @@ class PollIOLoop(IOLoop): self._events.pop(fd, None) try: self._impl.unregister(fd) - except (OSError, IOError): - gen_log.debug("Error deleting fd from IOLoop", exc_info=True) + except Exception: + gen_log.debug("Error deleting fd from IOLoop", exc_info = True) def set_blocking_signal_threshold(self, seconds, action): if not hasattr(signal, "setitimer"): @@ -501,7 +500,7 @@ class PollIOLoop(IOLoop): # IOLoop is just started once at the beginning. signal.set_wakeup_fd(old_wakeup_fd) old_wakeup_fd = None - except ValueError: # non-main thread + except ValueError: # non-main thread pass while True: @@ -569,17 +568,18 @@ class PollIOLoop(IOLoop): while self._events: fd, events = self._events.popitem() try: - self._handlers[fd](fd, events) + hdlr = self._handlers.get(fd) + if hdlr: hdlr(fd, events) except (OSError, IOError), e: if e.args[0] == errno.EPIPE: # Happens when the client closes the connection pass else: app_log.error("Exception in I/O handler for fd %s", - fd, exc_info=True) + fd, exc_info = True) except Exception: app_log.error("Exception in I/O handler for fd %s", - fd, exc_info=True) + fd, exc_info = True) # reset the stopped flag so another start/stop pair can be issued self._stopped = False if self._blocking_signal_threshold is not None: @@ -609,12 +609,13 @@ class PollIOLoop(IOLoop): # collection pass whenever there are too many dead timeouts. timeout.callback = None - def add_callback(self, callback): + def add_callback(self, callback, *args, **kwargs): with self._callback_lock: if self._closing: raise RuntimeError("IOLoop is closing") list_empty = not self._callbacks - self._callbacks.append(stack_context.wrap(callback)) + self._callbacks.append(functools.partial( + stack_context.wrap(callback), *args, **kwargs)) if list_empty and thread.get_ident() != self._thread_ident: # If we're in the IOLoop's thread, we know it's not currently # polling. If we're not, and we added the first callback to an @@ -624,12 +625,12 @@ class PollIOLoop(IOLoop): # avoid it when we can. self._waker.wake() - def add_callback_from_signal(self, callback): + def add_callback_from_signal(self, callback, *args, **kwargs): with stack_context.NullContext(): if thread.get_ident() != self._thread_ident: # if the signal is handled on another thread, we can add # it normally (modulo the NullContext) - self.add_callback(callback) + self.add_callback(callback, *args, **kwargs) else: # If we're on the IOLoop's thread, we cannot use # the regular add_callback because it may deadlock on @@ -639,7 +640,8 @@ class PollIOLoop(IOLoop): # _callback_lock block in IOLoop.start, we may modify # either the old or new version of self._callbacks, # but either way will work. - self._callbacks.append(stack_context.wrap(callback)) + self._callbacks.append(functools.partial( + stack_context.wrap(callback), *args, **kwargs)) class _Timeout(object): @@ -682,7 +684,7 @@ class PeriodicCallback(object): `start` must be called after the PeriodicCallback is created. """ - def __init__(self, callback, callback_time, io_loop=None): + def __init__(self, callback, callback_time, io_loop = None): self.callback = callback if callback_time <= 0: raise ValueError("Periodic callback must have a positive callback_time") @@ -710,7 +712,7 @@ class PeriodicCallback(object): try: self.callback() except Exception: - app_log.error("Error in periodic callback", exc_info=True) + app_log.error("Error in periodic callback", exc_info = True) self._schedule_next() def _schedule_next(self): diff --git a/libs/tornado/iostream.py b/libs/tornado/iostream.py index 40ac496..6eec2a3 100755 --- a/libs/tornado/iostream.py +++ b/libs/tornado/iostream.py @@ -209,11 +209,19 @@ class BaseIOStream(object): """Call the given callback when the stream is closed.""" self._close_callback = stack_context.wrap(callback) - def close(self): - """Close this stream.""" + def close(self, exc_info=False): + """Close this stream. + + If ``exc_info`` is true, set the ``error`` attribute to the current + exception from `sys.exc_info()` (or if ``exc_info`` is a tuple, + use that instead of `sys.exc_info`). + """ if not self.closed(): - if any(sys.exc_info()): - self.error = sys.exc_info()[1] + if exc_info: + if not isinstance(exc_info, tuple): + exc_info = sys.exc_info() + if any(exc_info): + self.error = exc_info[1] if self._read_until_close: callback = self._read_callback self._read_callback = None @@ -285,7 +293,7 @@ class BaseIOStream(object): except Exception: gen_log.error("Uncaught exception, closing connection.", exc_info=True) - self.close() + self.close(exc_info=True) raise def _run_callback(self, callback, *args): @@ -300,7 +308,7 @@ class BaseIOStream(object): # (It would eventually get closed when the socket object is # gc'd, but we don't want to rely on gc happening before we # run out of file descriptors) - self.close() + self.close(exc_info=True) # Re-raise the exception so that IOLoop.handle_callback_exception # can see it and log the error raise @@ -348,7 +356,7 @@ class BaseIOStream(object): self._pending_callbacks -= 1 except Exception: gen_log.warning("error on read", exc_info=True) - self.close() + self.close(exc_info=True) return if self._read_from_buffer(): return @@ -397,9 +405,9 @@ class BaseIOStream(object): # Treat ECONNRESET as a connection close rather than # an error to minimize log spam (the exception will # be available on self.error for apps that care). - self.close() + self.close(exc_info=True) return - self.close() + self.close(exc_info=True) raise if chunk is None: return 0 @@ -503,7 +511,7 @@ class BaseIOStream(object): else: gen_log.warning("Write error on %d: %s", self.fileno(), e) - self.close() + self.close(exc_info=True) return if not self._write_buffer and self._write_callback: callback = self._write_callback @@ -664,7 +672,7 @@ class IOStream(BaseIOStream): if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK): gen_log.warning("Connect error on fd %d: %s", self.socket.fileno(), e) - self.close() + self.close(exc_info=True) return self._connect_callback = stack_context.wrap(callback) self._add_io_state(self.io_loop.WRITE) @@ -733,7 +741,7 @@ class SSLIOStream(IOStream): return elif err.args[0] in (ssl.SSL_ERROR_EOF, ssl.SSL_ERROR_ZERO_RETURN): - return self.close() + return self.close(exc_info=True) elif err.args[0] == ssl.SSL_ERROR_SSL: try: peer = self.socket.getpeername() @@ -741,11 +749,11 @@ class SSLIOStream(IOStream): peer = '(not connected)' gen_log.warning("SSL Error on %d %s: %s", self.socket.fileno(), peer, err) - return self.close() + return self.close(exc_info=True) raise except socket.error, err: if err.args[0] in (errno.ECONNABORTED, errno.ECONNRESET): - return self.close() + return self.close(exc_info=True) else: self._ssl_accepting = False if self._ssl_connect_callback is not None: @@ -842,7 +850,7 @@ class PipeIOStream(BaseIOStream): elif e.args[0] == errno.EBADF: # If the writing half of a pipe is closed, select will # report it as readable but reads will fail with EBADF. - self.close() + self.close(exc_info=True) return None else: raise diff --git a/libs/tornado/platform/twisted.py b/libs/tornado/platform/twisted.py index 6c3cbf9..1efc82b 100755 --- a/libs/tornado/platform/twisted.py +++ b/libs/tornado/platform/twisted.py @@ -431,6 +431,8 @@ class TwistedIOLoop(tornado.ioloop.IOLoop): self.reactor.removeWriter(self.fds[fd]) def remove_handler(self, fd): + if fd not in self.fds: + return self.fds[fd].lost = True if self.fds[fd].reading: self.reactor.removeReader(self.fds[fd]) @@ -444,6 +446,12 @@ class TwistedIOLoop(tornado.ioloop.IOLoop): def stop(self): self.reactor.crash() + def _run_callback(self, callback, *args, **kwargs): + try: + callback(*args, **kwargs) + except Exception: + self.handle_callback_exception(callback) + def add_timeout(self, deadline, callback): if isinstance(deadline, (int, long, float)): delay = max(deadline - self.time(), 0) @@ -451,13 +459,14 @@ class TwistedIOLoop(tornado.ioloop.IOLoop): delay = deadline.total_seconds() else: raise TypeError("Unsupported deadline %r") - return self.reactor.callLater(delay, wrap(callback)) + return self.reactor.callLater(delay, self._run_callback, wrap(callback)) def remove_timeout(self, timeout): timeout.cancel() - def add_callback(self, callback): - self.reactor.callFromThread(wrap(callback)) + def add_callback(self, callback, *args, **kwargs): + self.reactor.callFromThread(self._run_callback, + wrap(callback), *args, **kwargs) - def add_callback_from_signal(self, callback): - self.add_callback(callback) + def add_callback_from_signal(self, callback, *args, **kwargs): + self.add_callback(callback, *args, **kwargs) diff --git a/libs/tornado/process.py b/libs/tornado/process.py index 9e048c1..fa0be55 100755 --- a/libs/tornado/process.py +++ b/libs/tornado/process.py @@ -268,7 +268,7 @@ class Subprocess(object): assert ret_pid == pid subproc = cls._waiting.pop(pid) subproc.io_loop.add_callback_from_signal( - functools.partial(subproc._set_returncode, status)) + subproc._set_returncode, status) def _set_returncode(self, status): if os.WIFSIGNALED(status): diff --git a/libs/tornado/simple_httpclient.py b/libs/tornado/simple_httpclient.py index faff83c..7000d98 100755 --- a/libs/tornado/simple_httpclient.py +++ b/libs/tornado/simple_httpclient.py @@ -12,7 +12,6 @@ from tornado.util import b, GzipDecompressor import base64 import collections -import contextlib import copy import functools import os.path @@ -134,7 +133,7 @@ class _HTTPConnection(object): self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None - with stack_context.StackContext(self.cleanup): + with stack_context.ExceptionStackContext(self._handle_exception): self.parsed = urlparse.urlsplit(_unicode(self.request.url)) if ssl is None and self.parsed.scheme == "https": raise ValueError("HTTPS requires either python2.6+ or " @@ -309,19 +308,24 @@ class _HTTPConnection(object): if self.final_callback is not None: final_callback = self.final_callback self.final_callback = None - final_callback(response) - - @contextlib.contextmanager - def cleanup(self): - try: - yield - except Exception, e: - gen_log.warning("uncaught exception", exc_info=True) - self._run_callback(HTTPResponse(self.request, 599, error=e, + self.io_loop.add_callback(final_callback, response) + + def _handle_exception(self, typ, value, tb): + if self.final_callback: + gen_log.warning("uncaught exception", exc_info=(typ, value, tb)) + self._run_callback(HTTPResponse(self.request, 599, error=value, request_time=self.io_loop.time() - self.start_time, )) + if hasattr(self, "stream"): self.stream.close() + return True + else: + # If our callback has already been called, we are probably + # catching an exception that is not caused by us but rather + # some child of our callback. Rather than drop it on the floor, + # pass it along. + return False def _on_close(self): if self.final_callback is not None: diff --git a/libs/tornado/testing.py b/libs/tornado/testing.py index 5945643..2237662 100755 --- a/libs/tornado/testing.py +++ b/libs/tornado/testing.py @@ -36,9 +36,8 @@ except ImportError: netutil = None SimpleAsyncHTTPClient = None from tornado.log import gen_log -from tornado.stack_context import StackContext +from tornado.stack_context import ExceptionStackContext from tornado.util import raise_exc_info -import contextlib import logging import os import re @@ -167,13 +166,10 @@ class AsyncTestCase(unittest.TestCase): ''' return IOLoop() - @contextlib.contextmanager - def _stack_context(self): - try: - yield - except Exception: - self.__failure = sys.exc_info() - self.stop() + def _handle_exception(self, typ, value, tb): + self.__failure = sys.exc_info() + self.stop() + return True def __rethrow(self): if self.__failure is not None: @@ -182,7 +178,7 @@ class AsyncTestCase(unittest.TestCase): raise_exc_info(failure) def run(self, result=None): - with StackContext(self._stack_context): + with ExceptionStackContext(self._handle_exception): super(AsyncTestCase, self).run(result) # In case an exception escaped super.run or the StackContext caught # an exception when there wasn't a wait() to re-raise it, do so here. diff --git a/libs/tornado/web.py b/libs/tornado/web.py index 7d45ce5..41ce95d 100755 --- a/libs/tornado/web.py +++ b/libs/tornado/web.py @@ -1317,10 +1317,8 @@ class Application(object): def add_handlers(self, host_pattern, host_handlers): """Appends the given handlers to our handler list. - Note that host patterns are processed sequentially in the - order they were added, and only the first matching pattern is - used. This means that all handlers for a given host must be - added in a single add_handlers call. + Host patterns are processed sequentially in the order they were + added. All matching patterns will be considered. """ if not host_pattern.endswith("$"): host_pattern += "$" @@ -1365,15 +1363,16 @@ class Application(object): def _get_host_handlers(self, request): host = request.host.lower().split(':')[0] + matches = [] for pattern, handlers in self.handlers: if pattern.match(host): - return handlers + matches.extend(handlers) # Look for default host if not behind load balancer (for debugging) - if "X-Real-Ip" not in request.headers: + if not matches and "X-Real-Ip" not in request.headers: for pattern, handlers in self.handlers: if pattern.match(self.default_host): - return handlers - return None + matches.extend(handlers) + return matches or None def _load_ui_methods(self, methods): if type(methods) is types.ModuleType: