Browse Source

Merge branch 'refs/heads/develop' into desktop

tags/build/2.0.5
Ruud 13 years ago
parent
commit
fa7cac7538
  1. 7
      couchpotato/api.py
  2. 5
      couchpotato/core/downloaders/base.py
  3. 3
      couchpotato/core/downloaders/nzbget/__init__.py
  4. 46
      couchpotato/core/downloaders/nzbvortex/__init__.py
  5. 176
      couchpotato/core/downloaders/nzbvortex/main.py
  6. 2
      couchpotato/core/downloaders/pneumatic/__init__.py
  7. 2
      couchpotato/core/downloaders/sabnzbd/__init__.py
  8. 6
      couchpotato/core/downloaders/sabnzbd/main.py
  9. 2
      couchpotato/core/downloaders/synology/__init__.py
  10. 2
      couchpotato/core/downloaders/transmission/__init__.py
  11. 2
      couchpotato/core/downloaders/utorrent/__init__.py
  12. 5
      couchpotato/core/notifications/growl/main.py
  13. 32
      couchpotato/core/notifications/toasty/__init__.py
  14. 30
      couchpotato/core/notifications/toasty/main.py
  15. 1
      couchpotato/core/notifications/xbmc/__init__.py
  16. 125
      couchpotato/core/notifications/xbmc/main.py
  17. 2
      couchpotato/core/plugins/automation/main.py
  18. 4
      couchpotato/core/plugins/base.py
  19. 1
      couchpotato/core/plugins/renamer/main.py
  20. 4
      couchpotato/core/plugins/scanner/main.py
  21. 7
      couchpotato/core/providers/automation/base.py
  22. 19
      couchpotato/core/providers/automation/bluray/main.py
  23. 3
      couchpotato/core/providers/automation/cp/main.py
  24. 8
      couchpotato/core/providers/automation/imdb/main.py
  25. 35
      couchpotato/core/providers/automation/itunes/__init__.py
  26. 63
      couchpotato/core/providers/automation/itunes/main.py
  27. 16
      couchpotato/core/providers/automation/kinepolis/main.py
  28. 23
      couchpotato/core/providers/automation/moviemeter/__init__.py
  29. 28
      couchpotato/core/providers/automation/moviemeter/main.py
  30. 25
      couchpotato/core/providers/automation/movies_io/main.py
  31. 25
      couchpotato/core/providers/automation/trakt/main.py
  32. 69
      couchpotato/core/providers/base.py
  33. 6
      couchpotato/core/providers/movie/imdbapi/__init__.py
  34. 6
      couchpotato/core/providers/movie/omdbapi/__init__.py
  35. 10
      couchpotato/core/providers/movie/omdbapi/main.py
  36. 36
      couchpotato/core/providers/nzb/ftdworld/main.py
  37. 8
      couchpotato/core/providers/nzb/newznab/__init__.py
  38. 4
      couchpotato/core/providers/nzb/newznab/main.py
  39. 2
      couchpotato/core/providers/nzb/nzbx/__init__.py
  40. 3
      couchpotato/core/providers/nzb/nzbx/main.py
  41. 2
      couchpotato/core/providers/nzb/omgwtfnzbs/main.py
  42. 2
      couchpotato/runner.py
  43. 2
      couchpotato/static/scripts/api.js
  44. 104
      init/freebsd
  45. 96
      libs/gntp/__init__.py
  46. 107
      libs/gntp/notifier.py
  47. 5
      libs/tornado/curl_httpclient.py
  48. 44
      libs/tornado/ioloop.py
  49. 38
      libs/tornado/iostream.py
  50. 19
      libs/tornado/platform/twisted.py
  51. 2
      libs/tornado/process.py
  52. 24
      libs/tornado/simple_httpclient.py
  53. 12
      libs/tornado/testing.py
  54. 15
      libs/tornado/web.py

7
couchpotato/api.py

@ -1,6 +1,5 @@
from flask.blueprints import Blueprint from flask.blueprints import Blueprint
from flask.helpers import url_for from flask.helpers import url_for
from tornado.ioloop import IOLoop
from tornado.web import RequestHandler, asynchronous from tornado.web import RequestHandler, asynchronous
from werkzeug.utils import redirect from werkzeug.utils import redirect
@ -11,7 +10,11 @@ api_nonblock = {}
class NonBlockHandler(RequestHandler): class NonBlockHandler(RequestHandler):
stoppers = []
def __init__(self, application, request, **kwargs):
cls = NonBlockHandler
cls.stoppers = []
super(NonBlockHandler, self).__init__(application, request, **kwargs)
@asynchronous @asynchronous
def get(self, route): def get(self, route):

5
couchpotato/core/downloaders/base.py

@ -1,16 +1,17 @@
from base64 import b32decode, b16encode from base64 import b32decode, b16encode
from couchpotato.core.event import addEvent from couchpotato.core.event import addEvent
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin from couchpotato.core.providers.base import Provider
import random import random
import re import re
log = CPLog(__name__) log = CPLog(__name__)
class Downloader(Plugin): class Downloader(Provider):
type = [] type = []
http_time_between_calls = 0
torrent_sources = [ torrent_sources = [
'http://torrage.com/torrent/%s.torrent', 'http://torrage.com/torrent/%s.torrent',

3
couchpotato/core/downloaders/nzbget/__init__.py

@ -10,7 +10,7 @@ config = [{
'tab': 'downloaders', 'tab': 'downloaders',
'name': 'nzbget', 'name': 'nzbget',
'label': 'NZBGet', 'label': 'NZBGet',
'description': 'Send NZBs to your NZBGet installation.', 'description': 'Use <a href="http://nzbget.sourceforge.net/Main_Page" target="_blank">NZBGet</a> to download NZBs.',
'options': [ 'options': [
{ {
'name': 'enabled', 'name': 'enabled',
@ -25,6 +25,7 @@ config = [{
}, },
{ {
'name': 'password', 'name': 'password',
'type': 'password',
'description': 'Default NZBGet password is <i>tegbzn6789</i>', 'description': 'Default NZBGet password is <i>tegbzn6789</i>',
}, },
{ {

46
couchpotato/core/downloaders/nzbvortex/__init__.py

@ -0,0 +1,46 @@
from .main import NZBVortex
def start():
return NZBVortex()
config = [{
'name': 'nzbvortex',
'groups': [
{
'tab': 'downloaders',
'name': 'nzbvortex',
'label': 'NZBVortex',
'description': 'Use <a href="http://www.nzbvortex.com/landing/" target="_blank">NZBVortex</a> to download NZBs.',
'wizard': True,
'options': [
{
'name': 'enabled',
'default': 0,
'type': 'enabler',
'radio_group': 'nzb',
},
{
'name': 'host',
'default': 'https://localhost:4321',
},
{
'name': 'api_key',
'label': 'Api Key',
},
{
'name': 'manual',
'default': False,
'type': 'bool',
'advanced': True,
'description': 'Disable this downloader for automated searches, but use it when I manually send a release.',
},
{
'name': 'delete_failed',
'default': True,
'type': 'bool',
'description': 'Delete a release after the download has failed.',
},
],
}
],
}]

176
couchpotato/core/downloaders/nzbvortex/main.py

@ -0,0 +1,176 @@
from base64 import b64encode
from couchpotato.core.downloaders.base import Downloader
from couchpotato.core.helpers.encoding import tryUrlencode, ss
from couchpotato.core.helpers.variable import cleanHost
from couchpotato.core.logger import CPLog
from urllib2 import URLError
from uuid import uuid4
import hashlib
import httplib
import json
import socket
import ssl
import sys
import traceback
import urllib2
log = CPLog(__name__)
class NZBVortex(Downloader):
type = ['nzb']
api_level = None
session_id = None
def download(self, data = {}, movie = {}, manual = False, filedata = None):
if self.isDisabled(manual) or not self.isCorrectType(data.get('type')) or not self.getApiLevel():
return
# Send the nzb
try:
nzb_filename = self.createFileName(data, filedata, movie)
self.call('nzb/add', params = {'file': (ss(nzb_filename), filedata)}, multipart = True)
return True
except:
log.error('Something went wrong sending the NZB file: %s', traceback.format_exc())
return False
def getAllDownloadStatus(self):
if self.isDisabled(manual = True):
return False
raw_statuses = self.call('nzb')
statuses = []
for item in raw_statuses.get('nzbs', []):
# Check status
status = 'busy'
if item['state'] == 20:
status = 'completed'
elif item['state'] in [21, 22, 24]:
status = 'failed'
statuses.append({
'id': item['id'],
'name': item['uiTitle'],
'status': status,
'original_status': item['state'],
'timeleft':-1,
})
return statuses
def removeFailed(self, item):
if not self.conf('delete_failed', default = True):
return False
log.info('%s failed downloading, deleting...', item['name'])
try:
self.call('nzb/%s/cancel' % item['id'])
except:
log.error('Failed deleting: %s', traceback.format_exc(0))
return False
return True
def login(self):
nonce = self.call('auth/nonce', auth = False).get('authNonce')
cnonce = uuid4().hex
hashed = b64encode(hashlib.sha256('%s:%s:%s' % (nonce, cnonce, self.conf('api_key'))).digest())
params = {
'nonce': nonce,
'cnonce': cnonce,
'hash': hashed
}
login_data = self.call('auth/login', parameters = params, auth = False)
# Save for later
if login_data.get('loginResult') == 'successful':
self.session_id = login_data.get('sessionID')
return True
log.error('Login failed, please check you api-key')
return False
def call(self, call, parameters = {}, repeat = False, auth = True, *args, **kwargs):
# Login first
if not self.session_id and auth:
self.login()
# Always add session id to request
if self.session_id:
parameters['sessionid'] = self.session_id
params = tryUrlencode(parameters)
url = cleanHost(self.conf('host')) + 'api/' + call
url_opener = urllib2.build_opener(HTTPSHandler())
try:
data = self.urlopen('%s?%s' % (url, params), opener = url_opener, *args, **kwargs)
if data:
return json.loads(data)
except URLError, e:
if hasattr(e, 'code') and e.code == 403:
# Try login and do again
if not repeat:
self.login()
return self.call(call, parameters = parameters, repeat = True, *args, **kwargs)
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
except:
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
return {}
def getApiLevel(self):
if not self.api_level:
url = cleanHost(self.conf('host')) + 'api/app/apilevel'
url_opener = urllib2.build_opener(HTTPSHandler())
try:
data = self.urlopen(url, opener = url_opener, show_error = False)
self.api_level = float(json.loads(data).get('apilevel'))
except URLError, e:
if hasattr(e, 'code') and e.code == 403:
log.error('This version of NZBVortex isn\'t supported. Please update to 2.8.6 or higher')
else:
log.error('NZBVortex doesn\'t seem to be running or maybe the remote option isn\'t enabled yet: %s', traceback.format_exc(1))
return self.api_level
class HTTPSConnection(httplib.HTTPSConnection):
def __init__(self, *args, **kwargs):
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
def connect(self):
sock = socket.create_connection((self.host, self.port), self.timeout)
if sys.version_info < (2, 6, 7):
if hasattr(self, '_tunnel_host'):
self.sock = sock
self._tunnel()
else:
if self._tunnel_host:
self.sock = sock
self._tunnel()
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version = ssl.PROTOCOL_TLSv1)
class HTTPSHandler(urllib2.HTTPSHandler):
def https_open(self, req):
return self.do_open(HTTPSConnection, req)

2
couchpotato/core/downloaders/pneumatic/__init__.py

@ -11,7 +11,7 @@ config = [{
'tab': 'downloaders', 'tab': 'downloaders',
'name': 'pneumatic', 'name': 'pneumatic',
'label': 'Pneumatic', 'label': 'Pneumatic',
'description': 'Download the .strm file to a specific folder.', 'description': 'Use <a href="http://forum.xbmc.org/showthread.php?tid=97657" target="_blank">Pneumatic</a> to download .strm files.',
'options': [ 'options': [
{ {
'name': 'enabled', 'name': 'enabled',

2
couchpotato/core/downloaders/sabnzbd/__init__.py

@ -10,7 +10,7 @@ config = [{
'tab': 'downloaders', 'tab': 'downloaders',
'name': 'sabnzbd', 'name': 'sabnzbd',
'label': 'Sabnzbd', 'label': 'Sabnzbd',
'description': 'Send NZBs to your Sabnzbd installation.', 'description': 'Use <a href="http://sabnzbd.org/" target="_blank">SABnzbd</a> to download NZBs.',
'wizard': True, 'wizard': True,
'options': [ 'options': [
{ {

6
couchpotato/core/downloaders/sabnzbd/main.py

@ -1,5 +1,5 @@
from couchpotato.core.downloaders.base import Downloader from couchpotato.core.downloaders.base import Downloader
from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.encoding import tryUrlencode, ss
from couchpotato.core.helpers.variable import cleanHost, mergeDicts from couchpotato.core.helpers.variable import cleanHost, mergeDicts
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from urllib2 import URLError from urllib2 import URLError
@ -41,7 +41,7 @@ class Sabnzbd(Downloader):
try: try:
if params.get('mode') is 'addfile': if params.get('mode') is 'addfile':
sab = self.urlopen(url, timeout = 60, params = {'nzbfile': (nzb_filename, filedata)}, multipart = True, show_error = False) sab = self.urlopen(url, timeout = 60, params = {'nzbfile': (ss(nzb_filename), filedata)}, multipart = True, show_error = False)
else: else:
sab = self.urlopen(url, timeout = 60, show_error = False) sab = self.urlopen(url, timeout = 60, show_error = False)
except URLError: except URLError:
@ -65,7 +65,7 @@ class Sabnzbd(Downloader):
return False return False
def getAllDownloadStatus(self): def getAllDownloadStatus(self):
if self.isDisabled(manual = False): if self.isDisabled(manual = True):
return False return False
log.debug('Checking SABnzbd download status.') log.debug('Checking SABnzbd download status.')

2
couchpotato/core/downloaders/synology/__init__.py

@ -10,7 +10,7 @@ config = [{
'tab': 'downloaders', 'tab': 'downloaders',
'name': 'synology', 'name': 'synology',
'label': 'Synology', 'label': 'Synology',
'description': 'Send torrents to Synology\'s Download Station.', 'description': 'Use <a href="http://www.synology.com/dsm/home_home_applications_download_station.php" target="_blank">Synology Download Station</a> to download.',
'wizard': True, 'wizard': True,
'options': [ 'options': [
{ {

2
couchpotato/core/downloaders/transmission/__init__.py

@ -10,7 +10,7 @@ config = [{
'tab': 'downloaders', 'tab': 'downloaders',
'name': 'transmission', 'name': 'transmission',
'label': 'Transmission', 'label': 'Transmission',
'description': 'Send torrents to Transmission.', 'description': 'Use <a href="http://www.transmissionbt.com/" target="_blank">Transmission</a> to download torrents.',
'wizard': True, 'wizard': True,
'options': [ 'options': [
{ {

2
couchpotato/core/downloaders/utorrent/__init__.py

@ -10,7 +10,7 @@ config = [{
'tab': 'downloaders', 'tab': 'downloaders',
'name': 'utorrent', 'name': 'utorrent',
'label': 'uTorrent', 'label': 'uTorrent',
'description': 'Send torrents to uTorrent.', 'description': 'Use <a href="http://www.utorrent.com/" target="_blank">uTorrent</a> to download torrents.',
'wizard': True, 'wizard': True,
'options': [ 'options': [
{ {

5
couchpotato/core/notifications/growl/main.py

@ -37,7 +37,10 @@ class Growl(Notification):
) )
self.growl.register() self.growl.register()
self.registered = True self.registered = True
except: except Exception, e:
if 'timed out' in str(e):
self.registered = True
else:
log.error('Failed register of growl: %s', traceback.format_exc()) log.error('Failed register of growl: %s', traceback.format_exc())
def notify(self, message = '', data = {}, listener = None): def notify(self, message = '', data = {}, listener = None):

32
couchpotato/core/notifications/toasty/__init__.py

@ -0,0 +1,32 @@
from .main import Toasty
def start():
return Toasty()
config = [{
'name': 'toasty',
'groups': [
{
'tab': 'notifications',
'name': 'toasty',
'options': [
{
'name': 'enabled',
'default': 0,
'type': 'enabler',
},
{
'name': 'api_key',
'label': 'Device ID',
},
{
'name': 'on_snatch',
'default': 0,
'type': 'bool',
'advanced': True,
'description': 'Also send message when movie is snatched.',
},
],
}
],
}]

30
couchpotato/core/notifications/toasty/main.py

@ -0,0 +1,30 @@
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
from couchpotato.core.logger import CPLog
from couchpotato.core.notifications.base import Notification
import traceback
log = CPLog(__name__)
class Toasty(Notification):
urls = {
'api': 'http://api.supertoasty.com/notify/%s?%s'
}
def notify(self, message = '', data = {}, listener = None):
if self.isDisabled(): return
data = {
'title': self.default_title,
'text': toUnicode(message),
'sender': toUnicode("CouchPotato"),
'image': 'https://raw.github.com/RuudBurger/CouchPotatoServer/master/couchpotato/static/images/homescreen.png',
}
try:
self.urlopen(self.urls['api'] % (self.conf('api_key'), tryUrlencode(data)), show_error = False)
return True
except:
log.error('Toasty failed: %s', traceback.format_exc())
return False

1
couchpotato/core/notifications/xbmc/__init__.py

@ -10,6 +10,7 @@ config = [{
'tab': 'notifications', 'tab': 'notifications',
'name': 'xbmc', 'name': 'xbmc',
'label': 'XBMC', 'label': 'XBMC',
'description': 'v11 (Eden) and v12 (Frodo)',
'options': [ 'options': [
{ {
'name': 'enabled', 'name': 'enabled',

125
couchpotato/core/notifications/xbmc/main.py

@ -4,6 +4,7 @@ from couchpotato.core.notifications.base import Notification
from flask.helpers import json from flask.helpers import json
import base64 import base64
import traceback import traceback
import urllib
log = CPLog(__name__) log = CPLog(__name__)
@ -11,27 +12,147 @@ log = CPLog(__name__)
class XBMC(Notification): class XBMC(Notification):
listen_to = ['renamer.after'] listen_to = ['renamer.after']
use_json_notifications = {}
def notify(self, message = '', data = {}, listener = None): def notify(self, message = '', data = {}, listener = None):
if self.isDisabled(): return if self.isDisabled(): return
hosts = splitString(self.conf('host')) hosts = splitString(self.conf('host'))
successful = 0 successful = 0
for host in hosts: for host in hosts:
if self.use_json_notifications.get(host) is None:
self.getXBMCJSONversion(host, message = message)
if self.use_json_notifications.get(host):
response = self.request(host, [ response = self.request(host, [
('GUI.ShowNotification', {"title":"CouchPotato", "message":message}), ('GUI.ShowNotification', {'title':self.default_title, 'message':message}),
('VideoLibrary.Scan', {}), ('VideoLibrary.Scan', {}),
]) ])
else:
response = self.notifyXBMCnoJSON(host, {'title':self.default_title, 'message':message})
response += self.request(host, [('VideoLibrary.Scan', {})])
try: try:
for result in response: for result in response:
if result['result'] == "OK": if (result.get('result') and result['result'] == 'OK'):
successful += 1 successful += 1
elif (result.get('error')):
log.error('XBMC error; %s: %s (%s)', (result['id'], result['error']['message'], result['error']['code']))
except: except:
log.error('Failed parsing results: %s', traceback.format_exc()) log.error('Failed parsing results: %s', traceback.format_exc())
return successful == len(hosts) * 2 return successful == len(hosts) * 2
def getXBMCJSONversion(self, host, message = ''):
success = False
# XBMC JSON-RPC version request
response = self.request(host, [
('JSONRPC.Version', {})
])
for result in response:
if (result.get('result') and type(result['result']['version']).__name__ == 'int'):
# only v2 and v4 return an int object
# v6 (as of XBMC v12(Frodo)) is required to send notifications
xbmc_rpc_version = str(result['result']['version'])
log.debug('XBMC JSON-RPC Version: %s ; Notifications by JSON-RPC only supported for v6 [as of XBMC v12(Frodo)]', xbmc_rpc_version)
# disable JSON use
self.use_json_notifications[host] = False
# send the text message
resp = self.notifyXBMCnoJSON(host, {'title':self.default_title, 'message':message})
for result in resp:
if (result.get('result') and result['result'] == 'OK'):
log.debug('Message delivered successfully!')
success = True
break
elif (result.get('error')):
log.error('XBMC error; %s: %s (%s)', (result['id'], result['error']['message'], result['error']['code']))
break
elif (result.get('result') and type(result['result']['version']).__name__ == 'dict'):
# XBMC JSON-RPC v6 returns an array object containing
# major, minor and patch number
xbmc_rpc_version = str(result['result']['version']['major'])
xbmc_rpc_version += '.' + str(result['result']['version']['minor'])
xbmc_rpc_version += '.' + str(result['result']['version']['patch'])
log.debug('XBMC JSON-RPC Version: %s', xbmc_rpc_version)
# ok, XBMC version is supported
self.use_json_notifications[host] = True
# send the text message
resp = self.request(host, [('GUI.ShowNotification', {'title':self.default_title, 'message':message})])
for result in resp:
if (result.get('result') and result['result'] == 'OK'):
log.debug('Message delivered successfully!')
success = True
break
elif (result.get('error')):
log.error('XBMC error; %s: %s (%s)', (result['id'], result['error']['message'], result['error']['code']))
break
# error getting version info (we do have contact with XBMC though)
elif (result.get('error')):
log.error('XBMC error; %s: %s (%s)', (result['id'], result['error']['message'], result['error']['code']))
log.debug('Use JSON notifications: %s ', self.use_json_notifications)
return success
def notifyXBMCnoJSON(self, host, data):
server = 'http://%s/xbmcCmds/' % host
# title, message [, timeout , image #can be added!]
cmd = "xbmcHttp?command=ExecBuiltIn(Notification('%s','%s'))" % (urllib.quote(data['title']), urllib.quote(data['message']))
server += cmd
# I have no idea what to set to, just tried text/plain and seems to be working :)
headers = {
'Content-Type': 'text/plain',
}
# authentication support
if self.conf('password'):
base64string = base64.encodestring('%s:%s' % (self.conf('username'), self.conf('password'))).replace('\n', '')
headers['Authorization'] = 'Basic %s' % base64string
try:
log.debug('Sending non-JSON-type request to %s: %s', (host, data))
# response wil either be 'OK':
# <html>
# <li>OK
# </html>
#
# or 'Error':
# <html>
# <li>Error:<message>
# </html>
#
response = self.urlopen(server, headers = headers)
if 'OK' in response:
log.debug('Returned from non-JSON-type request %s: %s', (host, response))
# manually fake expected response array
return [{'result': 'OK'}]
else:
log.error('Returned from non-JSON-type request %s: %s', (host, response))
# manually fake expected response array
return [{'result': 'Error'}]
except:
log.error('Failed sending non-JSON-type request to XBMC: %s', traceback.format_exc())
return [{'result': 'Error'}]
def request(self, host, requests): def request(self, host, requests):
server = 'http://%s/jsonrpc' % host server = 'http://%s/jsonrpc' % host

2
couchpotato/core/plugins/automation/main.py

@ -12,7 +12,7 @@ class Automation(Plugin):
fireEvent('schedule.interval', 'automation.add_movies', self.addMovies, hours = self.conf('hour', default = 12)) fireEvent('schedule.interval', 'automation.add_movies', self.addMovies, hours = self.conf('hour', default = 12))
if not Env.get('dev'): if Env.get('dev'):
addEvent('app.load', self.addMovies) addEvent('app.load', self.addMovies)
def addMovies(self): def addMovies(self):

4
couchpotato/core/plugins/base.py

@ -98,6 +98,7 @@ class Plugin(object):
# http request # http request
def urlopen(self, url, timeout = 30, params = None, headers = None, opener = None, multipart = False, show_error = True): def urlopen(self, url, timeout = 30, params = None, headers = None, opener = None, multipart = False, show_error = True):
url = ss(url)
if not headers: headers = {} if not headers: headers = {}
if not params: params = {} if not params: params = {}
@ -129,6 +130,9 @@ class Plugin(object):
log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data')) log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data'))
request = urllib2.Request(url, params, headers) request = urllib2.Request(url, params, headers)
if opener:
opener.add_handler(MultipartPostHandler())
else:
cookies = cookielib.CookieJar() cookies = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies), MultipartPostHandler) opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies), MultipartPostHandler)

1
couchpotato/core/plugins/renamer/main.py

@ -33,6 +33,7 @@ class Renamer(Plugin):
addEvent('renamer.check_snatched', self.checkSnatched) addEvent('renamer.check_snatched', self.checkSnatched)
addEvent('app.load', self.scan) addEvent('app.load', self.scan)
addEvent('app.load', self.checkSnatched)
if self.conf('run_every') > 0: if self.conf('run_every') > 0:
fireEvent('schedule.interval', 'renamer.check_snatched', self.checkSnatched, minutes = self.conf('run_every')) fireEvent('schedule.interval', 'renamer.check_snatched', self.checkSnatched, minutes = self.conf('run_every'))

4
couchpotato/core/plugins/scanner/main.py

@ -89,7 +89,7 @@ class Scanner(Plugin):
'()([ab])(\.....?)$' #*a.mkv '()([ab])(\.....?)$' #*a.mkv
] ]
cp_imdb = '(\.cp\((?P<id>tt[0-9{7}]+)\))' cp_imdb = '(.cp.(?P<id>tt[0-9{7}]+).)'
def __init__(self): def __init__(self):
@ -341,7 +341,7 @@ class Scanner(Plugin):
group['files']['movie'] = self.getMediaFiles(group['unsorted_files']) group['files']['movie'] = self.getMediaFiles(group['unsorted_files'])
if len(group['files']['movie']) == 0: if len(group['files']['movie']) == 0:
log.error('Couldn\t find any movie files for %s', identifier) log.error('Couldn\'t find any movie files for %s', identifier)
continue continue
log.debug('Getting metadata for %s', identifier) log.debug('Getting metadata for %s', identifier)

7
couchpotato/core/providers/automation/base.py

@ -1,13 +1,13 @@
from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin from couchpotato.core.providers.base import Provider
from couchpotato.environment import Env from couchpotato.environment import Env
import time import time
log = CPLog(__name__) log = CPLog(__name__)
class Automation(Plugin): class Automation(Provider):
enabled_option = 'automation_enabled' enabled_option = 'automation_enabled'
@ -19,6 +19,9 @@ class Automation(Plugin):
def _getMovies(self): def _getMovies(self):
if self.isDisabled():
return
if not self.canCheck(): if not self.canCheck():
log.debug('Just checked, skipping %s', self.getName()) log.debug('Just checked, skipping %s', self.getName())
return [] return []

19
couchpotato/core/providers/automation/bluray/main.py

@ -1,8 +1,7 @@
from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import md5, tryInt from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.providers.automation.base import Automation from couchpotato.core.providers.automation.base import Automation
import xml.etree.ElementTree as XMLTree
log = CPLog(__name__) log = CPLog(__name__)
@ -14,23 +13,15 @@ class Bluray(Automation, RSS):
def getIMDBids(self): def getIMDBids(self):
if self.isDisabled():
return
movies = [] movies = []
cache_key = 'bluray.%s' % md5(self.rss_url) rss_movies = self.getRSSData(self.rss_url)
rss_data = self.getCache(cache_key, self.rss_url)
data = XMLTree.fromstring(rss_data)
if data is not None:
rss_movies = self.getElements(data, 'channel/item')
for movie in rss_movies: for movie in rss_movies:
name = self.getTextElement(movie, "title").lower().split("blu-ray")[0].strip("(").rstrip() name = self.getTextElement(movie, 'title').lower().split('blu-ray')[0].strip('(').rstrip()
year = self.getTextElement(movie, "description").split("|")[1].strip("(").strip() year = self.getTextElement(movie, 'description').split('|')[1].strip('(').strip()
if not name.find("/") == -1: # make sure it is not a double movie release if not name.find('/') == -1: # make sure it is not a double movie release
continue continue
if tryInt(year) < self.getMinimal('year'): if tryInt(year) < self.getMinimal('year'):

3
couchpotato/core/providers/automation/cp/main.py

@ -8,7 +8,4 @@ class CP(Automation):
def getMovies(self): def getMovies(self):
if self.isDisabled():
return
return [] return []

8
couchpotato/core/providers/automation/imdb/main.py

@ -1,5 +1,5 @@
from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import md5, getImdb, splitString, tryInt from couchpotato.core.helpers.variable import getImdb, splitString, tryInt
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.providers.automation.base import Automation from couchpotato.core.providers.automation.base import Automation
import traceback import traceback
@ -13,9 +13,6 @@ class IMDB(Automation, RSS):
def getIMDBids(self): def getIMDBids(self):
if self.isDisabled():
return
movies = [] movies = []
enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))] enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))]
@ -29,8 +26,7 @@ class IMDB(Automation, RSS):
continue continue
try: try:
cache_key = 'imdb.rss.%s' % md5(url) rss_data = self.getHTMLData(url)
rss_data = self.getCache(cache_key, url)
imdbs = getImdb(rss_data, multiple = True) imdbs = getImdb(rss_data, multiple = True)
for imdb in imdbs: for imdb in imdbs:

35
couchpotato/core/providers/automation/itunes/__init__.py

@ -0,0 +1,35 @@
from .main import ITunes
def start():
return ITunes()
config = [{
'name': 'itunes',
'groups': [
{
'tab': 'automation',
'name': 'itunes_automation',
'label': 'iTunes',
'description': 'From any <a href="http://itunes.apple.com/rss">iTunes</a> Store feed. Url should be the RSS link. (uses minimal requirements)',
'options': [
{
'name': 'automation_enabled',
'default': False,
'type': 'enabler',
},
{
'name': 'automation_urls_use',
'label': 'Use',
'default': ',',
},
{
'name': 'automation_urls',
'label': 'url',
'type': 'combined',
'combine': ['automation_urls_use', 'automation_urls'],
'default': 'https://itunes.apple.com/rss/topmovies/limit=25/xml,',
},
],
},
],
}]

63
couchpotato/core/providers/automation/itunes/main.py

@ -0,0 +1,63 @@
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import md5, splitString, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.automation.base import Automation
from xml.etree.ElementTree import QName
import datetime
import traceback
import xml.etree.ElementTree as XMLTree
log = CPLog(__name__)
class ITunes(Automation, RSS):
interval = 1800
def getIMDBids(self):
if self.isDisabled():
return
movies = []
enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))]
urls = splitString(self.conf('automation_urls'))
namespace = 'http://www.w3.org/2005/Atom'
namespaceIM = 'http://itunes.apple.com/rss'
index = -1
for url in urls:
index += 1
if not enablers[index]:
continue
try:
cache_key = 'itunes.rss.%s' % md5(url)
rss_data = self.getCache(cache_key, url)
data = XMLTree.fromstring(rss_data)
if data is not None:
entry_tag = str(QName(namespace, 'entry'))
rss_movies = self.getElements(data, entry_tag)
for movie in rss_movies:
name_tag = str(QName(namespaceIM, 'name'))
name = self.getTextElement(movie, name_tag)
releaseDate_tag = str(QName(namespaceIM, 'releaseDate'))
releaseDateText = self.getTextElement(movie, releaseDate_tag)
year = datetime.datetime.strptime(releaseDateText, '%Y-%m-%dT00:00:00-07:00').strftime("%Y")
imdb = self.search(name, year)
if imdb and self.isMinimalMovie(imdb):
movies.append(imdb['imdb'])
except:
log.error('Failed loading iTunes rss feed: %s %s', (url, traceback.format_exc()))
return movies

16
couchpotato/core/providers/automation/kinepolis/main.py

@ -1,9 +1,7 @@
from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import md5
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.providers.automation.base import Automation from couchpotato.core.providers.automation.base import Automation
import datetime import datetime
import xml.etree.ElementTree as XMLTree
log = CPLog(__name__) log = CPLog(__name__)
@ -15,21 +13,13 @@ class Kinepolis(Automation, RSS):
def getIMDBids(self): def getIMDBids(self):
if self.isDisabled():
return
movies = [] movies = []
cache_key = 'kinepolis.%s' % md5(self.rss_url) rss_movies = self.getRSSData(self.rss_url)
rss_data = self.getCache(cache_key, self.rss_url)
data = XMLTree.fromstring(rss_data)
if data is not None:
rss_movies = self.getElements(data, 'channel/item')
for movie in rss_movies: for movie in rss_movies:
name = self.getTextElement(movie, "title") name = self.getTextElement(movie, 'title')
year = datetime.datetime.now().strftime("%Y") year = datetime.datetime.now().strftime('%Y')
imdb = self.search(name, year) imdb = self.search(name, year)

23
couchpotato/core/providers/automation/moviemeter/__init__.py

@ -0,0 +1,23 @@
from .main import Moviemeter
def start():
return Moviemeter()
config = [{
'name': 'moviemeter',
'groups': [
{
'tab': 'automation',
'name': 'moviemeter_automation',
'label': 'Moviemeter',
'description': 'Imports movies from the current top 10 of moviemeter.nl. (uses minimal requirements)',
'options': [
{
'name': 'automation_enabled',
'default': False,
'type': 'enabler',
},
],
},
],
}]

28
couchpotato/core/providers/automation/moviemeter/main.py

@ -0,0 +1,28 @@
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.automation.base import Automation
log = CPLog(__name__)
class Moviemeter(Automation, RSS):
interval = 1800
rss_url = 'http://www.moviemeter.nl/rss/cinema'
def getIMDBids(self):
movies = []
rss_movies = self.getRSSData(self.rss_url)
for movie in rss_movies:
name_year = fireEvent('scanner.name_year', self.getTextElement(movie, 'title'), single = True)
imdb = self.search(name_year.get('name'), name_year.get('year'))
if imdb and self.isMinimalMovie(imdb):
movies.append(imdb['imdb'])
return movies

25
couchpotato/core/providers/automation/movies_io/main.py

@ -1,11 +1,8 @@
from couchpotato.core.event import fireEvent from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import md5 from couchpotato.core.helpers.variable import tryInt, splitString
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.providers.automation.base import Automation from couchpotato.core.providers.automation.base import Automation
from xml.etree.ElementTree import ParseError
import traceback
import xml.etree.ElementTree as XMLTree
log = CPLog(__name__) log = CPLog(__name__)
@ -16,39 +13,27 @@ class MoviesIO(Automation, RSS):
def getIMDBids(self): def getIMDBids(self):
if self.isDisabled():
return
movies = [] movies = []
enablers = self.conf('automation_urls_use').split(',') enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))]
index = -1 index = -1
for rss_url in self.conf('automation_urls').split(','): for rss_url in splitString(self.conf('automation_urls')):
index += 1 index += 1
if not enablers[index]: if not enablers[index]:
continue continue
try: rss_movies = self.getRSSData(rss_url, headers = {'Referer': ''})
cache_key = 'imdb.rss.%s' % md5(rss_url)
rss_data = self.getCache(cache_key, rss_url, headers = {'Referer': ''})
data = XMLTree.fromstring(rss_data)
rss_movies = self.getElements(data, 'channel/item')
for movie in rss_movies: for movie in rss_movies:
nameyear = fireEvent('scanner.name_year', self.getTextElement(movie, "title"), single = True) nameyear = fireEvent('scanner.name_year', self.getTextElement(movie, 'title'), single = True)
imdb = self.search(nameyear.get('name'), nameyear.get('year'), imdb_only = True) imdb = self.search(nameyear.get('name'), nameyear.get('year'), imdb_only = True)
if not imdb: if not imdb:
continue continue
movies.append(imdb) movies.append(imdb)
except ParseError:
log.debug('Failed loading Movies.io watchlist, probably empty: %s', (rss_url))
except:
log.error('Failed loading Movies.io watchlist: %s %s', (rss_url, traceback.format_exc()))
return movies return movies

25
couchpotato/core/providers/automation/trakt/main.py

@ -1,9 +1,8 @@
from couchpotato.core.event import addEvent from couchpotato.core.event import addEvent
from couchpotato.core.helpers.variable import md5, sha1 from couchpotato.core.helpers.variable import sha1
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.providers.automation.base import Automation from couchpotato.core.providers.automation.base import Automation
import base64 import base64
import json
log = CPLog(__name__) log = CPLog(__name__)
@ -25,9 +24,6 @@ class Trakt(Automation):
def getIMDBids(self): def getIMDBids(self):
if self.isDisabled():
return
movies = [] movies = []
for movie in self.getWatchlist(): for movie in self.getWatchlist():
movies.append(movie.get('imdb_id')) movies.append(movie.get('imdb_id'))
@ -38,22 +34,11 @@ class Trakt(Automation):
method = (self.urls['watchlist'] % self.conf('automation_api_key')) + self.conf('automation_username') method = (self.urls['watchlist'] % self.conf('automation_api_key')) + self.conf('automation_username')
return self.call(method) return self.call(method)
def call(self, method_url): def call(self, method_url):
try:
if self.conf('automation_password'):
headers = {
'Authorization': 'Basic %s' % base64.encodestring('%s:%s' % (self.conf('automation_username'), self.conf('automation_password')))[:-1]
}
else:
headers = {} headers = {}
if self.conf('automation_password'):
headers['Authorization'] = 'Basic %s' % base64.encodestring('%s:%s' % (self.conf('automation_username'), self.conf('automation_password')))[:-1]
cache_key = 'trakt.%s' % md5(method_url) data = self.getJsonData(self.urls['base'] + method_url, headers = headers)
json_string = self.getCache(cache_key, self.urls['base'] + method_url, headers = headers) return data if data else []
if json_string:
return json.loads(json_string)
except:
log.error('Failed to get data from trakt, check your login.')
return []

69
couchpotato/core/providers/base.py

@ -44,6 +44,34 @@ class Provider(Plugin):
return self.is_available.get(host, False) return self.is_available.get(host, False)
def getJsonData(self, url, **kwargs):
data = self.getCache(md5(url), url, **kwargs)
if data:
try:
return json.loads(data)
except:
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
return []
def getRSSData(self, url, **kwargs):
data = self.getCache(md5(url), url, **kwargs)
if data:
try:
data = XMLTree.fromstring(data)
return self.getElements(data, 'channel/item')
except:
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
return []
def getHTMLData(self, url, **kwargs):
return self.getCache(md5(url), url, **kwargs)
class YarrProvider(Provider): class YarrProvider(Provider):
@ -67,8 +95,10 @@ class YarrProvider(Provider):
urllib2.install_opener(opener) urllib2.install_opener(opener)
log.info2('Logging into %s', self.urls['login']) log.info2('Logging into %s', self.urls['login'])
f = opener.open(self.urls['login'], self.getLoginParams()) f = opener.open(self.urls['login'], self.getLoginParams())
f.read() output = f.read()
f.close() f.close()
if self.loginSuccess(output):
self.login_opener = opener self.login_opener = opener
return True return True
except: except:
@ -76,6 +106,9 @@ class YarrProvider(Provider):
return False return False
def loginSuccess(self, output):
return True
def loginDownload(self, url = '', nzb_id = ''): def loginDownload(self, url = '', nzb_id = ''):
try: try:
if not self.login_opener and not self.login(): if not self.login_opener and not self.login():
@ -106,11 +139,11 @@ class YarrProvider(Provider):
return [] return []
# Create result container # Create result container
imdb_result = hasattr(self, '_search') imdb_results = hasattr(self, '_search')
results = ResultList(self, movie, quality, imdb_result = imdb_result) results = ResultList(self, movie, quality, imdb_results = imdb_results)
# Do search based on imdb id # Do search based on imdb id
if imdb_result: if imdb_results:
self._search(movie, quality, results) self._search(movie, quality, results)
# Search possible titles # Search possible titles
else: else:
@ -165,34 +198,6 @@ class YarrProvider(Provider):
return [self.cat_backup_id] return [self.cat_backup_id]
def getJsonData(self, url, **kwargs):
data = self.getCache(md5(url), url, **kwargs)
if data:
try:
return json.loads(data)
except:
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
return []
def getRSSData(self, url, **kwargs):
data = self.getCache(md5(url), url, **kwargs)
if data:
try:
data = XMLTree.fromstring(data)
return self.getElements(data, 'channel/item')
except:
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
return []
def getHTMLData(self, url, **kwargs):
return self.getCache(md5(url), url, **kwargs)
class ResultList(list): class ResultList(list):

6
couchpotato/core/providers/movie/imdbapi/__init__.py

@ -1,6 +0,0 @@
from .main import IMDBAPI
def start():
return IMDBAPI()
config = []

6
couchpotato/core/providers/movie/omdbapi/__init__.py

@ -0,0 +1,6 @@
from .main import OMDBAPI
def start():
return OMDBAPI()
config = []

10
couchpotato/core/providers/movie/imdbapi/main.py → couchpotato/core/providers/movie/omdbapi/main.py

@ -10,11 +10,11 @@ import traceback
log = CPLog(__name__) log = CPLog(__name__)
class IMDBAPI(MovieProvider): class OMDBAPI(MovieProvider):
urls = { urls = {
'search': 'http://www.imdbapi.com/?%s', 'search': 'http://www.omdbapi.com/?%s',
'info': 'http://www.imdbapi.com/?i=%s', 'info': 'http://www.omdbapi.com/?i=%s',
} }
http_time_between_calls = 0 http_time_between_calls = 0
@ -32,7 +32,7 @@ class IMDBAPI(MovieProvider):
'name': q 'name': q
} }
cache_key = 'imdbapi.cache.%s' % q cache_key = 'omdbapi.cache.%s' % q
cached = self.getCache(cache_key, self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')}), timeout = 3) cached = self.getCache(cache_key, self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')}), timeout = 3)
if cached: if cached:
@ -50,7 +50,7 @@ class IMDBAPI(MovieProvider):
if not identifier: if not identifier:
return {} return {}
cache_key = 'imdbapi.cache.%s' % identifier cache_key = 'omdbapi.cache.%s' % identifier
cached = self.getCache(cache_key, self.urls['info'] % identifier, timeout = 3) cached = self.getCache(cache_key, self.urls['info'] % identifier, timeout = 3)
if cached: if cached:

36
couchpotato/core/providers/nzb/ftdworld/main.py

@ -1,12 +1,10 @@
from bs4 import BeautifulSoup
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
from couchpotato.core.helpers.variable import tryInt from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider from couchpotato.core.providers.nzb.base import NZBProvider
from couchpotato.environment import Env from couchpotato.environment import Env
from dateutil.parser import parse from dateutil.parser import parse
import re import traceback
import time
log = CPLog(__name__) log = CPLog(__name__)
@ -14,7 +12,7 @@ log = CPLog(__name__)
class FTDWorld(NZBProvider): class FTDWorld(NZBProvider):
urls = { urls = {
'search': 'http://ftdworld.net/category.php?%s', 'search': 'http://ftdworld.net/api/index.php?%s',
'detail': 'http://ftdworld.net/spotinfo.php?id=%s', 'detail': 'http://ftdworld.net/spotinfo.php?id=%s',
'download': 'http://ftdworld.net/cgi-bin/nzbdown.pl?fileID=%s', 'download': 'http://ftdworld.net/cgi-bin/nzbdown.pl?fileID=%s',
'login': 'http://ftdworld.net/index.php', 'login': 'http://ftdworld.net/index.php',
@ -25,7 +23,7 @@ class FTDWorld(NZBProvider):
cat_ids = [ cat_ids = [
([4, 11], ['dvdr']), ([4, 11], ['dvdr']),
([1], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr', 'brrip']), ([1], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr', 'brrip']),
([10, 13, 14], ['bd50', '720p', '1080p']), ([7, 10, 13, 14], ['bd50', '720p', '1080p']),
] ]
cat_backup_id = 1 cat_backup_id = 1
@ -43,38 +41,29 @@ class FTDWorld(NZBProvider):
'ctype': ','.join([str(x) for x in self.getCatId(quality['identifier'])]), 'ctype': ','.join([str(x) for x in self.getCatId(quality['identifier'])]),
}) })
data = self.getHTMLData(self.urls['search'] % params, opener = self.login_opener) data = self.getJsonData(self.urls['search'] % params, opener = self.login_opener)
if data: if data:
try: try:
html = BeautifulSoup(data) if data.get('numRes') == 0:
main_table = html.find('table', attrs = {'id':'ftdresult'})
if not main_table:
return return
items = main_table.find_all('tr', attrs = {'class': re.compile('tcontent')}) for item in data.get('data'):
for item in items:
tds = item.find_all('td')
nzb_id = tryInt(item.attrs['data-spot'])
up = item.find('img', attrs = {'src': re.compile('up.png')})
down = item.find('img', attrs = {'src': re.compile('down.png')})
nzb_id = tryInt(item.get('id'))
results.append({ results.append({
'id': nzb_id, 'id': nzb_id,
'name': toUnicode(item.find('a', attrs = {'href': re.compile('./spotinfo')}).text.strip()), 'name': toUnicode(item.get('Title')),
'age': self.calculateAge(int(time.mktime(parse(tds[2].text).timetuple()))), 'age': self.calculateAge(tryInt(item.get('Created'))),
'url': self.urls['download'] % nzb_id, 'url': self.urls['download'] % nzb_id,
'download': self.loginDownload, 'download': self.loginDownload,
'detail_url': self.urls['detail'] % nzb_id, 'detail_url': self.urls['detail'] % nzb_id,
'score': (tryInt(up.attrs['title'].split(' ')[0]) * 3) - (tryInt(down.attrs['title'].split(' ')[0]) * 3) if up else 0, 'score': (tryInt(item.get('webPlus', 0)) - tryInt(item.get('webMin', 0))) * 3,
}) })
except: except:
log.error('Failed to parse HTML response from FTDWorld') log.error('Failed to parse HTML response from FTDWorld: %s', traceback.format_exc())
def getLoginParams(self): def getLoginParams(self):
return tryUrlencode({ return tryUrlencode({
@ -82,3 +71,6 @@ class FTDWorld(NZBProvider):
'passlogin': self.conf('password'), 'passlogin': self.conf('password'),
'submit': 'Log In', 'submit': 'Log In',
}) })
def loginSuccess(self, output):
return 'password is incorrect' not in output

8
couchpotato/core/providers/nzb/newznab/__init__.py

@ -13,7 +13,7 @@ config = [{
'order': 10, 'order': 10,
'description': 'Enable <a href="http://newznab.com/" target="_blank">NewzNab providers</a> such as <a href="https://nzb.su" target="_blank">NZB.su</a>, \ 'description': 'Enable <a href="http://newznab.com/" target="_blank">NewzNab providers</a> such as <a href="https://nzb.su" target="_blank">NZB.su</a>, \
<a href="https://nzbs.org" target="_blank">NZBs.org</a>, <a href="http://dognzb.cr/" target="_blank">DOGnzb.cr</a>, \ <a href="https://nzbs.org" target="_blank">NZBs.org</a>, <a href="http://dognzb.cr/" target="_blank">DOGnzb.cr</a>, \
<a href="https://github.com/spotweb/spotweb" target="_blank">Spotweb</a>', <a href="https://github.com/spotweb/spotweb" target="_blank">Spotweb</a> or <a href="https://nzbgeek.info/" target="_blank">NZBGeek</a>',
'wizard': True, 'wizard': True,
'options': [ 'options': [
{ {
@ -22,16 +22,16 @@ config = [{
}, },
{ {
'name': 'use', 'name': 'use',
'default': '0,0,0' 'default': '0,0,0,0'
}, },
{ {
'name': 'host', 'name': 'host',
'default': 'nzb.su,dognzb.cr,nzbs.org', 'default': 'nzb.su,dognzb.cr,nzbs.org,https://index.nzbgeek.info',
'description': 'The hostname of your newznab provider', 'description': 'The hostname of your newznab provider',
}, },
{ {
'name': 'api_key', 'name': 'api_key',
'default': ',,', 'default': ',,,',
'label': 'Api Key', 'label': 'Api Key',
'description': 'Can be found on your profile page', 'description': 'Can be found on your profile page',
'type': 'combined', 'type': 'combined',

4
couchpotato/core/providers/nzb/newznab/main.py

@ -29,7 +29,7 @@ class Newznab(NZBProvider, RSS):
def search(self, movie, quality): def search(self, movie, quality):
hosts = self.getHosts() hosts = self.getHosts()
results = ResultList(self, movie, quality, imdb_result = True) results = ResultList(self, movie, quality, imdb_results = True)
for host in hosts: for host in hosts:
if self.isDisabled(host): if self.isDisabled(host):
@ -136,6 +136,6 @@ class Newznab(NZBProvider, RSS):
self.limits_reached[host] = time.time() self.limits_reached[host] = time.time()
return 'try_next' return 'try_next'
log.error('Failed download from %s', (host, traceback.format_exc())) log.error('Failed download from %s: %s', (host, traceback.format_exc()))
return 'try_next' return 'try_next'

2
couchpotato/core/providers/nzb/nzbx/__init__.py

@ -10,7 +10,7 @@ config = [{
'tab': 'searcher', 'tab': 'searcher',
'subtab': 'nzb_providers', 'subtab': 'nzb_providers',
'name': 'nzbX', 'name': 'nzbX',
'description': 'Free provider, less accurate. See <a href="https://www.nzbx.co/">nzbX</a>', 'description': 'Free provider. See <a href="https://www.nzbx.co/">nzbX</a>',
'options': [ 'options': [
{ {
'name': 'enabled', 'name': 'enabled',

3
couchpotato/core/providers/nzb/nzbx/main.py

@ -2,6 +2,7 @@ from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import tryInt from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider from couchpotato.core.providers.nzb.base import NZBProvider
from couchpotato.environment import Env
log = CPLog(__name__) log = CPLog(__name__)
@ -22,7 +23,7 @@ class Nzbx(NZBProvider):
'q': movie['library']['identifier'].replace('tt', ''), 'q': movie['library']['identifier'].replace('tt', ''),
'sf': quality.get('size_min'), 'sf': quality.get('size_min'),
}) })
nzbs = self.getJsonData(self.urls['search'] % arguments) nzbs = self.getJsonData(self.urls['search'] % arguments, headers = {'User-Agent': Env.getIdentifier()})
for nzb in nzbs: for nzb in nzbs:

2
couchpotato/core/providers/nzb/omgwtfnzbs/main.py

@ -14,7 +14,7 @@ log = CPLog(__name__)
class OMGWTFNZBs(NZBProvider, RSS): class OMGWTFNZBs(NZBProvider, RSS):
urls = { urls = {
'search': 'http://rss.omgwtfnzbs.com/rss-search.php?%s', 'search': 'http://rss.omgwtfnzbs.org/rss-search.php?%s',
} }
http_time_between_calls = 1 #seconds http_time_between_calls = 1 #seconds

2
couchpotato/runner.py

@ -4,7 +4,6 @@ from couchpotato.api import api, NonBlockHandler
from couchpotato.core.event import fireEventAsync, fireEvent from couchpotato.core.event import fireEventAsync, fireEvent
from couchpotato.core.helpers.variable import getDataDir, tryInt from couchpotato.core.helpers.variable import getDataDir, tryInt
from logging import handlers from logging import handlers
from tornado.ioloop import IOLoop
from tornado.web import Application, FallbackHandler from tornado.web import Application, FallbackHandler
from tornado.wsgi import WSGIContainer from tornado.wsgi import WSGIContainer
from werkzeug.contrib.cache import FileSystemCache from werkzeug.contrib.cache import FileSystemCache
@ -231,6 +230,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
fireEventAsync('app.load') fireEventAsync('app.load')
# Go go go! # Go go go!
from tornado.ioloop import IOLoop
web_container = WSGIContainer(app) web_container = WSGIContainer(app)
web_container._log = _log web_container._log = _log
loop = IOLoop.instance() loop = IOLoop.instance()

2
couchpotato/static/scripts/api.js

@ -13,7 +13,7 @@ var ApiClass = new Class({
return new Request[r_type](Object.merge({ return new Request[r_type](Object.merge({
'callbackKey': 'callback_func', 'callbackKey': 'callback_func',
'method': 'get', 'method': 'get',
'url': self.createUrl(type), 'url': self.createUrl(type, {'t': randomString()}),
}, options)).send() }, options)).send()
}, },

104
init/freebsd

@ -1,89 +1,49 @@
#!/bin/sh #!/bin/sh
# #
# PROVIDE: couchpotato # PROVIDE: couchpotato
# REQUIRE: sabnzbd # REQUIRE: DAEMON
# KEYWORD: shutdown # KEYWORD: shutdown
#
# Add the following lines to /etc/rc.conf.local or /etc/rc.conf # Add the following lines to /etc/rc.conf to enable couchpotato:
# to enable this service: # couchpotato_enable: Set to NO by default. Set it to YES to enable it.
#
# couchpotato_enable (bool): Set to NO by default.
# Set it to YES to enable it.
# couchpotato_user: The user account CouchPotato daemon runs as what # couchpotato_user: The user account CouchPotato daemon runs as what
# you want it to be. It uses '_sabnzbd' user by # you want it to be.
# default. Do not sets it as empty or it will run
# as root.
# couchpotato_dir: Directory where CouchPotato lives. # couchpotato_dir: Directory where CouchPotato lives.
# Default: /usr/local/couchpotato # Default: /usr/local/CouchPotatoServer
# couchpotato_chdir: Change to this directory before running CouchPotato. # couchpotato_datadir: Directory where CouchPotato user data lives.
# Default is same as couchpotato_dir. # Default: $couchpotato_dir/data
# couchpotato_pid: The name of the pidfile to create. # couchpotato_conf: Directory where CouchPotato user data lives.
# Default is couchpotato.pid in couchpotato_dir. # Default: $couchpotato_datadir/settings.conf
# couchpotato_pid: Full path to PID file.
# Default: $couchpotato_datadir/couchpotato.pid
# couchpotato_flags: Set additonal flags as needed.
. /etc/rc.subr . /etc/rc.subr
name="couchpotato" name="couchpotato"
rcvar=${name}_enable rcvar=couchpotato_enable
load_rc_config ${name} load_rc_config ${name}
: ${couchpotato_enable:="NO"} : ${couchpotato_enable:=NO}
: ${couchpotato_user:="_sabnzbd"} : ${couchpotato_user:=} #default is root
: ${couchpotato_dir:="/usr/local/couchpotato"} : ${couchpotato_dir:=/usr/local/CouchPotatoServer}
: ${couchpotato_chdir:="${couchpotato_dir}"} : ${couchpotato_datadir:=${couchpotato_dir}/data}
: ${couchpotato_pid:="${couchpotato_dir}/couchpotato.pid"} : ${couchpotato_conf:=} #default is datadir/settings.conf
: ${couchpotato_conf:="${couchpotato_dir}/data/settings.conf"} : ${couchpotato_pid:=} #default is datadir/couchpotato.pid
: ${couchpotato_flags:=}
WGET="/usr/local/bin/wget" # You need wget for this script to safely shutdown CouchPotato.
if [ -e "${couchpotato_conf}" ]; then command="${couchpotato_dir}/CouchPotato.py"
HOST=`grep -A14 "\[core\]" "${couchpotato_conf}"|egrep "^host"|perl -wple 's/^host = (.*)$/$1/'` command_interpreter="/usr/local/bin/python"
PORT=`grep -A14 "\[core\]" "${couchpotato_conf}"|egrep "^port"|perl -wple 's/^port = (.*)$/$1/'` command_args="--daemon --data_dir ${couchpotato_datadir}"
CPAPI=`grep -A14 "\[core\]" "${couchpotato_conf}"|egrep "^api_key"|perl -wple 's/^api_key = (.*)$/$1/'`
# append optional flags
if [ -n "${couchpotato_pid}" ]; then
pidfile=${couchpotato_pid}
couchpotato_flags="${couchpotato_flags} --pid_file ${couchpotato_pid}"
fi fi
if [ -n "${couchpotato_conf}" ]; then
status_cmd="${name}_status" couchpotato_flags="${couchpotato_flags} --config_file ${couchpotato_conf}"
stop_cmd="${name}_stop"
command="/usr/sbin/daemon"
command_args="-f -p ${couchpotato_pid} python ${couchpotato_dir}/CouchPotato.py ${couchpotato_flags}"
# Check for wget and refuse to start without it.
if [ ! -x "${WGET}" ]; then
warn "couchpotato not started: You need wget to safely shut down CouchPotato."
exit 1
fi fi
# Ensure user is root when running this script.
if [ `id -u` != "0" ]; then
echo "Oops, you should be root before running this!"
exit 1
fi
verify_couchpotato_pid() {
# Make sure the pid corresponds to the CouchPotato process.
pid=`cat ${couchpotato_pid} 2>/dev/null`
ps -p ${pid} | grep -q "python ${couchpotato_dir}/CouchPotato.py"
return $?
}
# Try to stop CouchPotato cleanly by calling shutdown over http.
couchpotato_stop() {
if [ ! -e "${couchpotato_conf}" ]; then
echo "CouchPotato's settings file does not exist. Try starting CouchPotato, as this should create the file."
exit 1
fi
echo "Stopping $name"
verify_couchpotato_pid
${WGET} -O - -q "http://${HOST}:${PORT}/api/${CPAPI}/app.shutdown/" >/dev/null
if [ -n "${pid}" ]; then
wait_for_pids ${pid}
echo "Stopped"
fi
}
couchpotato_status() {
verify_couchpotato_pid && echo "$name is running as ${pid}" || echo "$name is not running"
}
run_rc_command "$1" run_rc_command "$1"

96
libs/gntp/__init__.py

@ -1,8 +1,9 @@
import hashlib
import re import re
import hashlib
import time import time
import StringIO
__version__ = '0.6' __version__ = '0.8'
#GNTP/<version> <messagetype> <encryptionAlgorithmID>[:<ivValue>][ <keyHashAlgorithmID>:<keyHash>.<salt>] #GNTP/<version> <messagetype> <encryptionAlgorithmID>[:<ivValue>][ <keyHashAlgorithmID>:<keyHash>.<salt>]
GNTP_INFO_LINE = re.compile( GNTP_INFO_LINE = re.compile(
@ -19,7 +20,7 @@ GNTP_INFO_LINE_SHORT = re.compile(
GNTP_HEADER = re.compile('([\w-]+):(.+)') GNTP_HEADER = re.compile('([\w-]+):(.+)')
GNTP_EOL = u'\r\n' GNTP_EOL = '\r\n'
class BaseError(Exception): class BaseError(Exception):
@ -43,6 +44,14 @@ class UnsupportedError(BaseError):
errordesc = 'Currently unsupported by gntp.py' errordesc = 'Currently unsupported by gntp.py'
class _GNTPBuffer(StringIO.StringIO):
"""GNTP Buffer class"""
def writefmt(self, message = "", *args):
"""Shortcut function for writing GNTP Headers"""
self.write((message % args).encode('utf8', 'replace'))
self.write(GNTP_EOL)
class _GNTPBase(object): class _GNTPBase(object):
"""Base initilization """Base initilization
@ -206,8 +215,8 @@ class _GNTPBase(object):
if not match: if not match:
continue continue
key = match.group(1).strip() key = unicode(match.group(1).strip(), 'utf8', 'replace')
val = match.group(2).strip() val = unicode(match.group(2).strip(), 'utf8', 'replace')
dict[key] = val dict[key] = val
return dict return dict
@ -217,6 +226,15 @@ class _GNTPBase(object):
else: else:
self.headers[key] = unicode('%s' % value, 'utf8', 'replace') self.headers[key] = unicode('%s' % value, 'utf8', 'replace')
def add_resource(self, data):
"""Add binary resource
:param string data: Binary Data
"""
identifier = hashlib.md5(data).hexdigest()
self.resources[identifier] = data
return 'x-growl-resource://%s' % identifier
def decode(self, data, password = None): def decode(self, data, password = None):
"""Decode GNTP Message """Decode GNTP Message
@ -229,19 +247,30 @@ class _GNTPBase(object):
self.headers = self._parse_dict(parts[0]) self.headers = self._parse_dict(parts[0])
def encode(self): def encode(self):
"""Encode a GNTP Message """Encode a generic GNTP Message
:return string: Encoded GNTP Message ready to be sent :return string: GNTP Message ready to be sent
""" """
self.validate()
message = self._format_info() + GNTP_EOL buffer = _GNTPBuffer()
buffer.writefmt(self._format_info())
#Headers #Headers
for k, v in self.headers.iteritems(): for k, v in self.headers.iteritems():
message += u'%s: %s%s' % (k, v, GNTP_EOL) buffer.writefmt('%s: %s', k, v)
buffer.writefmt()
message += GNTP_EOL #Resources
return message for resource, data in self.resources.iteritems():
buffer.writefmt('Identifier: %s', resource)
buffer.writefmt('Length: %d', len(data))
buffer.writefmt()
buffer.write(data)
buffer.writefmt()
buffer.writefmt()
return buffer.getvalue()
class GNTPRegister(_GNTPBase): class GNTPRegister(_GNTPBase):
@ -319,22 +348,33 @@ class GNTPRegister(_GNTPBase):
:return string: Encoded GNTP Registration message :return string: Encoded GNTP Registration message
""" """
self.validate()
message = self._format_info() + GNTP_EOL buffer = _GNTPBuffer()
buffer.writefmt(self._format_info())
#Headers #Headers
for k, v in self.headers.iteritems(): for k, v in self.headers.iteritems():
message += u'%s: %s%s' % (k, v, GNTP_EOL) buffer.writefmt('%s: %s', k, v)
buffer.writefmt()
#Notifications #Notifications
if len(self.notifications) > 0: if len(self.notifications) > 0:
for notice in self.notifications: for notice in self.notifications:
message += GNTP_EOL
for k, v in notice.iteritems(): for k, v in notice.iteritems():
message += u'%s: %s%s' % (k, v, GNTP_EOL) buffer.writefmt('%s: %s', k, v)
buffer.writefmt()
#Resources
for resource, data in self.resources.iteritems():
buffer.writefmt('Identifier: %s', resource)
buffer.writefmt('Length: %d', len(data))
buffer.writefmt()
buffer.write(data)
buffer.writefmt()
buffer.writefmt()
message += GNTP_EOL return buffer.getvalue()
return message
class GNTPNotice(_GNTPBase): class GNTPNotice(_GNTPBase):
@ -388,21 +428,6 @@ class GNTPNotice(_GNTPBase):
#open('notice.png','wblol').write(notice['Data']) #open('notice.png','wblol').write(notice['Data'])
self.resources[notice.get('Identifier')] = notice self.resources[notice.get('Identifier')] = notice
def encode(self):
"""Encode a GNTP Notification Message
:return string: GNTP Notification Message ready to be sent
"""
self.validate()
message = self._format_info() + GNTP_EOL
#Headers
for k, v in self.headers.iteritems():
message += u'%s: %s%s' % (k, v, GNTP_EOL)
message += GNTP_EOL
return message
class GNTPSubscribe(_GNTPBase): class GNTPSubscribe(_GNTPBase):
"""Represents a GNTP Subscribe Command """Represents a GNTP Subscribe Command
@ -457,7 +482,8 @@ class GNTPError(_GNTPBase):
self.add_header('Error-Description', errordesc) self.add_header('Error-Description', errordesc)
def error(self): def error(self):
return self.headers['Error-Code'], self.headers['Error-Description'] return (self.headers.get('Error-Code', None),
self.headers.get('Error-Description', None))
def parse_gntp(data, password = None): def parse_gntp(data, password = None):

107
libs/gntp/notifier.py

@ -22,43 +22,6 @@ __all__ = [
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def mini(description, applicationName = 'PythonMini', noteType = "Message",
title = "Mini Message", applicationIcon = None, hostname = 'localhost',
password = None, port = 23053, sticky = False, priority = None,
callback = None):
"""Single notification function
Simple notification function in one line. Has only one required parameter
and attempts to use reasonable defaults for everything else
:param string description: Notification message
.. warning::
For now, only URL callbacks are supported. In the future, the
callback argument will also support a function
"""
growl = GrowlNotifier(
applicationName = applicationName,
notifications = [noteType],
defaultNotifications = [noteType],
hostname = hostname,
password = password,
port = port,
)
result = growl.register()
if result is not True:
return result
return growl.notify(
noteType = noteType,
title = title,
description = description,
icon = applicationIcon,
sticky = sticky,
priority = priority,
callback = callback,
)
class GrowlNotifier(object): class GrowlNotifier(object):
"""Helper class to simplfy sending Growl messages """Helper class to simplfy sending Growl messages
@ -93,10 +56,12 @@ class GrowlNotifier(object):
def _checkIcon(self, data): def _checkIcon(self, data):
''' '''
Check the icon to see if it's valid Check the icon to see if it's valid
@param data:
@todo Consider checking for a valid URL If it's a simple URL icon, then we return True. If it's a data icon
then we return False
''' '''
return data logger.info('Checking icon')
return data.startswith('http')
def register(self): def register(self):
"""Send GNTP Registration """Send GNTP Registration
@ -112,7 +77,11 @@ class GrowlNotifier(object):
enabled = notification in self.defaultNotifications enabled = notification in self.defaultNotifications
register.add_notification(notification, enabled) register.add_notification(notification, enabled)
if self.applicationIcon: if self.applicationIcon:
if self._checkIcon(self.applicationIcon):
register.add_header('Application-Icon', self.applicationIcon) register.add_header('Application-Icon', self.applicationIcon)
else:
id = register.add_resource(self.applicationIcon)
register.add_header('Application-Icon', id)
if self.password: if self.password:
register.set_password(self.password, self.passwordHash) register.set_password(self.password, self.passwordHash)
self.add_origin_info(register) self.add_origin_info(register)
@ -120,7 +89,7 @@ class GrowlNotifier(object):
return self._send('register', register) return self._send('register', register)
def notify(self, noteType, title, description, icon = None, sticky = False, def notify(self, noteType, title, description, icon = None, sticky = False,
priority = None, callback = None): priority = None, callback = None, identifier = None):
"""Send a GNTP notifications """Send a GNTP notifications
.. warning:: .. warning::
@ -151,11 +120,18 @@ class GrowlNotifier(object):
if priority: if priority:
notice.add_header('Notification-Priority', priority) notice.add_header('Notification-Priority', priority)
if icon: if icon:
notice.add_header('Notification-Icon', self._checkIcon(icon)) if self._checkIcon(icon):
notice.add_header('Notification-Icon', icon)
else:
id = notice.add_resource(icon)
notice.add_header('Notification-Icon', id)
if description: if description:
notice.add_header('Notification-Text', description) notice.add_header('Notification-Text', description)
if callback: if callback:
notice.add_header('Notification-Callback-Target', callback) notice.add_header('Notification-Callback-Target', callback)
if identifier:
notice.add_header('Notification-Coalescing-ID', identifier)
self.add_origin_info(notice) self.add_origin_info(notice)
self.notify_hook(notice) self.notify_hook(notice)
@ -193,9 +169,10 @@ class GrowlNotifier(object):
def subscribe_hook(self, packet): def subscribe_hook(self, packet):
pass pass
def _send(self, type, packet): def _send(self, messagetype, packet):
"""Send the GNTP Packet""" """Send the GNTP Packet"""
packet.validate()
data = packet.encode() data = packet.encode()
logger.debug('To : %s:%s <%s>\n%s', self.hostname, self.port, packet.__class__, data) logger.debug('To : %s:%s <%s>\n%s', self.hostname, self.port, packet.__class__, data)
@ -203,7 +180,7 @@ class GrowlNotifier(object):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(self.socketTimeout) s.settimeout(self.socketTimeout)
s.connect((self.hostname, self.port)) s.connect((self.hostname, self.port))
s.send(data.encode('utf8', 'replace')) s.send(data)
recv_data = s.recv(1024) recv_data = s.recv(1024)
while not recv_data.endswith("\r\n\r\n"): while not recv_data.endswith("\r\n\r\n"):
recv_data += s.recv(1024) recv_data += s.recv(1024)
@ -212,11 +189,51 @@ class GrowlNotifier(object):
logger.debug('From : %s:%s <%s>\n%s', self.hostname, self.port, response.__class__, response) logger.debug('From : %s:%s <%s>\n%s', self.hostname, self.port, response.__class__, response)
if response.info['messagetype'] == '-OK': if type(response) == gntp.GNTPOK:
return True return True
logger.error('Invalid response: %s', response.error()) logger.error('Invalid response: %s', response.error())
return response.error() return response.error()
def mini(description, applicationName = 'PythonMini', noteType = "Message",
title = "Mini Message", applicationIcon = None, hostname = 'localhost',
password = None, port = 23053, sticky = False, priority = None,
callback = None, notificationIcon = None, identifier = None,
notifierFactory = GrowlNotifier):
"""Single notification function
Simple notification function in one line. Has only one required parameter
and attempts to use reasonable defaults for everything else
:param string description: Notification message
.. warning::
For now, only URL callbacks are supported. In the future, the
callback argument will also support a function
"""
growl = notifierFactory(
applicationName = applicationName,
notifications = [noteType],
defaultNotifications = [noteType],
applicationIcon = applicationIcon,
hostname = hostname,
password = password,
port = port,
)
result = growl.register()
if result is not True:
return result
return growl.notify(
noteType = noteType,
title = title,
description = description,
icon = notificationIcon,
sticky = sticky,
priority = priority,
callback = callback,
identifier = identifier,
)
if __name__ == '__main__': if __name__ == '__main__':
# If we're running this module directly we're likely running it as a test # If we're running this module directly we're likely running it as a test
# so extra debugging is useful # so extra debugging is useful

5
libs/tornado/curl_httpclient.py

@ -96,17 +96,18 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
pycurl.POLL_INOUT: ioloop.IOLoop.READ | ioloop.IOLoop.WRITE pycurl.POLL_INOUT: ioloop.IOLoop.READ | ioloop.IOLoop.WRITE
} }
if event == pycurl.POLL_REMOVE: if event == pycurl.POLL_REMOVE:
if fd in self._fds:
self.io_loop.remove_handler(fd) self.io_loop.remove_handler(fd)
del self._fds[fd] del self._fds[fd]
else: else:
ioloop_event = event_map[event] ioloop_event = event_map[event]
if fd not in self._fds: if fd not in self._fds:
self._fds[fd] = ioloop_event
self.io_loop.add_handler(fd, self._handle_events, self.io_loop.add_handler(fd, self._handle_events,
ioloop_event) ioloop_event)
else:
self._fds[fd] = ioloop_event self._fds[fd] = ioloop_event
else:
self.io_loop.update_handler(fd, ioloop_event) self.io_loop.update_handler(fd, ioloop_event)
self._fds[fd] = ioloop_event
def _set_timeout(self, msecs): def _set_timeout(self, msecs):
"""Called by libcurl to schedule a timeout.""" """Called by libcurl to schedule a timeout."""

44
libs/tornado/ioloop.py

@ -194,7 +194,7 @@ class IOLoop(Configurable):
def initialize(self): def initialize(self):
pass pass
def close(self, all_fds=False): def close(self, all_fds = False):
"""Closes the IOLoop, freeing any resources used. """Closes the IOLoop, freeing any resources used.
If ``all_fds`` is true, all file descriptors registered on the If ``all_fds`` is true, all file descriptors registered on the
@ -320,7 +320,7 @@ class IOLoop(Configurable):
""" """
raise NotImplementedError() raise NotImplementedError()
def add_callback(self, callback): def add_callback(self, callback, *args, **kwargs):
"""Calls the given callback on the next I/O loop iteration. """Calls the given callback on the next I/O loop iteration.
It is safe to call this method from any thread at any time, It is safe to call this method from any thread at any time,
@ -335,7 +335,7 @@ class IOLoop(Configurable):
""" """
raise NotImplementedError() raise NotImplementedError()
def add_callback_from_signal(self, callback): def add_callback_from_signal(self, callback, *args, **kwargs):
"""Calls the given callback on the next I/O loop iteration. """Calls the given callback on the next I/O loop iteration.
Safe for use from a Python signal handler; should not be used Safe for use from a Python signal handler; should not be used
@ -359,8 +359,7 @@ class IOLoop(Configurable):
assert isinstance(future, IOLoop._FUTURE_TYPES) assert isinstance(future, IOLoop._FUTURE_TYPES)
callback = stack_context.wrap(callback) callback = stack_context.wrap(callback)
future.add_done_callback( future.add_done_callback(
lambda future: self.add_callback( lambda future: self.add_callback(callback, future))
functools.partial(callback, future)))
def _run_callback(self, callback): def _run_callback(self, callback):
"""Runs a callback with error handling. """Runs a callback with error handling.
@ -382,7 +381,7 @@ class IOLoop(Configurable):
The exception itself is not passed explicitly, but is available The exception itself is not passed explicitly, but is available
in sys.exc_info. in sys.exc_info.
""" """
app_log.error("Exception in callback %r", callback, exc_info=True) app_log.error("Exception in callback %r", callback, exc_info = True)
@ -393,7 +392,7 @@ class PollIOLoop(IOLoop):
(Linux), `tornado.platform.kqueue.KQueueIOLoop` (BSD and Mac), or (Linux), `tornado.platform.kqueue.KQueueIOLoop` (BSD and Mac), or
`tornado.platform.select.SelectIOLoop` (all platforms). `tornado.platform.select.SelectIOLoop` (all platforms).
""" """
def initialize(self, impl, time_func=None): def initialize(self, impl, time_func = None):
super(PollIOLoop, self).initialize() super(PollIOLoop, self).initialize()
self._impl = impl self._impl = impl
if hasattr(self._impl, 'fileno'): if hasattr(self._impl, 'fileno'):
@ -417,7 +416,7 @@ class PollIOLoop(IOLoop):
lambda fd, events: self._waker.consume(), lambda fd, events: self._waker.consume(),
self.READ) self.READ)
def close(self, all_fds=False): def close(self, all_fds = False):
with self._callback_lock: with self._callback_lock:
self._closing = True self._closing = True
self.remove_handler(self._waker.fileno()) self.remove_handler(self._waker.fileno())
@ -426,7 +425,7 @@ class PollIOLoop(IOLoop):
try: try:
os.close(fd) os.close(fd)
except Exception: except Exception:
gen_log.debug("error closing fd %s", fd, exc_info=True) gen_log.debug("error closing fd %s", fd, exc_info = True)
self._waker.close() self._waker.close()
self._impl.close() self._impl.close()
@ -442,8 +441,8 @@ class PollIOLoop(IOLoop):
self._events.pop(fd, None) self._events.pop(fd, None)
try: try:
self._impl.unregister(fd) self._impl.unregister(fd)
except (OSError, IOError): except Exception:
gen_log.debug("Error deleting fd from IOLoop", exc_info=True) gen_log.debug("Error deleting fd from IOLoop", exc_info = True)
def set_blocking_signal_threshold(self, seconds, action): def set_blocking_signal_threshold(self, seconds, action):
if not hasattr(signal, "setitimer"): if not hasattr(signal, "setitimer"):
@ -569,17 +568,18 @@ class PollIOLoop(IOLoop):
while self._events: while self._events:
fd, events = self._events.popitem() fd, events = self._events.popitem()
try: try:
self._handlers[fd](fd, events) hdlr = self._handlers.get(fd)
if hdlr: hdlr(fd, events)
except (OSError, IOError), e: except (OSError, IOError), e:
if e.args[0] == errno.EPIPE: if e.args[0] == errno.EPIPE:
# Happens when the client closes the connection # Happens when the client closes the connection
pass pass
else: else:
app_log.error("Exception in I/O handler for fd %s", app_log.error("Exception in I/O handler for fd %s",
fd, exc_info=True) fd, exc_info = True)
except Exception: except Exception:
app_log.error("Exception in I/O handler for fd %s", app_log.error("Exception in I/O handler for fd %s",
fd, exc_info=True) fd, exc_info = True)
# reset the stopped flag so another start/stop pair can be issued # reset the stopped flag so another start/stop pair can be issued
self._stopped = False self._stopped = False
if self._blocking_signal_threshold is not None: if self._blocking_signal_threshold is not None:
@ -609,12 +609,13 @@ class PollIOLoop(IOLoop):
# collection pass whenever there are too many dead timeouts. # collection pass whenever there are too many dead timeouts.
timeout.callback = None timeout.callback = None
def add_callback(self, callback): def add_callback(self, callback, *args, **kwargs):
with self._callback_lock: with self._callback_lock:
if self._closing: if self._closing:
raise RuntimeError("IOLoop is closing") raise RuntimeError("IOLoop is closing")
list_empty = not self._callbacks list_empty = not self._callbacks
self._callbacks.append(stack_context.wrap(callback)) self._callbacks.append(functools.partial(
stack_context.wrap(callback), *args, **kwargs))
if list_empty and thread.get_ident() != self._thread_ident: if list_empty and thread.get_ident() != self._thread_ident:
# If we're in the IOLoop's thread, we know it's not currently # If we're in the IOLoop's thread, we know it's not currently
# polling. If we're not, and we added the first callback to an # polling. If we're not, and we added the first callback to an
@ -624,12 +625,12 @@ class PollIOLoop(IOLoop):
# avoid it when we can. # avoid it when we can.
self._waker.wake() self._waker.wake()
def add_callback_from_signal(self, callback): def add_callback_from_signal(self, callback, *args, **kwargs):
with stack_context.NullContext(): with stack_context.NullContext():
if thread.get_ident() != self._thread_ident: if thread.get_ident() != self._thread_ident:
# if the signal is handled on another thread, we can add # if the signal is handled on another thread, we can add
# it normally (modulo the NullContext) # it normally (modulo the NullContext)
self.add_callback(callback) self.add_callback(callback, *args, **kwargs)
else: else:
# If we're on the IOLoop's thread, we cannot use # If we're on the IOLoop's thread, we cannot use
# the regular add_callback because it may deadlock on # the regular add_callback because it may deadlock on
@ -639,7 +640,8 @@ class PollIOLoop(IOLoop):
# _callback_lock block in IOLoop.start, we may modify # _callback_lock block in IOLoop.start, we may modify
# either the old or new version of self._callbacks, # either the old or new version of self._callbacks,
# but either way will work. # but either way will work.
self._callbacks.append(stack_context.wrap(callback)) self._callbacks.append(functools.partial(
stack_context.wrap(callback), *args, **kwargs))
class _Timeout(object): class _Timeout(object):
@ -682,7 +684,7 @@ class PeriodicCallback(object):
`start` must be called after the PeriodicCallback is created. `start` must be called after the PeriodicCallback is created.
""" """
def __init__(self, callback, callback_time, io_loop=None): def __init__(self, callback, callback_time, io_loop = None):
self.callback = callback self.callback = callback
if callback_time <= 0: if callback_time <= 0:
raise ValueError("Periodic callback must have a positive callback_time") raise ValueError("Periodic callback must have a positive callback_time")
@ -710,7 +712,7 @@ class PeriodicCallback(object):
try: try:
self.callback() self.callback()
except Exception: except Exception:
app_log.error("Error in periodic callback", exc_info=True) app_log.error("Error in periodic callback", exc_info = True)
self._schedule_next() self._schedule_next()
def _schedule_next(self): def _schedule_next(self):

38
libs/tornado/iostream.py

@ -209,11 +209,19 @@ class BaseIOStream(object):
"""Call the given callback when the stream is closed.""" """Call the given callback when the stream is closed."""
self._close_callback = stack_context.wrap(callback) self._close_callback = stack_context.wrap(callback)
def close(self): def close(self, exc_info=False):
"""Close this stream.""" """Close this stream.
If ``exc_info`` is true, set the ``error`` attribute to the current
exception from `sys.exc_info()` (or if ``exc_info`` is a tuple,
use that instead of `sys.exc_info`).
"""
if not self.closed(): if not self.closed():
if any(sys.exc_info()): if exc_info:
self.error = sys.exc_info()[1] if not isinstance(exc_info, tuple):
exc_info = sys.exc_info()
if any(exc_info):
self.error = exc_info[1]
if self._read_until_close: if self._read_until_close:
callback = self._read_callback callback = self._read_callback
self._read_callback = None self._read_callback = None
@ -285,7 +293,7 @@ class BaseIOStream(object):
except Exception: except Exception:
gen_log.error("Uncaught exception, closing connection.", gen_log.error("Uncaught exception, closing connection.",
exc_info=True) exc_info=True)
self.close() self.close(exc_info=True)
raise raise
def _run_callback(self, callback, *args): def _run_callback(self, callback, *args):
@ -300,7 +308,7 @@ class BaseIOStream(object):
# (It would eventually get closed when the socket object is # (It would eventually get closed when the socket object is
# gc'd, but we don't want to rely on gc happening before we # gc'd, but we don't want to rely on gc happening before we
# run out of file descriptors) # run out of file descriptors)
self.close() self.close(exc_info=True)
# Re-raise the exception so that IOLoop.handle_callback_exception # Re-raise the exception so that IOLoop.handle_callback_exception
# can see it and log the error # can see it and log the error
raise raise
@ -348,7 +356,7 @@ class BaseIOStream(object):
self._pending_callbacks -= 1 self._pending_callbacks -= 1
except Exception: except Exception:
gen_log.warning("error on read", exc_info=True) gen_log.warning("error on read", exc_info=True)
self.close() self.close(exc_info=True)
return return
if self._read_from_buffer(): if self._read_from_buffer():
return return
@ -397,9 +405,9 @@ class BaseIOStream(object):
# Treat ECONNRESET as a connection close rather than # Treat ECONNRESET as a connection close rather than
# an error to minimize log spam (the exception will # an error to minimize log spam (the exception will
# be available on self.error for apps that care). # be available on self.error for apps that care).
self.close() self.close(exc_info=True)
return return
self.close() self.close(exc_info=True)
raise raise
if chunk is None: if chunk is None:
return 0 return 0
@ -503,7 +511,7 @@ class BaseIOStream(object):
else: else:
gen_log.warning("Write error on %d: %s", gen_log.warning("Write error on %d: %s",
self.fileno(), e) self.fileno(), e)
self.close() self.close(exc_info=True)
return return
if not self._write_buffer and self._write_callback: if not self._write_buffer and self._write_callback:
callback = self._write_callback callback = self._write_callback
@ -664,7 +672,7 @@ class IOStream(BaseIOStream):
if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK): if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK):
gen_log.warning("Connect error on fd %d: %s", gen_log.warning("Connect error on fd %d: %s",
self.socket.fileno(), e) self.socket.fileno(), e)
self.close() self.close(exc_info=True)
return return
self._connect_callback = stack_context.wrap(callback) self._connect_callback = stack_context.wrap(callback)
self._add_io_state(self.io_loop.WRITE) self._add_io_state(self.io_loop.WRITE)
@ -733,7 +741,7 @@ class SSLIOStream(IOStream):
return return
elif err.args[0] in (ssl.SSL_ERROR_EOF, elif err.args[0] in (ssl.SSL_ERROR_EOF,
ssl.SSL_ERROR_ZERO_RETURN): ssl.SSL_ERROR_ZERO_RETURN):
return self.close() return self.close(exc_info=True)
elif err.args[0] == ssl.SSL_ERROR_SSL: elif err.args[0] == ssl.SSL_ERROR_SSL:
try: try:
peer = self.socket.getpeername() peer = self.socket.getpeername()
@ -741,11 +749,11 @@ class SSLIOStream(IOStream):
peer = '(not connected)' peer = '(not connected)'
gen_log.warning("SSL Error on %d %s: %s", gen_log.warning("SSL Error on %d %s: %s",
self.socket.fileno(), peer, err) self.socket.fileno(), peer, err)
return self.close() return self.close(exc_info=True)
raise raise
except socket.error, err: except socket.error, err:
if err.args[0] in (errno.ECONNABORTED, errno.ECONNRESET): if err.args[0] in (errno.ECONNABORTED, errno.ECONNRESET):
return self.close() return self.close(exc_info=True)
else: else:
self._ssl_accepting = False self._ssl_accepting = False
if self._ssl_connect_callback is not None: if self._ssl_connect_callback is not None:
@ -842,7 +850,7 @@ class PipeIOStream(BaseIOStream):
elif e.args[0] == errno.EBADF: elif e.args[0] == errno.EBADF:
# If the writing half of a pipe is closed, select will # If the writing half of a pipe is closed, select will
# report it as readable but reads will fail with EBADF. # report it as readable but reads will fail with EBADF.
self.close() self.close(exc_info=True)
return None return None
else: else:
raise raise

19
libs/tornado/platform/twisted.py

@ -431,6 +431,8 @@ class TwistedIOLoop(tornado.ioloop.IOLoop):
self.reactor.removeWriter(self.fds[fd]) self.reactor.removeWriter(self.fds[fd])
def remove_handler(self, fd): def remove_handler(self, fd):
if fd not in self.fds:
return
self.fds[fd].lost = True self.fds[fd].lost = True
if self.fds[fd].reading: if self.fds[fd].reading:
self.reactor.removeReader(self.fds[fd]) self.reactor.removeReader(self.fds[fd])
@ -444,6 +446,12 @@ class TwistedIOLoop(tornado.ioloop.IOLoop):
def stop(self): def stop(self):
self.reactor.crash() self.reactor.crash()
def _run_callback(self, callback, *args, **kwargs):
try:
callback(*args, **kwargs)
except Exception:
self.handle_callback_exception(callback)
def add_timeout(self, deadline, callback): def add_timeout(self, deadline, callback):
if isinstance(deadline, (int, long, float)): if isinstance(deadline, (int, long, float)):
delay = max(deadline - self.time(), 0) delay = max(deadline - self.time(), 0)
@ -451,13 +459,14 @@ class TwistedIOLoop(tornado.ioloop.IOLoop):
delay = deadline.total_seconds() delay = deadline.total_seconds()
else: else:
raise TypeError("Unsupported deadline %r") raise TypeError("Unsupported deadline %r")
return self.reactor.callLater(delay, wrap(callback)) return self.reactor.callLater(delay, self._run_callback, wrap(callback))
def remove_timeout(self, timeout): def remove_timeout(self, timeout):
timeout.cancel() timeout.cancel()
def add_callback(self, callback): def add_callback(self, callback, *args, **kwargs):
self.reactor.callFromThread(wrap(callback)) self.reactor.callFromThread(self._run_callback,
wrap(callback), *args, **kwargs)
def add_callback_from_signal(self, callback): def add_callback_from_signal(self, callback, *args, **kwargs):
self.add_callback(callback) self.add_callback(callback, *args, **kwargs)

2
libs/tornado/process.py

@ -268,7 +268,7 @@ class Subprocess(object):
assert ret_pid == pid assert ret_pid == pid
subproc = cls._waiting.pop(pid) subproc = cls._waiting.pop(pid)
subproc.io_loop.add_callback_from_signal( subproc.io_loop.add_callback_from_signal(
functools.partial(subproc._set_returncode, status)) subproc._set_returncode, status)
def _set_returncode(self, status): def _set_returncode(self, status):
if os.WIFSIGNALED(status): if os.WIFSIGNALED(status):

24
libs/tornado/simple_httpclient.py

@ -12,7 +12,6 @@ from tornado.util import b, GzipDecompressor
import base64 import base64
import collections import collections
import contextlib
import copy import copy
import functools import functools
import os.path import os.path
@ -134,7 +133,7 @@ class _HTTPConnection(object):
self._decompressor = None self._decompressor = None
# Timeout handle returned by IOLoop.add_timeout # Timeout handle returned by IOLoop.add_timeout
self._timeout = None self._timeout = None
with stack_context.StackContext(self.cleanup): with stack_context.ExceptionStackContext(self._handle_exception):
self.parsed = urlparse.urlsplit(_unicode(self.request.url)) self.parsed = urlparse.urlsplit(_unicode(self.request.url))
if ssl is None and self.parsed.scheme == "https": if ssl is None and self.parsed.scheme == "https":
raise ValueError("HTTPS requires either python2.6+ or " raise ValueError("HTTPS requires either python2.6+ or "
@ -309,19 +308,24 @@ class _HTTPConnection(object):
if self.final_callback is not None: if self.final_callback is not None:
final_callback = self.final_callback final_callback = self.final_callback
self.final_callback = None self.final_callback = None
final_callback(response) self.io_loop.add_callback(final_callback, response)
@contextlib.contextmanager def _handle_exception(self, typ, value, tb):
def cleanup(self): if self.final_callback:
try: gen_log.warning("uncaught exception", exc_info=(typ, value, tb))
yield self._run_callback(HTTPResponse(self.request, 599, error=value,
except Exception, e:
gen_log.warning("uncaught exception", exc_info=True)
self._run_callback(HTTPResponse(self.request, 599, error=e,
request_time=self.io_loop.time() - self.start_time, request_time=self.io_loop.time() - self.start_time,
)) ))
if hasattr(self, "stream"): if hasattr(self, "stream"):
self.stream.close() self.stream.close()
return True
else:
# If our callback has already been called, we are probably
# catching an exception that is not caused by us but rather
# some child of our callback. Rather than drop it on the floor,
# pass it along.
return False
def _on_close(self): def _on_close(self):
if self.final_callback is not None: if self.final_callback is not None:

12
libs/tornado/testing.py

@ -36,9 +36,8 @@ except ImportError:
netutil = None netutil = None
SimpleAsyncHTTPClient = None SimpleAsyncHTTPClient = None
from tornado.log import gen_log from tornado.log import gen_log
from tornado.stack_context import StackContext from tornado.stack_context import ExceptionStackContext
from tornado.util import raise_exc_info from tornado.util import raise_exc_info
import contextlib
import logging import logging
import os import os
import re import re
@ -167,13 +166,10 @@ class AsyncTestCase(unittest.TestCase):
''' '''
return IOLoop() return IOLoop()
@contextlib.contextmanager def _handle_exception(self, typ, value, tb):
def _stack_context(self):
try:
yield
except Exception:
self.__failure = sys.exc_info() self.__failure = sys.exc_info()
self.stop() self.stop()
return True
def __rethrow(self): def __rethrow(self):
if self.__failure is not None: if self.__failure is not None:
@ -182,7 +178,7 @@ class AsyncTestCase(unittest.TestCase):
raise_exc_info(failure) raise_exc_info(failure)
def run(self, result=None): def run(self, result=None):
with StackContext(self._stack_context): with ExceptionStackContext(self._handle_exception):
super(AsyncTestCase, self).run(result) super(AsyncTestCase, self).run(result)
# In case an exception escaped super.run or the StackContext caught # In case an exception escaped super.run or the StackContext caught
# an exception when there wasn't a wait() to re-raise it, do so here. # an exception when there wasn't a wait() to re-raise it, do so here.

15
libs/tornado/web.py

@ -1317,10 +1317,8 @@ class Application(object):
def add_handlers(self, host_pattern, host_handlers): def add_handlers(self, host_pattern, host_handlers):
"""Appends the given handlers to our handler list. """Appends the given handlers to our handler list.
Note that host patterns are processed sequentially in the Host patterns are processed sequentially in the order they were
order they were added, and only the first matching pattern is added. All matching patterns will be considered.
used. This means that all handlers for a given host must be
added in a single add_handlers call.
""" """
if not host_pattern.endswith("$"): if not host_pattern.endswith("$"):
host_pattern += "$" host_pattern += "$"
@ -1365,15 +1363,16 @@ class Application(object):
def _get_host_handlers(self, request): def _get_host_handlers(self, request):
host = request.host.lower().split(':')[0] host = request.host.lower().split(':')[0]
matches = []
for pattern, handlers in self.handlers: for pattern, handlers in self.handlers:
if pattern.match(host): if pattern.match(host):
return handlers matches.extend(handlers)
# Look for default host if not behind load balancer (for debugging) # Look for default host if not behind load balancer (for debugging)
if "X-Real-Ip" not in request.headers: if not matches and "X-Real-Ip" not in request.headers:
for pattern, handlers in self.handlers: for pattern, handlers in self.handlers:
if pattern.match(self.default_host): if pattern.match(self.default_host):
return handlers matches.extend(handlers)
return None return matches or None
def _load_ui_methods(self, methods): def _load_ui_methods(self, methods):
if type(methods) is types.ModuleType: if type(methods) is types.ModuleType:

Loading…
Cancel
Save