diff --git a/couchpotato/core/downloaders/blackhole/main.py b/couchpotato/core/downloaders/blackhole/main.py index 922dbce..a0d142b 100644 --- a/couchpotato/core/downloaders/blackhole/main.py +++ b/couchpotato/core/downloaders/blackhole/main.py @@ -5,7 +5,6 @@ from couchpotato.core.logger import CPLog from inspect import isfunction import os import traceback -import urllib log = CPLog(__name__) @@ -30,12 +29,12 @@ class Blackhole(Downloader): log.info('Downloading %s to %s.' % (data.get('type'), fullPath)) if isfunction(data.get('download')): file = data.get('download')() - if not file: - log.debug('Failed download file: %s' % data.get('name')) - return False else: - log.info('Downloading: %s' % data.get('url')) - file = urllib.urlopen(data.get('url')).read() + file = self.urlopen(data.get('url')) + + if not file or file == '': + log.debug('Failed download file: %s' % data.get('name')) + return False with open(fullPath, 'wb') as f: f.write(file) diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index 023b63b..e49522f 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -6,7 +6,6 @@ from urllib import urlencode import base64 import os import re -import urllib2 log = CPLog(__name__) @@ -52,15 +51,14 @@ class Sabnzbd(Downloader): params['script'] = pp_script_fn url = cleanHost(self.conf('host')) + "api?" + urlencode(params) - log.info("URL: " + url) try: - r = urllib2.urlopen(url) + data = self.urlopen(url) except Exception, e: log.error("Unable to connect to SAB: %s" % e) return False - result = r.read().strip() + result = data.strip() if not result: log.error("SABnzbd didn't return anything.") return False diff --git a/couchpotato/core/notifications/nmj/main.py b/couchpotato/core/notifications/nmj/main.py index 03cb272..27dc06f 100644 --- a/couchpotato/core/notifications/nmj/main.py +++ b/couchpotato/core/notifications/nmj/main.py @@ -6,7 +6,6 @@ from couchpotato.environment import Env import re import telnetlib import urllib -import urllib2 try: import xml.etree.cElementTree as etree @@ -81,12 +80,10 @@ class NMJ(Notification): database = self.conf('database') if self.mount: - try: - req = urllib2.Request(mount) - log.debug('Try to mount network drive via url: %s' % (mount)) - handle = urllib2.urlopen(req) - except IOError, e: - log.error('Warning: Couldn\'t contact popcorn hour on host %s: %s' % (host, e)) + log.debug('Try to mount network drive via url: %s' % (mount)) + data = self.urlopen(mount) + if not data: + log.error('Warning: Couldn\'t contact popcorn hour on host %s' % host) return False params = { @@ -99,13 +96,9 @@ class NMJ(Notification): UPDATE_URL = 'http://%(host)s:8008/metadata_database?%(params)s' updateUrl = UPDATE_URL % {'host': host, 'params': params} - try: - req = urllib2.Request(updateUrl) - log.debug('Sending NMJ scan update command via url: %s' % (updateUrl)) - handle = urllib2.urlopen(req) - response = handle.read() - except IOError, e: - log.error('Warning: Couldn\'t contact Popcorn Hour on host %s: %s' % (host, e)) + response = self.urlopen(updateUrl) + if not response: + log.error('Warning: Couldn\'t contact Popcorn Hour on host %s' % host) return False try: diff --git a/couchpotato/core/notifications/notifo/main.py b/couchpotato/core/notifications/notifo/main.py index cc22c30..d4ea1f3 100644 --- a/couchpotato/core/notifications/notifo/main.py +++ b/couchpotato/core/notifications/notifo/main.py @@ -1,13 +1,9 @@ -from couchpotato.api import addApiView -from couchpotato.core.event import addEvent from couchpotato.core.helpers.encoding import toUnicode from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification from couchpotato.environment import Env from flask.helpers import json import base64 -import urllib -import urllib2 log = CPLog(__name__) @@ -23,15 +19,15 @@ class Notifo(Notification): if self.isDisabled(): return try: - data = urllib.urlencode({ + params = { 'msg': toUnicode(message), - }) + } - req = urllib2.Request(self.url) - authHeader = "Basic %s" % base64.encodestring('%s:%s' % (self.conf('username'), self.conf('api_key')))[:-1] - req.add_header("Authorization", authHeader) + headers = { + 'Authorization': "Basic %s" % base64.encodestring('%s:%s' % (self.conf('username'), self.conf('api_key')))[:-1] + } - handle = urllib2.urlopen(req, data) + handle = self.urlopen(self.url, params = params, headers = headers) result = json.load(handle) if result['status'] != 'success' or result['response_message'] != 'OK': diff --git a/couchpotato/core/notifications/plex/main.py b/couchpotato/core/notifications/plex/main.py index 79e6dd3..c73e985 100644 --- a/couchpotato/core/notifications/plex/main.py +++ b/couchpotato/core/notifications/plex/main.py @@ -1,9 +1,6 @@ -from couchpotato.api import addApiView -from couchpotato.core.event import addEvent from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification from xml.dom import minidom -import urllib log = CPLog(__name__) @@ -23,12 +20,12 @@ class Plex(Notification): refresh_url = '%s/%%s/refresh' % base_url try: - xml_sections = minidom.parse(urllib.urlopen(base_url)) + xml_sections = minidom.parse(self.urlopen(base_url)) sections = xml_sections.getElementsByTagName('Directory') for s in sections: if s.getAttribute('type') in source_type: url = refresh_url % s.getAttribute('key') - x = urllib.urlopen(url) + x = self.urlopen(url) except: log.error('Plex library update failed for %s.' % host) diff --git a/couchpotato/core/notifications/xbmc/main.py b/couchpotato/core/notifications/xbmc/main.py index c8e2f70..0149fc5 100644 --- a/couchpotato/core/notifications/xbmc/main.py +++ b/couchpotato/core/notifications/xbmc/main.py @@ -2,7 +2,6 @@ from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification import base64 import urllib -import urllib2 log = CPLog(__name__) @@ -22,15 +21,14 @@ class XBMC(Notification): url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, urllib.urlencode(command)) - try: - req = urllib2.Request(url) - if self.password: - authHeader = "Basic %s" % base64.encodestring('%s:%s' % (self.conf('username'), self.conf('password')))[:-1] - req.add_header("Authorization", authHeader) + headers = {} + if self.password: + headers = { + 'Authorization': "Basic %s" % base64.encodestring('%s:%s' % (self.conf('username'), self.conf('password')))[:-1] + } - urllib2.urlopen(req, timeout = 10).read() - except Exception, e: - log.error("Couldn't sent command to XBMC. %s" % e) + if not self.urlopen(url, headers = headers): + log.error("Couldn't sent command to XBMC") return False log.info('XBMC notification to %s successful.' % host) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index b70c749..ec84c2c 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -5,8 +5,14 @@ from couchpotato.core.logger import CPLog from couchpotato.environment import Env from flask.helpers import send_from_directory import glob +import math import os.path import re +import socket +import time +import urllib +import urllib2 +import urlparse log = CPLog(__name__) @@ -19,6 +25,9 @@ class Plugin(object): needs_shutdown = False running = [] + http_last_use = {} + http_time_between_calls = 0 + def registerPlugin(self): addEvent('app.shutdown', self.doShutdown) addEvent('plugin.running', self.isRunning) @@ -67,6 +76,40 @@ class Plugin(object): except Exception, e: log.error('Unable to create folder "%s": %s' % (path, e)) + # http request + def urlopen(self, url, timeout = 10, params = {}, headers = {}): + + socket.setdefaulttimeout(timeout) + + host = urlparse(url).hostname + self.wait(host) + + try: + log.info('Opening url: %s, params: %s' % (url, params)) + + data = urllib.urlencode(params) if len(params) > 0 else None + request = urllib2.Request(url, data, headers) + + data = urllib2.urlopen(request).read() + except IOError, e: + log.error('Failed opening url, %s: %s' % (url, e)) + data = '' + + self.http_last_use[host] = time.time() + + return data + + def wait(self, host = ''): + now = time.time() + + last_use = self.http_last_use.get(host, 0) + + wait = math.ceil(last_use - now + self.http_time_between_calls) + + if wait > 0: + log.debug('Waiting for %s, %d seconds' % (self.getName(), wait)) + time.sleep(last_use - now + self.http_time_between_calls) + def beforeCall(self, handler): log.debug('Calling %s.%s' % (self.getName(), handler.__name__)) self.isRunning('%s.%s' % (self.getName(), handler.__name__)) diff --git a/couchpotato/core/plugins/file/main.py b/couchpotato/core/plugins/file/main.py index 5a9c8a2..3cbc369 100644 --- a/couchpotato/core/plugins/file/main.py +++ b/couchpotato/core/plugins/file/main.py @@ -34,26 +34,24 @@ class FileManager(Plugin): def download(self, url = '', dest = None, overwrite = False): - try: - file = urllib2.urlopen(url) + file = self.urlopen(url) + if not file: + log.error('File is empty, don\'t download') + return False - if not dest: # to Cache - dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) + if not dest: # to Cache + dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) - if overwrite or not os.path.exists(dest): - log.debug('Writing file to: %s' % dest) - output = open(dest, 'wb') - output.write(file.read()) - output.close() - else: - log.debug('File already exists: %s' % dest) + if overwrite or not os.path.exists(dest): + log.debug('Writing file to: %s' % dest) + output = open(dest, 'wb') + output.write(file) + output.close() + else: + log.debug('File already exists: %s' % dest) - return dest + return dest - except Exception: - log.error('Unable to download file "%s": %s' % (url, traceback.format_exc())) - - return False def add(self, path = '', part = 1, type = (), available = 1, properties = {}): db = get_session() diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index aadaca1..df27865 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -4,12 +4,8 @@ from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env from urllib2 import URLError from urlparse import urlparse -import math import re -import socket import time -import urllib -import urllib2 log = CPLog(__name__) @@ -17,9 +13,8 @@ log = CPLog(__name__) class Provider(Plugin): type = None # movie, nzb, torrent, subtitle, trailer - time_between_searches = 10 # Default timeout for url requests + http_time_between_calls = 10 # Default timeout for url requests - last_use = {} last_available_check = {} is_available = {} @@ -42,45 +37,16 @@ class Provider(Plugin): if self.last_available_check.get(host) < now - 900: self.last_available_check[host] = now - try: - self.urlopen(test_url, 30) - self.is_available[host] = True - except (IOError, URLError): + + data = self.urlopen(test_url, 30) + if not data: log.error('%s unavailable, trying again in an 15 minutes.' % self.name) self.is_available[host] = False + else: + self.is_available[host] = True return self.is_available[host] - def urlopen(self, url, timeout = 10, params = {}): - - socket.setdefaulttimeout(timeout) - - host = urlparse(url).hostname - self.wait(host) - - try: - log.info('Opening url: %s, params: %s' % (url, params)) - request = urllib2.Request(url, urllib.urlencode(params)) - data = urllib2.urlopen(request).read() - except IOError, e: - log.error('Failed opening url, %s: %s' % (url, e)) - data = '' - - self.last_use[host] = time.time() - - return data - - def wait(self, host = ''): - now = time.time() - - last_use = self.last_use.get(host, 0) - - wait = math.ceil(last_use - now + self.time_between_searches) - - if wait > 0: - log.debug('Waiting for %s, %d seconds' % (self.getName(), wait)) - time.sleep(last_use - now + self.time_between_searches) - class MovieProvider(Provider): type = 'movie' @@ -129,8 +95,6 @@ class YarrProvider(Provider): class NZBProvider(YarrProvider): type = 'nzb' - time_between_searches = 10 # Seconds - def __init__(self): addEvent('provider.nzb.search', self.search) addEvent('provider.yarr.search', self.search) diff --git a/couchpotato/core/providers/nzb/newzbin/main.py b/couchpotato/core/providers/nzb/newzbin/main.py index e1651f7..8933be9 100644 --- a/couchpotato/core/providers/nzb/newzbin/main.py +++ b/couchpotato/core/providers/nzb/newzbin/main.py @@ -1,16 +1,10 @@ - -from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.event import fireEvent from couchpotato.core.helpers.rss import RSS from couchpotato.core.logger import CPLog from couchpotato.core.providers.base import NZBProvider -from couchpotato.environment import Env from dateutil.parser import parse from urllib import urlencode -from urllib2 import URLError -import httplib import time -import traceback -import urllib import xml.etree.ElementTree as XMLTree log = CPLog(__name__) @@ -66,14 +60,15 @@ class Newzbin(NZBProvider, RSS): cache_key = str('newzbin.%s.%s.%s' % (movie['library']['identifier'], str(format_id), str(cat_id))) single_cat = True - try: - data = self.getCache(cache_key) + data = self.getCache(cache_key) + if not data: + data = self.urlopen(url, params = {'username': self.conf('username'), 'password': self.conf('password')}) + self.setCache(cache_key, data) + if not data: - data = self.urlopen(url, params = {'username': self.conf('username'), 'password': self.conf('password')}) - self.setCache(cache_key, data) - except (IOError, URLError): - log.error('Failed to open %s.' % url) - return results + log.error('Failed to get data from %s.' % url) + return results + if data: try: diff --git a/couchpotato/core/providers/nzb/newznab/main.py b/couchpotato/core/providers/nzb/newznab/main.py index b495fe2..17ac383 100644 --- a/couchpotato/core/providers/nzb/newznab/main.py +++ b/couchpotato/core/providers/nzb/newznab/main.py @@ -1,12 +1,10 @@ -from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.event import fireEvent from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog from couchpotato.core.providers.base import NZBProvider -from couchpotato.environment import Env from dateutil.parser import parse from urllib import urlencode -from urllib2 import URLError import time import xml.etree.ElementTree as XMLTree @@ -102,14 +100,14 @@ class Newznab(NZBProvider, RSS): def createItems(self, url, cache_key, host, single_cat = False, movie = None, quality = None, for_feed = False): results = [] - try: - data = self.getCache(cache_key) + data = self.getCache(cache_key) + if not data: + data = self.urlopen(url) + self.setCache(cache_key, data) + if not data: - data = self.urlopen(url) - self.setCache(cache_key, data) - except (IOError, URLError): - log.error('Failed to open %s.' % url) - return results + log.error('Failed to get data from %s.' % url) + return results if data: try: diff --git a/couchpotato/core/providers/nzb/nzbindex/main.py b/couchpotato/core/providers/nzb/nzbindex/main.py index 2c59864..a1b377c 100644 --- a/couchpotato/core/providers/nzb/nzbindex/main.py +++ b/couchpotato/core/providers/nzb/nzbindex/main.py @@ -1,11 +1,10 @@ -from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.event import fireEvent from couchpotato.core.helpers.encoding import simplifyString from couchpotato.core.helpers.rss import RSS from couchpotato.core.logger import CPLog from couchpotato.core.providers.base import NZBProvider from dateutil.parser import parse from urllib import urlencode -from urllib2 import URLError import time import xml.etree.ElementTree as XMLTree @@ -38,14 +37,14 @@ class NzbIndex(NZBProvider, RSS): cache_key = 'nzbindex.%s.%s' % (movie['library'].get('identifier'), quality.get('identifier')) - try: - data = self.getCache(cache_key) + data = self.getCache(cache_key) + if not data: + data = self.urlopen(url) + self.setCache(cache_key, data) + if not data: - data = self.urlopen(url) - self.setCache(cache_key, data) - except (IOError, URLError): - log.error('Failed to open %s.' % url) - return results + log.error('Failed to get data from %s.' % url) + return results if data: try: diff --git a/couchpotato/core/providers/nzb/nzbmatrix/main.py b/couchpotato/core/providers/nzb/nzbmatrix/main.py index 23629f0..1104915 100644 --- a/couchpotato/core/providers/nzb/nzbmatrix/main.py +++ b/couchpotato/core/providers/nzb/nzbmatrix/main.py @@ -1,11 +1,10 @@ -from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.event import fireEvent from couchpotato.core.helpers.rss import RSS from couchpotato.core.logger import CPLog from couchpotato.core.providers.base import NZBProvider from couchpotato.environment import Env from dateutil.parser import parse from urllib import urlencode -from urllib2 import URLError import time import xml.etree.ElementTree as XMLTree @@ -52,14 +51,14 @@ class NZBMatrix(NZBProvider, RSS): cache_key = 'nzbmatrix.%s.%s' % (movie['library'].get('identifier'), cat_ids) single_cat = True - try: - data = self.getCache(cache_key) + data = self.getCache(cache_key) + if not data: + data = self.urlopen(url) + self.setCache(cache_key, data) + if not data: - data = self.urlopen(url) - self.setCache(cache_key, data) - except (IOError, URLError): - log.error('Failed to open %s.' % url) - return results + log.error('Failed to get data from %s.' % url) + return results if data: try: diff --git a/couchpotato/core/providers/nzb/nzbs/main.py b/couchpotato/core/providers/nzb/nzbs/main.py index b06b564..8648932 100644 --- a/couchpotato/core/providers/nzb/nzbs/main.py +++ b/couchpotato/core/providers/nzb/nzbs/main.py @@ -1,11 +1,10 @@ -from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.event import fireEvent from couchpotato.core.helpers.encoding import simplifyString from couchpotato.core.helpers.rss import RSS from couchpotato.core.logger import CPLog from couchpotato.core.providers.base import NZBProvider from dateutil.parser import parse from urllib import urlencode -from urllib2 import URLError import time import xml.etree.ElementTree as XMLTree @@ -48,14 +47,14 @@ class Nzbs(NZBProvider, RSS): cache_key = 'nzbs.%s.%s' % (movie['library'].get('identifier'), str(cat_id)) - try: - data = self.getCache(cache_key) + data = self.getCache(cache_key) + if not data: + data = self.urlopen(url) + self.setCache(cache_key, data) + if not data: - data = self.urlopen(url) - self.setCache(cache_key, data) - except (IOError, URLError): - log.error('Failed to open %s.' % url) - return results + log.error('Failed to get data from %s.' % url) + return results if data: try: diff --git a/couchpotato/core/providers/torrent/thepiratebay/main.py b/couchpotato/core/providers/torrent/thepiratebay/main.py index c2e5636..57728b6 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/main.py +++ b/couchpotato/core/providers/torrent/thepiratebay/main.py @@ -38,10 +38,9 @@ class ThePirateBay(TorrentProvider): log.info('Searching: %s' % url) - try: - data = urllib2.urlopen(url, timeout = self.timeout).read() - except (IOError, URLError): - log.error('Failed to open %s.' % url) + data = self.urlopen(url) + if not data: + log.error('Failed to get data from %s.' % url) return results try: @@ -129,11 +128,10 @@ class ThePirateBay(TorrentProvider): def getInfo(self, url): log.debug('Getting info: %s' % url) - try: - data = urllib2.urlopen(url, timeout = self.timeout).read() - pass - except IOError: - log.error('Failed to open %s.' % url) + + data = self.urlopen(url) + if not data: + log.error('Failed to get data from %s.' % url) return '' div = SoupStrainer('div')