|
|
|
@ -1,19 +1,15 @@ |
|
|
|
from StringIO import StringIO |
|
|
|
from couchpotato.core.event import fireEvent, addEvent |
|
|
|
from couchpotato.core.helpers.encoding import tryUrlencode, ss, toSafeString, \ |
|
|
|
from couchpotato.core.helpers.encoding import ss, toSafeString, \ |
|
|
|
toUnicode, sp |
|
|
|
from couchpotato.core.helpers.variable import getExt, md5, isLocalIP |
|
|
|
from couchpotato.core.logger import CPLog |
|
|
|
from couchpotato.environment import Env |
|
|
|
from multipartpost import MultipartPostHandler |
|
|
|
import requests |
|
|
|
from tornado import template |
|
|
|
from tornado.web import StaticFileHandler |
|
|
|
from urlparse import urlparse |
|
|
|
import cookielib |
|
|
|
import glob |
|
|
|
import gzip |
|
|
|
import inspect |
|
|
|
import math |
|
|
|
import os.path |
|
|
|
import re |
|
|
|
import time |
|
|
|
@ -39,7 +35,7 @@ class Plugin(object): |
|
|
|
http_time_between_calls = 0 |
|
|
|
http_failed_request = {} |
|
|
|
http_failed_disabled = {} |
|
|
|
http_opener = None |
|
|
|
http_opener = requests.Session() |
|
|
|
|
|
|
|
def __new__(typ, *args, **kwargs): |
|
|
|
new_plugin = super(Plugin, typ).__new__(typ) |
|
|
|
@ -121,11 +117,11 @@ class Plugin(object): |
|
|
|
return False |
|
|
|
|
|
|
|
# http request |
|
|
|
def urlopen(self, url, timeout = 30, params = None, headers = None, opener = None, multipart = False, show_error = True): |
|
|
|
def urlopen(self, url, timeout = 30, data = None, headers = None, files = None, show_error = True): |
|
|
|
url = urllib2.quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]") |
|
|
|
|
|
|
|
if not headers: headers = {} |
|
|
|
if not params: params = {} |
|
|
|
if not data: data = {} |
|
|
|
|
|
|
|
# Fill in some headers |
|
|
|
parsed_url = urlparse(url) |
|
|
|
@ -138,13 +134,7 @@ class Plugin(object): |
|
|
|
headers['Connection'] = headers.get('Connection', 'keep-alive') |
|
|
|
headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0') |
|
|
|
|
|
|
|
# Create opener |
|
|
|
if not opener: |
|
|
|
if not self.http_opener: |
|
|
|
cookies = cookielib.CookieJar() |
|
|
|
self.http_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies)) |
|
|
|
|
|
|
|
opener = self.http_opener |
|
|
|
r = self.http_opener |
|
|
|
|
|
|
|
# Don't try for failed requests |
|
|
|
if self.http_failed_disabled.get(host, 0) > 0: |
|
|
|
@ -161,37 +151,18 @@ class Plugin(object): |
|
|
|
self.wait(host) |
|
|
|
try: |
|
|
|
|
|
|
|
# Make sure opener has the correct headers |
|
|
|
opener.add_headers = headers |
|
|
|
|
|
|
|
if multipart: |
|
|
|
log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data')) |
|
|
|
request = urllib2.Request(url, params, headers) |
|
|
|
kwargs = { |
|
|
|
'headers': headers, |
|
|
|
'data': data if len(data) > 0 else None, |
|
|
|
'timeout': timeout, |
|
|
|
'files': files, |
|
|
|
} |
|
|
|
method = 'post' if len(data) > 0 or files else 'get' |
|
|
|
|
|
|
|
opener.add_handler(MultipartPostHandler()) |
|
|
|
log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.iterkeys()] if isinstance(data, dict) else 'with data')) |
|
|
|
response = r.request(method, url, **kwargs) |
|
|
|
|
|
|
|
response = opener.open(request, timeout = timeout) |
|
|
|
else: |
|
|
|
log.info('Opening url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data')) |
|
|
|
|
|
|
|
if isinstance(params, (str, unicode)) and len(params) > 0: |
|
|
|
data = params |
|
|
|
else: |
|
|
|
data = tryUrlencode(params) if len(params) > 0 else None |
|
|
|
|
|
|
|
request = urllib2.Request(url, data, headers) |
|
|
|
|
|
|
|
response = opener.open(request, timeout = timeout) |
|
|
|
|
|
|
|
# unzip if needed |
|
|
|
if response.info().get('Content-Encoding') == 'gzip': |
|
|
|
buf = StringIO(response.read()) |
|
|
|
f = gzip.GzipFile(fileobj = buf) |
|
|
|
data = f.read() |
|
|
|
f.close() |
|
|
|
else: |
|
|
|
data = response.read() |
|
|
|
response.close() |
|
|
|
data = response.text |
|
|
|
|
|
|
|
self.http_failed_request[host] = 0 |
|
|
|
except IOError: |
|
|
|
@ -274,18 +245,19 @@ class Plugin(object): |
|
|
|
try: |
|
|
|
|
|
|
|
cache_timeout = 300 |
|
|
|
if kwargs.get('cache_timeout'): |
|
|
|
if kwargs.has_key('cache_timeout'): |
|
|
|
cache_timeout = kwargs.get('cache_timeout') |
|
|
|
del kwargs['cache_timeout'] |
|
|
|
|
|
|
|
data = self.urlopen(url, **kwargs) |
|
|
|
if data: |
|
|
|
if data and cache_timeout > 0: |
|
|
|
self.setCache(cache_key, data, timeout = cache_timeout) |
|
|
|
return data |
|
|
|
except: |
|
|
|
if not kwargs.get('show_error', True): |
|
|
|
raise |
|
|
|
|
|
|
|
log.error('Failed getting cache: %s', (traceback.format_exc())) |
|
|
|
return '' |
|
|
|
|
|
|
|
def setCache(self, cache_key, value, timeout = 300): |
|
|
|
|
Ruud
11 years ago