diff --git a/couchpotato/core/event.py b/couchpotato/core/event.py index 5ec80dc..8365ae5 100644 --- a/couchpotato/core/event.py +++ b/couchpotato/core/event.py @@ -105,14 +105,14 @@ def fireEvent(name, *args, **kwargs): # Merge if options['merge'] and len(results) > 0: # Dict - if type(results[0]) == dict: + if isinstance(results[0], dict): merged = {} for result in results: merged = mergeDicts(merged, result) results = merged # Lists - elif type(results[0]) == list: + elif isinstance(results[0], list): merged = [] for result in results: merged += result diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index 4fe5edd..805affc 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -363,7 +363,7 @@ class Searcher(Plugin): return True # Check if nzb contains imdb link - if self.checkIMDB([nzb['description']], movie['library']['identifier']): + if self.checkIMDB([nzb.get('description', '')], movie['library']['identifier']): return True for raw_title in movie['library']['titles']: diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index 9cbbdb5..7885763 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -1,15 +1,16 @@ -from couchpotato.core.event import addEvent -from couchpotato.core.helpers.encoding import simplifyString -from couchpotato.core.helpers.variable import tryFloat, getTitle +from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.helpers.variable import tryFloat, mergeDicts, md5 from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env from urlparse import urlparse import cookielib +import json import re import time import traceback import urllib2 +import xml.etree.ElementTree as XMLTree log = CPLog(__name__) @@ -59,8 +60,6 @@ class YarrProvider(Provider): addEvent('%s.search' % self.type, self.search) addEvent('yarr.search', self.search) - addEvent('nzb.feed', self.feed) - def login(self): try: @@ -97,9 +96,6 @@ class YarrProvider(Provider): return 'try_next' - def feed(self): - return [] - def search(self, movie, quality): return [] @@ -149,22 +145,93 @@ class YarrProvider(Provider): return [self.cat_backup_id] - def found(self, new): - if not new.get('provider_extra'): - new['provider_extra'] = '' - else: - new['provider_extra'] = ', %s' % new['provider_extra'] + def getJsonData(self, url, **kwargs): + + data = self.getCache(md5(url), url, **kwargs) + + if data: + try: + return json.loads(data) + except: + log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) + + return [] + + def getRSSData(self, url, **kwargs): + + data = self.getCache(md5(url), url, **kwargs) + + if data: + try: + data = XMLTree.fromstring(data) + return self.getElements(data, 'channel/item') + except: + log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) + + return [] + + def getHTMLData(self, url, **kwargs): + return self.getCache(md5(url), url, **kwargs) + + +class ResultList(list): + + result_ids = None + provider = None + movie = None + quality = None + + def __init__(self, provider, movie, quality, **kwargs): + + self.result_ids = [] + self.provider = provider + self.movie = movie + self.quality = quality + self.kwargs = kwargs + + super(ResultList, self).__init__() - log.info('Found: score(%(score)s) on %(provider)s%(provider_extra)s: %(name)s', new) + def extend(self, results): + for r in results: + self.append(r) - def removeDuplicateResults(self, results): + def append(self, result): - result_ids = [] - new_results = [] + new_result = self.fillResult(result) - for result in results: - if result['id'] not in result_ids: - new_results.append(result) - result_ids.append(result['id']) + is_correct_movie = fireEvent('searcher.correct_movie', + nzb = new_result, movie = self.movie, quality = self.quality, + imdb_results = self.kwargs.get('imdb_results', False), single = True) + + if is_correct_movie and new_result['id'] not in self.result_ids: + new_result['score'] += fireEvent('score.calculate', new_result, self.movie, single = True) + + self.found(new_result) + self.result_ids.append(result['id']) + + super(ResultList, self).append(new_result) + + def fillResult(self, result): + + defaults = { + 'id': 0, + 'type': self.provider.type, + 'provider': self.provider.getName(), + 'download': self.provider.download, + 'url': '', + 'name': '', + 'age': 0, + 'size': 0, + 'description': '', + 'score': 0 + } + + return mergeDicts(defaults, result) + + def found(self, new_result): + if not new_result.get('provider_extra'): + new_result['provider_extra'] = '' + else: + new_result['provider_extra'] = ', %s' % new_result['provider_extra'] - return new_results + log.info('Found: score(%(score)s) on %(provider)s%(provider_extra)s: %(name)s', new_result) diff --git a/couchpotato/core/providers/nzb/ftdworld/main.py b/couchpotato/core/providers/nzb/ftdworld/main.py index e684e21..8e85023 100644 --- a/couchpotato/core/providers/nzb/ftdworld/main.py +++ b/couchpotato/core/providers/nzb/ftdworld/main.py @@ -1,8 +1,8 @@ from bs4 import BeautifulSoup -from couchpotato.core.event import fireEvent from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode from couchpotato.core.helpers.variable import tryInt, possibleTitles, getTitle from couchpotato.core.logger import CPLog +from couchpotato.core.providers.base import ResultList from couchpotato.core.providers.nzb.base import NZBProvider from couchpotato.environment import Env from dateutil.parser import parse @@ -35,18 +35,17 @@ class FTDWorld(NZBProvider): if self.isDisabled(): return [] - results = [] + results = ResultList(self, movie, quality) for title in possibleTitles(getTitle(movie['library'])): results.extend(self._search(title, movie, quality)) - return self.removeDuplicateResults(results) + return results def _search(self, title, movie, quality): - results = [] q = '"%s" %s' % (title, movie['library']['year']) - params = { + params = tryUrlencode({ 'ctitle': q, 'customQuery': 'usr', 'cage': Env.setting('retention', 'nzb'), @@ -54,10 +53,9 @@ class FTDWorld(NZBProvider): 'csizemax': quality.get('size_max'), 'ccategory': 14, 'ctype': ','.join([str(x) for x in self.getCatId(quality['identifier'])]), - } + }) - cache_key = 'ftdworld.%s.%s' % (movie['library']['identifier'], q) - data = self.getCache(cache_key, self.urls['search'] % tryUrlencode(params), opener = self.login_opener) + data = self.getHTMLData(self.urls['search'] % params, opener = self.login_opener) if data: try: @@ -66,8 +64,9 @@ class FTDWorld(NZBProvider): main_table = html.find('table', attrs = {'id':'ftdresult'}) if not main_table: - return results + return [] + results = ResultList(self, movie, quality) items = main_table.find_all('tr', attrs = {'class': re.compile('tcontent')}) for item in items: @@ -77,32 +76,20 @@ class FTDWorld(NZBProvider): up = item.find('img', attrs = {'src': re.compile('up.png')}) down = item.find('img', attrs = {'src': re.compile('down.png')}) - new = { + results.append({ 'id': nzb_id, - 'type': 'nzb', - 'provider': self.getName(), 'name': toUnicode(item.find('a', attrs = {'href': re.compile('./spotinfo')}).text.strip()), 'age': self.calculateAge(int(time.mktime(parse(tds[2].text).timetuple()))), - 'size': 0, 'url': self.urls['download'] % nzb_id, 'download': self.loginDownload, 'detail_url': self.urls['detail'] % nzb_id, - 'description': '', 'score': (tryInt(up.attrs['title'].split(' ')[0]) * 3) - (tryInt(down.attrs['title'].split(' ')[0]) * 3) if up else 0, - } - - is_correct_movie = fireEvent('searcher.correct_movie', - nzb = new, movie = movie, quality = quality, - imdb_results = False, single = True) - - if is_correct_movie: - new['score'] += fireEvent('score.calculate', new, movie, single = True) - results.append(new) - self.found(new) + }) return results - except SyntaxError: - log.error('Failed to parse XML response from NZBClub') + + except: + log.error('Failed to parse HTML response from FTDWorld') return results diff --git a/couchpotato/core/providers/nzb/newznab/main.py b/couchpotato/core/providers/nzb/newznab/main.py index 539d6fa..4e0bd92 100644 --- a/couchpotato/core/providers/nzb/newznab/main.py +++ b/couchpotato/core/providers/nzb/newznab/main.py @@ -1,8 +1,8 @@ -from couchpotato.core.event import fireEvent from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import cleanHost, splitString from couchpotato.core.logger import CPLog +from couchpotato.core.providers.base import ResultList from couchpotato.core.providers.nzb.base import NZBProvider from couchpotato.environment import Env from dateutil.parser import parse @@ -10,7 +10,6 @@ from urllib2 import HTTPError from urlparse import urlparse import time import traceback -import xml.etree.ElementTree as XMLTree log = CPLog(__name__) @@ -29,38 +28,6 @@ class Newznab(NZBProvider, RSS): http_time_between_calls = 1 # Seconds - def feed(self): - - hosts = self.getHosts() - - results = [] - for host in hosts: - result = self.singleFeed(host) - - if result: - results.extend(result) - - return results - - def singleFeed(self, host): - - results = [] - if self.isDisabled(host): - return results - - arguments = tryUrlencode({ - 't': self.cat_backup_id, - 'r': host['api_key'], - 'i': 58, - }) - url = '%s?%s' % (cleanHost(host['host']) + 'rss', arguments) - cache_key = 'newznab.%s.feed.%s' % (host['host'], arguments) - - results = self.createItems(url, cache_key, host, for_feed = True) - - return results - - def search(self, movie, quality): hosts = self.getHosts() @@ -76,9 +43,10 @@ class Newznab(NZBProvider, RSS): def singleSearch(self, host, movie, quality): - results = [] if self.isDisabled(host): - return results + return [] + + results = ResultList(self, movie, quality, imdb_result = True) cat_id = self.getCatId(quality['identifier']) arguments = tryUrlencode({ @@ -89,73 +57,35 @@ class Newznab(NZBProvider, RSS): }) url = '%s&%s' % (self.getUrl(host['host'], self.urls['search']), arguments) - cache_key = 'newznab.%s.%s.%s' % (host['host'], movie['library']['identifier'], cat_id) + nzbs = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()}) - results = self.createItems(url, cache_key, host, movie = movie, quality = quality) + for nzb in nzbs: - return results + date = None + for item in nzb: + if item.attrib.get('name') == 'usenetdate': + date = item.attrib.get('value') + break - def createItems(self, url, cache_key, host, movie = None, quality = None, for_feed = False): - results = [] + if not date: + date = self.getTextElement(nzb, 'pubDate') - data = self.getCache(cache_key, url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()}) - if data: - try: - try: - data = XMLTree.fromstring(data) - nzbs = self.getElements(data, 'channel/item') - except Exception, e: - log.debug('%s, %s', (self.getName(), e)) - return results - - results = [] - for nzb in nzbs: - - date = None - for item in nzb: - if item.attrib.get('name') == 'usenetdate': - date = item.attrib.get('value') - break - - if not date: - date = self.getTextElement(nzb, 'pubDate') - - nzb_id = self.getTextElement(nzb, 'guid').split('/')[-1:].pop() - name = self.getTextElement(nzb, 'title') - - if not name: - continue - - new = { - 'id': nzb_id, - 'provider': self.getName(), - 'provider_extra': host['host'], - 'type': 'nzb', - 'name': self.getTextElement(nzb, 'title'), - 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))), - 'size': int(self.getElement(nzb, 'enclosure').attrib['length']) / 1024 / 1024, - 'url': (self.getUrl(host['host'], self.urls['download']) % nzb_id) + self.getApiExt(host), - 'download': self.download, - 'detail_url': '%sdetails/%s' % (cleanHost(host['host']), nzb_id), - 'content': self.getTextElement(nzb, 'description'), - } - - if not for_feed: - is_correct_movie = fireEvent('searcher.correct_movie', - nzb = new, movie = movie, quality = quality, - imdb_results = True, single = True) - - if is_correct_movie: - new['score'] = fireEvent('score.calculate', new, movie, single = True) - results.append(new) - self.found(new) - else: - results.append(new) - - return results - except SyntaxError: - log.error('Failed to parse XML response from Newznab: %s', host) - return results + nzb_id = self.getTextElement(nzb, 'guid').split('/')[-1:].pop() + name = self.getTextElement(nzb, 'title') + + if not name: + continue + + results.append({ + 'id': nzb_id, + 'provider_extra': host['host'], + 'name': self.getTextElement(nzb, 'title'), + 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))), + 'size': int(self.getElement(nzb, 'enclosure').attrib['length']) / 1024 / 1024, + 'url': (self.getUrl(host['host'], self.urls['download']) % nzb_id) + self.getApiExt(host), + 'detail_url': '%sdetails/%s' % (cleanHost(host['host']), nzb_id), + 'content': self.getTextElement(nzb, 'description'), + }) def getHosts(self): diff --git a/couchpotato/core/providers/nzb/nzbclub/main.py b/couchpotato/core/providers/nzb/nzbclub/main.py index 487b9a6..ed5b460 100644 --- a/couchpotato/core/providers/nzb/nzbclub/main.py +++ b/couchpotato/core/providers/nzb/nzbclub/main.py @@ -1,13 +1,12 @@ from bs4 import BeautifulSoup -from couchpotato.core.event import fireEvent from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import tryInt, getTitle, possibleTitles from couchpotato.core.logger import CPLog +from couchpotato.core.providers.base import ResultList from couchpotato.core.providers.nzb.base import NZBProvider from dateutil.parser import parse import time -import xml.etree.ElementTree as XMLTree log = CPLog(__name__) @@ -25,81 +24,56 @@ class NZBClub(NZBProvider, RSS): if self.isDisabled(): return [] - results = [] + results = ResultList(self, movie, quality) + for title in possibleTitles(getTitle(movie['library'])): results.extend(self._search(title, movie, quality)) - return self.removeDuplicateResults(results) + return results def _search(self, title, movie, quality): results = [] q = '"%s %s" %s' % (title, movie['library']['year'], quality.get('identifier')) - params = { + params = tryUrlencode({ 'q': q, 'ig': '1', 'rpp': 200, 'st': 1, 'sp': 1, 'ns': 1, - } - - cache_key = 'nzbclub.%s.%s.%s' % (movie['library']['identifier'], quality.get('identifier'), q) - data = self.getCache(cache_key, self.urls['search'] % tryUrlencode(params)) - if data: - try: - try: - data = XMLTree.fromstring(data) - nzbs = self.getElements(data, 'channel/item') - except Exception, e: - log.debug('%s, %s', (self.getName(), e)) - return results - - for nzb in nzbs: - - nzbclub_id = tryInt(self.getTextElement(nzb, "link").split('/nzb_view/')[1].split('/')[0]) - enclosure = self.getElement(nzb, "enclosure").attrib - size = enclosure['length'] - date = self.getTextElement(nzb, "pubDate") - - def extra_check(item): - full_description = self.getCache('nzbclub.%s' % nzbclub_id, item['detail_url'], cache_timeout = 25920000) - - for ignored in ['ARCHIVE inside ARCHIVE', 'Incomplete', 'repair impossible']: - if ignored in full_description: - log.info('Wrong: Seems to be passworded or corrupted files: %s', new['name']) - return False - - return True - - new = { - 'id': nzbclub_id, - 'type': 'nzb', - 'provider': self.getName(), - 'name': toUnicode(self.getTextElement(nzb, "title")), - 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))), - 'size': tryInt(size) / 1024 / 1024, - 'url': enclosure['url'].replace(' ', '_'), - 'download': self.download, - 'detail_url': self.getTextElement(nzb, "link"), - 'description': '', - 'get_more_info': self.getMoreInfo, - 'extra_check': extra_check - } - - is_correct_movie = fireEvent('searcher.correct_movie', - nzb = new, movie = movie, quality = quality, - imdb_results = False, single = True) - - if is_correct_movie: - new['score'] = fireEvent('score.calculate', new, movie, single = True) - results.append(new) - self.found(new) - - return results - except SyntaxError: - log.error('Failed to parse XML response from NZBClub') + }) + + nzbs = self.getRSSData(self.urls['search'] % params) + + for nzb in nzbs: + + nzbclub_id = tryInt(self.getTextElement(nzb, "link").split('/nzb_view/')[1].split('/')[0]) + enclosure = self.getElement(nzb, "enclosure").attrib + size = enclosure['length'] + date = self.getTextElement(nzb, "pubDate") + + def extra_check(item): + full_description = self.getCache('nzbclub.%s' % nzbclub_id, item['detail_url'], cache_timeout = 25920000) + + for ignored in ['ARCHIVE inside ARCHIVE', 'Incomplete', 'repair impossible']: + if ignored in full_description: + log.info('Wrong: Seems to be passworded or corrupted files: %s', item['name']) + return False + + return True + + results.append({ + 'id': nzbclub_id, + 'name': toUnicode(self.getTextElement(nzb, "title")), + 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))), + 'size': tryInt(size) / 1024 / 1024, + 'url': enclosure['url'].replace(' ', '_'), + 'detail_url': self.getTextElement(nzb, "link"), + 'get_more_info': self.getMoreInfo, + 'extra_check': extra_check + }) return results diff --git a/couchpotato/core/providers/nzb/nzbindex/main.py b/couchpotato/core/providers/nzb/nzbindex/main.py index f4f1d6b..63cf30f 100644 --- a/couchpotato/core/providers/nzb/nzbindex/main.py +++ b/couchpotato/core/providers/nzb/nzbindex/main.py @@ -1,16 +1,14 @@ from bs4 import BeautifulSoup -from couchpotato.core.event import fireEvent from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import tryInt, getTitle, possibleTitles from couchpotato.core.logger import CPLog +from couchpotato.core.providers.base import ResultList from couchpotato.core.providers.nzb.base import NZBProvider from couchpotato.environment import Env from dateutil.parser import parse import re import time -import traceback -import xml.etree.ElementTree as XMLTree log = CPLog(__name__) @@ -19,7 +17,7 @@ class NzbIndex(NZBProvider, RSS): urls = { 'download': 'https://www.nzbindex.com/download/', - 'api': 'https://www.nzbindex.com/rss/', + 'search': 'https://www.nzbindex.com/rss/?%s', } http_time_between_calls = 1 # Seconds @@ -29,13 +27,14 @@ class NzbIndex(NZBProvider, RSS): if self.isDisabled(): return [] - results = [] + results = ResultList(self, movie, quality) for title in possibleTitles(getTitle(movie['library'])): results.extend(self._search(title, movie, quality)) - return self.removeDuplicateResults(results) + return results def _search(self, title, movie, quality): + results = [] q = '"%s" %s %s' % (title, movie['library']['year'], quality.get('identifier')) @@ -50,66 +49,37 @@ class NzbIndex(NZBProvider, RSS): 'more': 1, 'complete': 1, }) - url = "%s?%s" % (self.urls['api'], arguments) - cache_key = 'nzbindex.%s.%s' % (movie['library']['identifier'], q) - data = self.getCache(cache_key, url) + nzbs = self.getRSSData(self.urls['search'] % arguments) + + for nzb in nzbs: + + enclosure = self.getElement(nzb, 'enclosure').attrib + nzbindex_id = int(self.getTextElement(nzb, "link").split('/')[4]) - if data: try: - try: - data = XMLTree.fromstring(data) - nzbs = self.getElements(data, 'channel/item') - except Exception, e: - log.debug('%s, %s', (self.getName(), e)) - return results - - for nzb in nzbs: - - enclosure = self.getElement(nzb, 'enclosure').attrib - - nzbindex_id = int(self.getTextElement(nzb, "link").split('/')[4]) - - try: - description = self.getTextElement(nzb, "description") - except: - description = '' - - def extra_check(new): - if '#c20000' in new['description'].lower(): - log.info('Wrong: Seems to be passworded: %s', new['name']) - return False - - return True - - new = { - 'id': nzbindex_id, - 'type': 'nzb', - 'provider': self.getName(), - 'download': self.download, - 'name': self.getTextElement(nzb, "title"), - 'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, "pubDate")).timetuple()))), - 'size': tryInt(enclosure['length']) / 1024 / 1024, - 'url': enclosure['url'], - 'detail_url': enclosure['url'].replace('/download/', '/release/'), - 'description': description, - 'get_more_info': self.getMoreInfo, - 'extra_check': extra_check, - 'check_nzb': True, - } - - is_correct_movie = fireEvent('searcher.correct_movie', - nzb = new, movie = movie, quality = quality, - imdb_results = False, single = True) - - if is_correct_movie: - new['score'] = fireEvent('score.calculate', new, movie, single = True) - results.append(new) - self.found(new) - - return results + description = self.getTextElement(nzb, "description") except: - log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) + description = '' + + def extra_check(item): + if '#c20000' in item['description'].lower(): + log.info('Wrong: Seems to be passworded: %s', item['name']) + return False + + return True + + results.append({ + 'id': nzbindex_id, + 'name': self.getTextElement(nzb, "title"), + 'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, "pubDate")).timetuple()))), + 'size': tryInt(enclosure['length']) / 1024 / 1024, + 'url': enclosure['url'], + 'detail_url': enclosure['url'].replace('/download/', '/release/'), + 'description': description, + 'get_more_info': self.getMoreInfo, + 'extra_check': extra_check, + }) return results @@ -123,5 +93,3 @@ class NzbIndex(NZBProvider, RSS): except: pass - def isEnabled(self): - return NZBProvider.isEnabled(self) and self.conf('enabled') diff --git a/couchpotato/core/providers/nzb/nzbsrus/main.py b/couchpotato/core/providers/nzb/nzbsrus/main.py index ee2b223..0c655eb 100644 --- a/couchpotato/core/providers/nzb/nzbsrus/main.py +++ b/couchpotato/core/providers/nzb/nzbsrus/main.py @@ -1,11 +1,10 @@ -from couchpotato.core.event import fireEvent from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.rss import RSS from couchpotato.core.logger import CPLog +from couchpotato.core.providers.base import ResultList from couchpotato.core.providers.nzb.base import NZBProvider from couchpotato.environment import Env import time -import xml.etree.ElementTree as XMLTree log = CPLog(__name__) @@ -25,13 +24,12 @@ class Nzbsrus(NZBProvider, RSS): def search(self, movie, quality): - results = [] - if self.isDisabled(): - return results + return [] - cat_id_string = '&'.join(['c%s=1' % x for x in self.getCatId(quality.get('identifier'))]) + results = ResultList(self, movie, quality, imdb_result = True) + cat_id_string = '&'.join(['c%s=1' % x for x in self.getCatId(quality.get('identifier'))]) arguments = tryUrlencode({ 'searchtext': 'imdb:' + movie['library']['identifier'][2:], 'uid': self.conf('userid'), @@ -42,57 +40,29 @@ class Nzbsrus(NZBProvider, RSS): # check for english_only if self.conf('english_only'): - arguments += "&lang0=1&lang3=1&lang1=1" - - url = "%s&%s&%s" % (self.urls['search'], arguments , cat_id_string) - - cache_key = 'nzbsrus.%s.%s' % (movie['library'].get('identifier'), cat_id_string) - - data = self.getCache(cache_key, url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()}) - if data: - try: - try: - data = XMLTree.fromstring(data) - nzbs = self.getElements(data, 'results/result') - except Exception, e: - log.debug('%s, %s', (self.getName(), e)) - return results - - for nzb in nzbs: - - title = self.getTextElement(nzb, "name") - if 'error' in title.lower(): continue - - id = self.getTextElement(nzb, "id") - size = int(round(int(self.getTextElement(nzb, "size")) / 1048576)) - age = int(round((time.time() - int(self.getTextElement(nzb, "postdate"))) / 86400)) - - new = { - 'id': id, - 'type': 'nzb', - 'provider': self.getName(), - 'name': title, - 'age': age, - 'size': size, - 'url': self.urls['download'] % id + self.getApiExt() + self.getTextElement(nzb, "key"), - 'download': self.download, - 'detail_url': self.urls['detail'] % id, - 'description': self.getTextElement(nzb, "addtext"), - 'check_nzb': True, - } - - is_correct_movie = fireEvent('searcher.correct_movie', - nzb = new, movie = movie, quality = quality, - imdb_results = True, single = True) - - if is_correct_movie: - new['score'] = fireEvent('score.calculate', new, movie, single = True) - results.append(new) - self.found(new) - - return results - except SyntaxError: - log.error('Failed to parse XML response from Nzbsrus.com') + arguments += '&lang0=1&lang3=1&lang1=1' + + url = '%s&%s&%s' % (self.urls['search'], arguments , cat_id_string) + nzbs = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()}) + + for nzb in nzbs: + + title = self.getTextElement(nzb, 'name') + if 'error' in title.lower(): continue + + nzb_id = self.getTextElement(nzb, 'id') + size = int(round(int(self.getTextElement(nzb, 'size')) / 1048576)) + age = int(round((time.time() - int(self.getTextElement(nzb, 'postdate'))) / 86400)) + + results.append({ + 'id': nzb_id, + 'name': title, + 'age': age, + 'size': size, + 'url': self.urls['download'] % id + self.getApiExt() + self.getTextElement(nzb, 'key'), + 'detail_url': self.urls['detail'] % nzb_id, + 'description': self.getTextElement(nzb, 'addtext'), + }) return results diff --git a/couchpotato/core/providers/nzb/nzbx/main.py b/couchpotato/core/providers/nzb/nzbx/main.py index a67f1e6..1f199ea 100644 --- a/couchpotato/core/providers/nzb/nzbx/main.py +++ b/couchpotato/core/providers/nzb/nzbx/main.py @@ -1,91 +1,45 @@ -from couchpotato.core.event import fireEvent from couchpotato.core.helpers.encoding import tryUrlencode -from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog +from couchpotato.core.providers.base import ResultList from couchpotato.core.providers.nzb.base import NZBProvider -import json -import traceback log = CPLog(__name__) -class Nzbx(NZBProvider, RSS): - endpoint = 'https://nzbx.co/api/' +class Nzbx(NZBProvider): urls = { - 'search': 'https://nzbx.co/api/search', + 'search': 'https://nzbx.co/api/search?%s', 'details': 'https://nzbx.co/api/details?guid=%s', - 'comments': 'https://nzbx.co/api/get-comments?guid=%s', - 'ratings': 'https://nzbx.co/api/get-votes?guid=%s', - 'downloads': 'https://nzbx.co/api/get-downloads-count?guid=%s', - 'categories': 'https://nzbx.co/api/categories', - 'groups': 'https://nzbx.co/api/groups', } http_time_between_calls = 1 # Seconds def search(self, movie, quality): - results = [] if self.isDisabled(): - return results + return [] + results = ResultList(self, movie, quality, imdb_result = True) + + # Get nbzs arguments = tryUrlencode({ 'q': movie['library']['identifier'].replace('tt', ''), 'sf': quality.get('size_min'), }) - url = "%s?%s" % (self.urls['search'], arguments) - - cache_key = 'nzbx.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) - - data = self.getCache(cache_key, url) - - if data: - try: - try: - nzbs = json.loads(data) - except Exception, e: - log.debug('%s, %s', (self.getName(), e)) - return results - - for nzb in nzbs: + nzbs = self.getJsonData(self.urls['search'] % arguments) - nzbx_guid = nzb['guid'] + for nzb in nzbs: - def extra_score(item): - score = 0 - if item['votes']['upvotes'] > item['votes']['downvotes']: - score += 5 - return score - - new = { - 'guid': nzbx_guid, - 'type': 'nzb', - 'provider': self.getName(), - 'download': self.download, - 'url': nzb['nzb'], - 'name': nzb['name'], - 'age': self.calculateAge(int(nzb['postdate'])), - 'size': tryInt(nzb['size']) / 1024 / 1024, - 'description': '', - 'extra_score': extra_score, - 'votes': nzb['votes'], - 'check_nzb': True, - } - - is_correct_movie = fireEvent('searcher.correct_movie', - nzb = new, movie = movie, quality = quality, - imdb_results = False, single = True) - - if is_correct_movie: - new['score'] = fireEvent('score.calculate', new, movie, single = True) - results.append(new) - self.found(new) - - return results - except: - log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) + results.append({ + 'id': nzb['guid'], + 'url': nzb['nzb'], + 'detail_url': self.urls['details'] % nzb['guid'], + 'name': nzb['name'], + 'age': self.calculateAge(int(nzb['postdate'])), + 'size': tryInt(nzb['size']) / 1024 / 1024, + 'score': 5 if nzb['votes']['upvotes'] > nzb['votes']['downvotes'] else 0 + }) return results - diff --git a/couchpotato/core/providers/nzb/omgwtfnzbs/main.py b/couchpotato/core/providers/nzb/omgwtfnzbs/main.py index 533190b..ee2d600 100644 --- a/couchpotato/core/providers/nzb/omgwtfnzbs/main.py +++ b/couchpotato/core/providers/nzb/omgwtfnzbs/main.py @@ -4,6 +4,7 @@ from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import tryInt, getTitle, possibleTitles from couchpotato.core.logger import CPLog +from couchpotato.core.providers.base import ResultList from couchpotato.core.providers.nzb.base import NZBProvider from dateutil.parser import parse from urlparse import urlparse, parse_qs @@ -35,66 +36,39 @@ class OMGWTFNZBs(NZBProvider, RSS): if self.isDisabled() or quality['identifier'] in pre_releases: return [] - results = [] + results = ResultList(self, movie, quality) for title in possibleTitles(getTitle(movie['library'])): results.extend(self._search(title, movie, quality)) - return self.removeDuplicateResults(results) + return results def _search(self, title, movie, quality): results = [] q = '%s %s' % (title, movie['library']['year']) - - params = { + params = tryUrlencode({ 'search': q, 'catid': ','.join([str(x) for x in self.getCatId(quality['identifier'])]), 'user': self.conf('username', default = ''), 'api': self.conf('api_key', default = ''), - } - - cache_key = 'omgwtfnzbs.%s.%s' % (movie['library']['identifier'], q) - data = self.getCache(cache_key, self.urls['search'] % tryUrlencode(params)) - if data: - try: - try: - data = XMLTree.fromstring(data) - nzbs = self.getElements(data, 'channel/item') - except Exception, e: - log.debug('%s, %s', (self.getName(), e)) - return results - - for nzb in nzbs: - - nzb_id = parse_qs(urlparse(self.getTextElement(nzb, "link")).query).get('id')[0] - enclosure = self.getElement(nzb, "enclosure").attrib - size = enclosure['length'] - date = self.getTextElement(nzb, "pubDate") - - new = { - 'id': nzb_id, - 'type': 'nzb', - 'provider': self.getName(), - 'name': toUnicode(self.getTextElement(nzb, "title")), - 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))), - 'size': tryInt(size) / 1024 / 1024, - 'url': enclosure['url'], - 'download': self.download, - 'detail_url': self.getTextElement(nzb, "link"), - 'description': self.getTextElement(nzb, 'description') - } - - is_correct_movie = fireEvent('searcher.correct_movie', - nzb = new, movie = movie, quality = quality, - imdb_results = False, single = True) - - if is_correct_movie: - new['score'] = fireEvent('score.calculate', new, movie, single = True) - results.append(new) - self.found(new) - - return results - except SyntaxError: - log.error('Failed to parse XML response from omgwtfnzbs') + }) + + nzbs = self.getRSSData(self.urls['search'] % params) + + for nzb in nzbs: + + nzb_id = parse_qs(urlparse(self.getTextElement(nzb, 'link')).query).get('id')[0] + enclosure = self.getElement(nzb, 'enclosure').attrib + size = enclosure['length'] + + results.append({ + 'id': nzb_id, + 'name': toUnicode(self.getTextElement(nzb, 'title')), + 'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, 'pubDate')).timetuple()))), + 'size': tryInt(size) / 1024 / 1024, + 'url': enclosure['url'], + 'detail_url': self.getTextElement(nzb, 'link'), + 'description': self.getTextElement(nzb, 'description') + }) return results