NZB Provider cleanup

couchpotato/core/event.py

@@ -105,14 +105,14 @@ def fireEvent(name, *args, **kwargs):
             # Merge
             if options['merge'] and len(results) > 0:
                 # Dict
-                if type(results[0]) == dict:
+                if isinstance(results[0], dict):
                     merged = {}
                     for result in results:
                         merged = mergeDicts(merged, result)
 
                     results = merged
                 # Lists
-                elif type(results[0]) == list:
+                elif isinstance(results[0], list):
                     merged = []
                     for result in results:
                         merged += result

couchpotato/core/plugins/searcher/main.py

@@ -363,7 +363,7 @@ class Searcher(Plugin):
             return True
 
         # Check if nzb contains imdb link
-        if self.checkIMDB([nzb['description']], movie['library']['identifier']):
+        if self.checkIMDB([nzb.get('description', '')], movie['library']['identifier']):
             return True
 
         for raw_title in movie['library']['titles']:

couchpotato/core/providers/base.py

@@ -1,15 +1,16 @@
-from couchpotato.core.event import addEvent
-from couchpotato.core.helpers.encoding import simplifyString
-from couchpotato.core.helpers.variable import tryFloat, getTitle
+from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.helpers.variable import tryFloat, mergeDicts, md5
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.environment import Env
 from urlparse import urlparse
 import cookielib
+import json
 import re
 import time
 import traceback
 import urllib2
+import xml.etree.ElementTree as XMLTree
 
 
 log = CPLog(__name__)
@@ -59,8 +60,6 @@ class YarrProvider(Provider):
         addEvent('%s.search' % self.type, self.search)
         addEvent('yarr.search', self.search)
 
-        addEvent('nzb.feed', self.feed)
-
     def login(self):
 
         try:
@@ -97,9 +96,6 @@ class YarrProvider(Provider):
 
         return 'try_next'
 
-    def feed(self):
-        return []
-
     def search(self, movie, quality):
         return []
 
@@ -149,22 +145,93 @@ class YarrProvider(Provider):
 
         return [self.cat_backup_id]
 
-    def found(self, new):
-        if not new.get('provider_extra'):
-            new['provider_extra'] = ''
-        else:
-            new['provider_extra'] = ', %s' % new['provider_extra']
+    def getJsonData(self, url, **kwargs):
+
+        data = self.getCache(md5(url), url, **kwargs)
+
+        if data:
+            try:
+                return json.loads(data)
+            except:
+                log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
+
+        return []
+
+    def getRSSData(self, url, **kwargs):
+
+        data = self.getCache(md5(url), url, **kwargs)
+
+        if data:
+            try:
+                data = XMLTree.fromstring(data)
+                return self.getElements(data, 'channel/item')
+            except:
+                log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
+
+        return []
+
+    def getHTMLData(self, url, **kwargs):
+        return self.getCache(md5(url), url, **kwargs)
+
+
+class ResultList(list):
+
+    result_ids = None
+    provider = None
+    movie = None
+    quality = None
+
+    def __init__(self, provider, movie, quality, **kwargs):
+
+        self.result_ids = []
+        self.provider = provider
+        self.movie = movie
+        self.quality = quality
+        self.kwargs = kwargs
+
+        super(ResultList, self).__init__()
 
-        log.info('Found: score(%(score)s) on %(provider)s%(provider_extra)s: %(name)s', new)
+    def extend(self, results):
+        for r in results:
+            self.append(r)
 
-    def removeDuplicateResults(self, results):
+    def append(self, result):
 
-        result_ids = []
-        new_results = []
+        new_result = self.fillResult(result)
 
-        for result in results:
-            if result['id'] not in result_ids:
-                new_results.append(result)
-                result_ids.append(result['id'])
+        is_correct_movie = fireEvent('searcher.correct_movie',
+                                     nzb = new_result, movie = self.movie, quality = self.quality,
+                                     imdb_results = self.kwargs.get('imdb_results', False), single = True)
+
+        if is_correct_movie and new_result['id'] not in self.result_ids:
+            new_result['score'] += fireEvent('score.calculate', new_result, self.movie, single = True)
+
+            self.found(new_result)
+            self.result_ids.append(result['id'])
+
+            super(ResultList, self).append(new_result)
+
+    def fillResult(self, result):
+
+        defaults = {
+            'id': 0,
+            'type': self.provider.type,
+            'provider': self.provider.getName(),
+            'download': self.provider.download,
+            'url': '',
+            'name': '',
+            'age': 0,
+            'size': 0,
+            'description': '',
+            'score': 0
+        }
+
+        return mergeDicts(defaults, result)
+
+    def found(self, new_result):
+        if not new_result.get('provider_extra'):
+            new_result['provider_extra'] = ''
+        else:
+            new_result['provider_extra'] = ', %s' % new_result['provider_extra']
 
-        return new_results
+        log.info('Found: score(%(score)s) on %(provider)s%(provider_extra)s: %(name)s', new_result)

couchpotato/core/providers/nzb/ftdworld/main.py

@@ -1,8 +1,8 @@
 from bs4 import BeautifulSoup
-from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
 from couchpotato.core.helpers.variable import tryInt, possibleTitles, getTitle
 from couchpotato.core.logger import CPLog
+from couchpotato.core.providers.base import ResultList
 from couchpotato.core.providers.nzb.base import NZBProvider
 from couchpotato.environment import Env
 from dateutil.parser import parse
@@ -35,18 +35,17 @@ class FTDWorld(NZBProvider):
         if self.isDisabled():
             return []
 
-        results = []
+        results = ResultList(self, movie, quality)
         for title in possibleTitles(getTitle(movie['library'])):
             results.extend(self._search(title, movie, quality))
 
-        return self.removeDuplicateResults(results)
+        return results
 
     def _search(self, title, movie, quality):
-        results = []
 
         q = '"%s" %s' % (title, movie['library']['year'])
 
-        params = {
+        params = tryUrlencode({
             'ctitle': q,
             'customQuery': 'usr',
             'cage': Env.setting('retention', 'nzb'),
@@ -54,10 +53,9 @@ class FTDWorld(NZBProvider):
             'csizemax': quality.get('size_max'),
             'ccategory': 14,
             'ctype': ','.join([str(x) for x in self.getCatId(quality['identifier'])]),
-        }
+        })
 
-        cache_key = 'ftdworld.%s.%s' % (movie['library']['identifier'], q)
-        data = self.getCache(cache_key, self.urls['search'] % tryUrlencode(params), opener = self.login_opener)
+        data = self.getHTMLData(self.urls['search'] % params, opener = self.login_opener)
 
         if data:
             try:
@@ -66,8 +64,9 @@ class FTDWorld(NZBProvider):
                 main_table = html.find('table', attrs = {'id':'ftdresult'})
 
                 if not main_table:
-                    return results
+                    return []
 
+                results = ResultList(self, movie, quality)
                 items = main_table.find_all('tr', attrs = {'class': re.compile('tcontent')})
 
                 for item in items:
@@ -77,32 +76,20 @@ class FTDWorld(NZBProvider):
                     up = item.find('img', attrs = {'src': re.compile('up.png')})
                     down = item.find('img', attrs = {'src': re.compile('down.png')})
 
-                    new = {
+                    results.append({
                         'id': nzb_id,
-                        'type': 'nzb',
-                        'provider': self.getName(),
                         'name': toUnicode(item.find('a', attrs = {'href': re.compile('./spotinfo')}).text.strip()),
                         'age': self.calculateAge(int(time.mktime(parse(tds[2].text).timetuple()))),
-                        'size': 0,
                         'url': self.urls['download'] % nzb_id,
                         'download': self.loginDownload,
                         'detail_url': self.urls['detail'] % nzb_id,
-                        'description': '',
                         'score': (tryInt(up.attrs['title'].split(' ')[0]) * 3) - (tryInt(down.attrs['title'].split(' ')[0]) * 3) if up else 0,
-                    }
-
-                    is_correct_movie = fireEvent('searcher.correct_movie',
-                                                 nzb = new, movie = movie, quality = quality,
-                                                 imdb_results = False, single = True)
-
-                    if is_correct_movie:
-                        new['score'] += fireEvent('score.calculate', new, movie, single = True)
-                        results.append(new)
-                        self.found(new)
+                    })
 
                 return results
-            except SyntaxError:
-                log.error('Failed to parse XML response from NZBClub')
+
+            except:
+                log.error('Failed to parse HTML response from FTDWorld')
 
         return results
 

couchpotato/core/providers/nzb/newznab/main.py

@@ -1,8 +1,8 @@
-from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import cleanHost, splitString
 from couchpotato.core.logger import CPLog
+from couchpotato.core.providers.base import ResultList
 from couchpotato.core.providers.nzb.base import NZBProvider
 from couchpotato.environment import Env
 from dateutil.parser import parse
@@ -10,7 +10,6 @@ from urllib2 import HTTPError
 from urlparse import urlparse
 import time
 import traceback
-import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)
 
@@ -29,38 +28,6 @@ class Newznab(NZBProvider, RSS):
 
     http_time_between_calls = 1 # Seconds
 
-    def feed(self):
-
-        hosts = self.getHosts()
-
-        results = []
-        for host in hosts:
-            result = self.singleFeed(host)
-
-            if result:
-                results.extend(result)
-
-        return results
-
-    def singleFeed(self, host):
-
-        results = []
-        if self.isDisabled(host):
-            return results
-
-        arguments = tryUrlencode({
-            't': self.cat_backup_id,
-            'r': host['api_key'],
-            'i': 58,
-        })
-        url = '%s?%s' % (cleanHost(host['host']) + 'rss', arguments)
-        cache_key = 'newznab.%s.feed.%s' % (host['host'], arguments)
-
-        results = self.createItems(url, cache_key, host, for_feed = True)
-
-        return results
-
-
     def search(self, movie, quality):
 
         hosts = self.getHosts()
@@ -76,9 +43,10 @@ class Newznab(NZBProvider, RSS):
 
     def singleSearch(self, host, movie, quality):
 
-        results = []
         if self.isDisabled(host):
-            return results
+            return []
+
+        results = ResultList(self, movie, quality, imdb_result = True)
 
         cat_id = self.getCatId(quality['identifier'])
         arguments = tryUrlencode({
@@ -89,73 +57,35 @@ class Newznab(NZBProvider, RSS):
         })
         url = '%s&%s' % (self.getUrl(host['host'], self.urls['search']), arguments)
 
-        cache_key = 'newznab.%s.%s.%s' % (host['host'], movie['library']['identifier'], cat_id)
+        nzbs = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})
 
-        results = self.createItems(url, cache_key, host, movie = movie, quality = quality)
+        for nzb in nzbs:
 
-        return results
+            date = None
+            for item in nzb:
+                if item.attrib.get('name') == 'usenetdate':
+                    date = item.attrib.get('value')
+                    break
 
-    def createItems(self, url, cache_key, host, movie = None, quality = None, for_feed = False):
-        results = []
+            if not date:
+                date = self.getTextElement(nzb, 'pubDate')
 
-        data = self.getCache(cache_key, url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})
-        if data:
-            try:
-                try:
-                    data = XMLTree.fromstring(data)
-                    nzbs = self.getElements(data, 'channel/item')
-                except Exception, e:
-                    log.debug('%s, %s', (self.getName(), e))
-                    return results
-
-                results = []
-                for nzb in nzbs:
-
-                    date = None
-                    for item in nzb:
-                        if item.attrib.get('name') == 'usenetdate':
-                            date = item.attrib.get('value')
-                            break
-
-                    if not date:
-                        date = self.getTextElement(nzb, 'pubDate')
-
-                    nzb_id = self.getTextElement(nzb, 'guid').split('/')[-1:].pop()
-                    name = self.getTextElement(nzb, 'title')
-
-                    if not name:
-                        continue
-
-                    new = {
-                        'id': nzb_id,
-                        'provider': self.getName(),
-                        'provider_extra': host['host'],
-                        'type': 'nzb',
-                        'name': self.getTextElement(nzb, 'title'),
-                        'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
-                        'size': int(self.getElement(nzb, 'enclosure').attrib['length']) / 1024 / 1024,
-                        'url': (self.getUrl(host['host'], self.urls['download']) % nzb_id) + self.getApiExt(host),
-                        'download': self.download,
-                        'detail_url': '%sdetails/%s' % (cleanHost(host['host']), nzb_id),
-                        'content': self.getTextElement(nzb, 'description'),
-                    }
-
-                    if not for_feed:
-                        is_correct_movie = fireEvent('searcher.correct_movie',
-                                                     nzb = new, movie = movie, quality = quality,
-                                                     imdb_results = True, single = True)
-
-                        if is_correct_movie:
-                            new['score'] = fireEvent('score.calculate', new, movie, single = True)
-                            results.append(new)
-                            self.found(new)
-                    else:
-                        results.append(new)
-
-                return results
-            except SyntaxError:
-                log.error('Failed to parse XML response from Newznab: %s', host)
-                return results
+            nzb_id = self.getTextElement(nzb, 'guid').split('/')[-1:].pop()
+            name = self.getTextElement(nzb, 'title')
+
+            if not name:
+                continue
+
+            results.append({
+                'id': nzb_id,
+                'provider_extra': host['host'],
+                'name': self.getTextElement(nzb, 'title'),
+                'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
+                'size': int(self.getElement(nzb, 'enclosure').attrib['length']) / 1024 / 1024,
+                'url': (self.getUrl(host['host'], self.urls['download']) % nzb_id) + self.getApiExt(host),
+                'detail_url': '%sdetails/%s' % (cleanHost(host['host']), nzb_id),
+                'content': self.getTextElement(nzb, 'description'),
+            })
 
     def getHosts(self):
 

couchpotato/core/providers/nzb/nzbclub/main.py

@@ -1,13 +1,12 @@
 from bs4 import BeautifulSoup
-from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import tryInt, getTitle, possibleTitles
 from couchpotato.core.logger import CPLog
+from couchpotato.core.providers.base import ResultList
 from couchpotato.core.providers.nzb.base import NZBProvider
 from dateutil.parser import parse
 import time
-import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)
 
@@ -25,81 +24,56 @@ class NZBClub(NZBProvider, RSS):
         if self.isDisabled():
             return []
 
-        results = []
+        results = ResultList(self, movie, quality)
+
         for title in possibleTitles(getTitle(movie['library'])):
             results.extend(self._search(title, movie, quality))
 
-        return self.removeDuplicateResults(results)
+        return results
 
     def _search(self, title, movie, quality):
         results = []
 
         q = '"%s %s" %s' % (title, movie['library']['year'], quality.get('identifier'))
 
-        params = {
+        params = tryUrlencode({
             'q': q,
             'ig': '1',
             'rpp': 200,
             'st': 1,
             'sp': 1,
             'ns': 1,
-        }
-
-        cache_key = 'nzbclub.%s.%s.%s' % (movie['library']['identifier'], quality.get('identifier'), q)
-        data = self.getCache(cache_key, self.urls['search'] % tryUrlencode(params))
-        if data:
-            try:
-                try:
-                    data = XMLTree.fromstring(data)
-                    nzbs = self.getElements(data, 'channel/item')
-                except Exception, e:
-                    log.debug('%s, %s', (self.getName(), e))
-                    return results
-
-                for nzb in nzbs:
-
-                    nzbclub_id = tryInt(self.getTextElement(nzb, "link").split('/nzb_view/')[1].split('/')[0])
-                    enclosure = self.getElement(nzb, "enclosure").attrib
-                    size = enclosure['length']
-                    date = self.getTextElement(nzb, "pubDate")
-
-                    def extra_check(item):
-                        full_description = self.getCache('nzbclub.%s' % nzbclub_id, item['detail_url'], cache_timeout = 25920000)
-
-                        for ignored in ['ARCHIVE inside ARCHIVE', 'Incomplete', 'repair impossible']:
-                            if ignored in full_description:
-                                log.info('Wrong: Seems to be passworded or corrupted files: %s', new['name'])
-                                return False
-
-                        return True
-
-                    new = {
-                        'id': nzbclub_id,
-                        'type': 'nzb',
-                        'provider': self.getName(),
-                        'name': toUnicode(self.getTextElement(nzb, "title")),
-                        'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
-                        'size': tryInt(size) / 1024 / 1024,
-                        'url': enclosure['url'].replace(' ', '_'),
-                        'download': self.download,
-                        'detail_url': self.getTextElement(nzb, "link"),
-                        'description': '',
-                        'get_more_info': self.getMoreInfo,
-                        'extra_check': extra_check
-                    }
-
-                    is_correct_movie = fireEvent('searcher.correct_movie',
-                                                 nzb = new, movie = movie, quality = quality,
-                                                 imdb_results = False, single = True)
-
-                    if is_correct_movie:
-                        new['score'] = fireEvent('score.calculate', new, movie, single = True)
-                        results.append(new)
-                        self.found(new)
-
-                return results
-            except SyntaxError:
-                log.error('Failed to parse XML response from NZBClub')
+        })
+
+        nzbs = self.getRSSData(self.urls['search'] % params)
+
+        for nzb in nzbs:
+
+            nzbclub_id = tryInt(self.getTextElement(nzb, "link").split('/nzb_view/')[1].split('/')[0])
+            enclosure = self.getElement(nzb, "enclosure").attrib
+            size = enclosure['length']
+            date = self.getTextElement(nzb, "pubDate")
+
+            def extra_check(item):
+                full_description = self.getCache('nzbclub.%s' % nzbclub_id, item['detail_url'], cache_timeout = 25920000)
+
+                for ignored in ['ARCHIVE inside ARCHIVE', 'Incomplete', 'repair impossible']:
+                    if ignored in full_description:
+                        log.info('Wrong: Seems to be passworded or corrupted files: %s', item['name'])
+                        return False
+
+                return True
+
+            results.append({
+                'id': nzbclub_id,
+                'name': toUnicode(self.getTextElement(nzb, "title")),
+                'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
+                'size': tryInt(size) / 1024 / 1024,
+                'url': enclosure['url'].replace(' ', '_'),
+                'detail_url': self.getTextElement(nzb, "link"),
+                'get_more_info': self.getMoreInfo,
+                'extra_check': extra_check
+            })
 
         return results
 

couchpotato/core/providers/nzb/nzbindex/main.py

@@ -1,16 +1,14 @@
 from bs4 import BeautifulSoup
-from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import tryInt, getTitle, possibleTitles
 from couchpotato.core.logger import CPLog
+from couchpotato.core.providers.base import ResultList
 from couchpotato.core.providers.nzb.base import NZBProvider
 from couchpotato.environment import Env
 from dateutil.parser import parse
 import re
 import time
-import traceback
-import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)
 
@@ -19,7 +17,7 @@ class NzbIndex(NZBProvider, RSS):
 
     urls = {
         'download': 'https://www.nzbindex.com/download/',
-        'api': 'https://www.nzbindex.com/rss/',
+        'search': 'https://www.nzbindex.com/rss/?%s',
     }
 
     http_time_between_calls = 1 # Seconds
@@ -29,13 +27,14 @@ class NzbIndex(NZBProvider, RSS):
         if self.isDisabled():
             return []
 
-        results = []
+        results = ResultList(self, movie, quality)
         for title in possibleTitles(getTitle(movie['library'])):
             results.extend(self._search(title, movie, quality))
 
-        return self.removeDuplicateResults(results)
+        return results
 
     def _search(self, title, movie, quality):
+
         results = []
 
         q = '"%s" %s %s' % (title, movie['library']['year'], quality.get('identifier'))
@@ -50,66 +49,37 @@ class NzbIndex(NZBProvider, RSS):
             'more': 1,
             'complete': 1,
         })
-        url = "%s?%s" % (self.urls['api'], arguments)
 
-        cache_key = 'nzbindex.%s.%s' % (movie['library']['identifier'], q)
-        data = self.getCache(cache_key, url)
+        nzbs = self.getRSSData(self.urls['search'] % arguments)
+
+        for nzb in nzbs:
+
+            enclosure = self.getElement(nzb, 'enclosure').attrib
+            nzbindex_id = int(self.getTextElement(nzb, "link").split('/')[4])
 
-        if data:
             try:
-                try:
-                    data = XMLTree.fromstring(data)
-                    nzbs = self.getElements(data, 'channel/item')
-                except Exception, e:
-                    log.debug('%s, %s', (self.getName(), e))
-                    return results
-
-                for nzb in nzbs:
-
-                    enclosure = self.getElement(nzb, 'enclosure').attrib
-
-                    nzbindex_id = int(self.getTextElement(nzb, "link").split('/')[4])
-
-                    try:
-                        description = self.getTextElement(nzb, "description")
-                    except:
-                        description = ''
-
-                    def extra_check(new):
-                        if '#c20000' in new['description'].lower():
-                            log.info('Wrong: Seems to be passworded: %s', new['name'])
-                            return False
-
-                        return True
-
-                    new = {
-                        'id': nzbindex_id,
-                        'type': 'nzb',
-                        'provider': self.getName(),
-                        'download': self.download,
-                        'name': self.getTextElement(nzb, "title"),
-                        'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, "pubDate")).timetuple()))),
-                        'size': tryInt(enclosure['length']) / 1024 / 1024,
-                        'url': enclosure['url'],
-                        'detail_url': enclosure['url'].replace('/download/', '/release/'),
-                        'description': description,
-                        'get_more_info': self.getMoreInfo,
-                        'extra_check': extra_check,
-                        'check_nzb': True,
-                    }
-
-                    is_correct_movie = fireEvent('searcher.correct_movie',
-                                                 nzb = new, movie = movie, quality = quality,
-                                                 imdb_results = False, single = True)
-
-                    if is_correct_movie:
-                        new['score'] = fireEvent('score.calculate', new, movie, single = True)
-                        results.append(new)
-                        self.found(new)
-
-                return results
+                description = self.getTextElement(nzb, "description")
             except:
-                log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
+                description = ''
+
+            def extra_check(item):
+                if '#c20000' in item['description'].lower():
+                    log.info('Wrong: Seems to be passworded: %s', item['name'])
+                    return False
+
+                return True
+
+            results.append({
+                'id': nzbindex_id,
+                'name': self.getTextElement(nzb, "title"),
+                'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, "pubDate")).timetuple()))),
+                'size': tryInt(enclosure['length']) / 1024 / 1024,
+                'url': enclosure['url'],
+                'detail_url': enclosure['url'].replace('/download/', '/release/'),
+                'description': description,
+                'get_more_info': self.getMoreInfo,
+                'extra_check': extra_check,
+            })
 
         return results
 
@@ -123,5 +93,3 @@ class NzbIndex(NZBProvider, RSS):
         except:
             pass
 
-    def isEnabled(self):
-        return NZBProvider.isEnabled(self) and self.conf('enabled')

couchpotato/core/providers/nzb/nzbsrus/main.py

@@ -1,11 +1,10 @@
-from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.logger import CPLog
+from couchpotato.core.providers.base import ResultList
 from couchpotato.core.providers.nzb.base import NZBProvider
 from couchpotato.environment import Env
 import time
-import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)
 
@@ -25,13 +24,12 @@ class Nzbsrus(NZBProvider, RSS):
 
     def search(self, movie, quality):
 
-        results = []
-
         if self.isDisabled():
-            return results
+            return []
 
-        cat_id_string = '&'.join(['c%s=1' % x for x in self.getCatId(quality.get('identifier'))])
+        results = ResultList(self, movie, quality, imdb_result = True)
 
+        cat_id_string = '&'.join(['c%s=1' % x for x in self.getCatId(quality.get('identifier'))])
         arguments = tryUrlencode({
             'searchtext': 'imdb:' + movie['library']['identifier'][2:],
             'uid': self.conf('userid'),
@@ -42,57 +40,29 @@ class Nzbsrus(NZBProvider, RSS):
 
         # check for english_only
         if self.conf('english_only'):
-            arguments += "&lang0=1&lang3=1&lang1=1"
-
-        url = "%s&%s&%s" % (self.urls['search'], arguments , cat_id_string)
-
-        cache_key = 'nzbsrus.%s.%s' % (movie['library'].get('identifier'), cat_id_string)
-
-        data = self.getCache(cache_key, url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})
-        if data:
-            try:
-                try:
-                    data = XMLTree.fromstring(data)
-                    nzbs = self.getElements(data, 'results/result')
-                except Exception, e:
-                    log.debug('%s, %s', (self.getName(), e))
-                    return results
-
-                for nzb in nzbs:
-
-                    title = self.getTextElement(nzb, "name")
-                    if 'error' in title.lower(): continue
-
-                    id = self.getTextElement(nzb, "id")
-                    size = int(round(int(self.getTextElement(nzb, "size")) / 1048576))
-                    age = int(round((time.time() - int(self.getTextElement(nzb, "postdate"))) / 86400))
-
-                    new = {
-                        'id': id,
-                        'type': 'nzb',
-                        'provider': self.getName(),
-                        'name': title,
-                        'age': age,
-                        'size': size,
-                        'url': self.urls['download'] % id + self.getApiExt() + self.getTextElement(nzb, "key"),
-                        'download': self.download,
-                        'detail_url': self.urls['detail'] % id,
-                        'description': self.getTextElement(nzb, "addtext"),
-                        'check_nzb': True,
-                    }
-
-                    is_correct_movie = fireEvent('searcher.correct_movie',
-                                                 nzb = new, movie = movie, quality = quality,
-                                                 imdb_results = True, single = True)
-
-                    if is_correct_movie:
-                        new['score'] = fireEvent('score.calculate', new, movie, single = True)
-                        results.append(new)
-                        self.found(new)
-
-                return results
-            except SyntaxError:
-                log.error('Failed to parse XML response from Nzbsrus.com')
+            arguments += '&lang0=1&lang3=1&lang1=1'
+
+        url = '%s&%s&%s' % (self.urls['search'], arguments , cat_id_string)
+        nzbs = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})
+
+        for nzb in nzbs:
+
+            title = self.getTextElement(nzb, 'name')
+            if 'error' in title.lower(): continue
+
+            nzb_id = self.getTextElement(nzb, 'id')
+            size = int(round(int(self.getTextElement(nzb, 'size')) / 1048576))
+            age = int(round((time.time() - int(self.getTextElement(nzb, 'postdate'))) / 86400))
+
+            results.append({
+                'id': nzb_id,
+                'name': title,
+                'age': age,
+                'size': size,
+                'url': self.urls['download'] % id + self.getApiExt() + self.getTextElement(nzb, 'key'),
+                'detail_url': self.urls['detail'] % nzb_id,
+                'description': self.getTextElement(nzb, 'addtext'),
+            })
 
         return results
 

couchpotato/core/providers/nzb/nzbx/main.py

@@ -1,91 +1,45 @@
-from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import tryUrlencode
-from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
+from couchpotato.core.providers.base import ResultList
 from couchpotato.core.providers.nzb.base import NZBProvider
-import json
-import traceback
 
 log = CPLog(__name__)
 
 
-class Nzbx(NZBProvider, RSS):
-    endpoint = 'https://nzbx.co/api/'
+class Nzbx(NZBProvider):
 
     urls = {
-        'search': 'https://nzbx.co/api/search',
+        'search': 'https://nzbx.co/api/search?%s',
         'details': 'https://nzbx.co/api/details?guid=%s',
-        'comments': 'https://nzbx.co/api/get-comments?guid=%s',
-        'ratings': 'https://nzbx.co/api/get-votes?guid=%s',
-        'downloads': 'https://nzbx.co/api/get-downloads-count?guid=%s',
-        'categories': 'https://nzbx.co/api/categories',
-        'groups': 'https://nzbx.co/api/groups',
     }
 
     http_time_between_calls = 1 # Seconds
 
     def search(self, movie, quality):
-        results = []
 
         if self.isDisabled():
-            return results
+            return []
 
+        results = ResultList(self, movie, quality, imdb_result = True)
+
+        # Get nbzs
         arguments = tryUrlencode({
             'q': movie['library']['identifier'].replace('tt', ''),
             'sf': quality.get('size_min'),
         })
-        url = "%s?%s" % (self.urls['search'], arguments)
-
-        cache_key = 'nzbx.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
-
-        data = self.getCache(cache_key, url)
-
-        if data:
-            try:
-                try:
-                    nzbs = json.loads(data)
-                except Exception, e:
-                    log.debug('%s, %s', (self.getName(), e))
-                    return results
-
-                for nzb in nzbs:
+        nzbs = self.getJsonData(self.urls['search'] % arguments)
 
-                    nzbx_guid = nzb['guid']
+        for nzb in nzbs:
 
-                    def extra_score(item):
-                        score = 0
-                        if item['votes']['upvotes'] > item['votes']['downvotes']:
-                            score += 5
-                        return score
-
-                    new = {
-                        'guid': nzbx_guid,
-                        'type': 'nzb',
-                        'provider': self.getName(),
-                        'download': self.download,
-                        'url': nzb['nzb'],
-                        'name': nzb['name'],
-                        'age': self.calculateAge(int(nzb['postdate'])),
-                        'size': tryInt(nzb['size']) / 1024 / 1024,
-                        'description': '',
-                        'extra_score': extra_score,
-                        'votes': nzb['votes'],
-                        'check_nzb': True,
-                    }
-
-                    is_correct_movie = fireEvent('searcher.correct_movie',
-                                                 nzb = new, movie = movie, quality = quality,
-                                                 imdb_results = False, single = True)
-
-                    if is_correct_movie:
-                        new['score'] = fireEvent('score.calculate', new, movie, single = True)
-                        results.append(new)
-                        self.found(new)
-
-                return results
-            except:
-                log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
+            results.append({
+                'id': nzb['guid'],
+                'url': nzb['nzb'],
+                'detail_url': self.urls['details'] % nzb['guid'],
+                'name': nzb['name'],
+                'age': self.calculateAge(int(nzb['postdate'])),
+                'size': tryInt(nzb['size']) / 1024 / 1024,
+                'score': 5 if nzb['votes']['upvotes'] > nzb['votes']['downvotes'] else 0
+            })
 
         return results
-

couchpotato/core/providers/nzb/omgwtfnzbs/main.py

@@ -4,6 +4,7 @@ from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import tryInt, getTitle, possibleTitles
 from couchpotato.core.logger import CPLog
+from couchpotato.core.providers.base import ResultList
 from couchpotato.core.providers.nzb.base import NZBProvider
 from dateutil.parser import parse
 from urlparse import urlparse, parse_qs
@@ -35,66 +36,39 @@ class OMGWTFNZBs(NZBProvider, RSS):
         if self.isDisabled() or quality['identifier'] in pre_releases:
             return []
 
-        results = []
+        results = ResultList(self, movie, quality)
         for title in possibleTitles(getTitle(movie['library'])):
             results.extend(self._search(title, movie, quality))
 
-        return self.removeDuplicateResults(results)
+        return results
 
     def _search(self, title, movie, quality):
         results = []
 
         q = '%s %s' % (title, movie['library']['year'])
-
-        params = {
+        params = tryUrlencode({
             'search': q,
             'catid': ','.join([str(x) for x in self.getCatId(quality['identifier'])]),
             'user': self.conf('username', default = ''),
             'api': self.conf('api_key', default = ''),
-        }
-
-        cache_key = 'omgwtfnzbs.%s.%s' % (movie['library']['identifier'], q)
-        data = self.getCache(cache_key, self.urls['search'] % tryUrlencode(params))
-        if data:
-            try:
-                try:
-                    data = XMLTree.fromstring(data)
-                    nzbs = self.getElements(data, 'channel/item')
-                except Exception, e:
-                    log.debug('%s, %s', (self.getName(), e))
-                    return results
-
-                for nzb in nzbs:
-
-                    nzb_id = parse_qs(urlparse(self.getTextElement(nzb, "link")).query).get('id')[0]
-                    enclosure = self.getElement(nzb, "enclosure").attrib
-                    size = enclosure['length']
-                    date = self.getTextElement(nzb, "pubDate")
-
-                    new = {
-                        'id': nzb_id,
-                        'type': 'nzb',
-                        'provider': self.getName(),
-                        'name': toUnicode(self.getTextElement(nzb, "title")),
-                        'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
-                        'size': tryInt(size) / 1024 / 1024,
-                        'url': enclosure['url'],
-                        'download': self.download,
-                        'detail_url': self.getTextElement(nzb, "link"),
-                        'description': self.getTextElement(nzb, 'description')
-                    }
-
-                    is_correct_movie = fireEvent('searcher.correct_movie',
-                                                 nzb = new, movie = movie, quality = quality,
-                                                 imdb_results = False, single = True)
-
-                    if is_correct_movie:
-                        new['score'] = fireEvent('score.calculate', new, movie, single = True)
-                        results.append(new)
-                        self.found(new)
-
-                return results
-            except SyntaxError:
-                log.error('Failed to parse XML response from omgwtfnzbs')
+        })
+
+        nzbs = self.getRSSData(self.urls['search'] % params)
+
+        for nzb in nzbs:
+
+            nzb_id = parse_qs(urlparse(self.getTextElement(nzb, 'link')).query).get('id')[0]
+            enclosure = self.getElement(nzb, 'enclosure').attrib
+            size = enclosure['length']
+
+            results.append({
+                'id': nzb_id,
+                'name': toUnicode(self.getTextElement(nzb, 'title')),
+                'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, 'pubDate')).timetuple()))),
+                'size': tryInt(size) / 1024 / 1024,
+                'url': enclosure['url'],
+                'detail_url': self.getTextElement(nzb, 'link'),
+                'description': self.getTextElement(nzb, 'description')
+            })
 
         return results