diff --git a/CHANGES.md b/CHANGES.md index 7e17b89..41dd745 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,14 @@ +### 0.18.9 (2019-01-08 01:00:00 UTC) + +* Change ensure utf-8 locale for Ubuntu snap +* Change remove non-release group stuff from newnab results +* Add detection of NZBHydra and NZBHydra 2 to config providers +* Remove Torrentz2 + + ### 0.18.8 (2018-12-18 21:00:00 UTC) * Change first run GUI defaults to enable fanart and episode view as home diff --git a/gui/slick/images/providers/torrentz2.png b/gui/slick/images/providers/torrentz2.png deleted file mode 100644 index 5433e32..0000000 Binary files a/gui/slick/images/providers/torrentz2.png and /dev/null differ diff --git a/gui/slick/images/providers/warning16.png b/gui/slick/images/providers/warning16.png new file mode 100644 index 0000000..2735496 Binary files /dev/null and b/gui/slick/images/providers/warning16.png differ diff --git a/gui/slick/interfaces/default/config_providers.tmpl b/gui/slick/interfaces/default/config_providers.tmpl index e934eb5..447a783 100644 --- a/gui/slick/interfaces/default/config_providers.tmpl +++ b/gui/slick/interfaces/default/config_providers.tmpl @@ -97,6 +97,7 @@ #set $cur_url = $cur_provider.url #set $show_type = $sickbeard.USE_NZBS and $sickbeard.USE_TORRENTS and $GenericProvider.NZB == $cur_provider.providerType #set $spotweb = ('', 'sw ')[$getattr($cur_provider, 'server_type', None) == $NewznabConstants.SERVER_SPOTWEB and type($cur_provider).__name__ not in ['TorrentRssProvider']] + #set $is_hydra = $getattr($cur_provider, 'server_type', None) in ($NewznabConstants.SERVER_HYDRA1, $NewznabConstants.SERVER_HYDRA2) #set $bad_url = not $cur_url and cur_provider.is_enabled() #set $tip = ($cur_provider.name + ('', ' (enable for link)')[not $cur_url and not cur_provider.is_enabled()], 'Site Down')[$bad_url] @@ -109,6 +110,8 @@ (PA) #end if##if $show_type##slurp ($spotweb$cur_provider.providerType) + #end if##if $is_hydra##slurp + (read this) #end if##if not $cur_provider.supports_backlog#*#set $backlog_only_tip=True##end if##slurp diff --git a/lib/cfscrape/__init__.py b/lib/cfscrape/__init__.py index ff41075..307abfb 100644 --- a/lib/cfscrape/__init__.py +++ b/lib/cfscrape/__init__.py @@ -1,3 +1,4 @@ +from base64 import b64encode from time import sleep import logging import random @@ -42,15 +43,33 @@ class CloudflareScraper(Session): def request(self, method, url, *args, **kwargs): resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs) - # Check if Cloudflare anti-bot is on + # Check if anti-bot is on if (isinstance(resp, type(Response())) - and 503 == resp.status_code - and re.search('(?i)cloudflare', resp.headers.get('Server', '')) - and b'jschl_vc' in resp.content - and b'jschl_answer' in resp.content): - resp = self.solve_cf_challenge(resp, **kwargs) + and resp.status_code in (503, 403)): + if (re.search('(?i)cloudflare', resp.headers.get('Server', '')) + and 'jschl_vc' in resp.content + and 'jschl_answer' in resp.content): + resp = self.solve_cf_challenge(resp, **kwargs) + elif 'ddgu' in resp.content: + resp = self.solve_ddg_challenge(resp, **kwargs) + + # Otherwise, no anti-bot detected + return resp - # Otherwise, no Cloudflare anti-bot detected + def solve_ddg_challenge(self, resp, **original_kwargs): + sleep(self.delay) + parsed_url = urlparse(resp.url) + try: + submit_url = parsed_url.scheme + ':' + re.findall('"frm"[^>]+?action="([^"]+)"', resp.text)[0] + kwargs = {k: v for k, v in original_kwargs.items() if k not in ['hooks']} + kwargs.setdefault('headers', {}) + kwargs.setdefault('data', dict( + h=b64encode('%s://%s' % (parsed_url.scheme, parsed_url.hostname)), + u=b64encode(parsed_url.path), p=b64encode(parsed_url.port or '') + )) + resp = self.request('POST', submit_url, **kwargs) + except(StandardError, BaseException): + pass return resp def solve_cf_challenge(self, resp, **original_kwargs): diff --git a/sickbeard/providers/__init__.py b/sickbeard/providers/__init__.py index c0e3e17..463da93 100755 --- a/sickbeard/providers/__init__.py +++ b/sickbeard/providers/__init__.py @@ -36,7 +36,7 @@ __all__ = [ 'immortalseed', 'iptorrents', 'limetorrents', 'magnetdl', 'milkie', 'morethan', 'nebulance', 'ncore', 'nyaa', 'pisexy', 'potuk', 'pretome', 'privatehd', 'ptf', 'rarbg', 'revtt', 'scenehd', 'scenetime', 'shazbat', 'showrss', 'skytorrents', 'snowfl', 'speedcd', - 'thepiratebay', 'torlock', 'torrentday', 'torrenting', 'torrentleech', 'torrentz2', 'tvchaosuk', + 'thepiratebay', 'torlock', 'torrentday', 'torrenting', 'torrentleech', 'tvchaosuk', 'wop', 'xspeeds', 'zooqle', # anime 'anizb', 'tokyotoshokan', diff --git a/sickbeard/providers/newznab.py b/sickbeard/providers/newznab.py index a89bfdb..aacade9 100755 --- a/sickbeard/providers/newznab.py +++ b/sickbeard/providers/newznab.py @@ -33,7 +33,7 @@ from . import generic from sickbeard import classes, db, helpers, logger, tvcache from sickbeard.common import neededQualities, Quality, DOWNLOADED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST from sickbeard.exceptions import AuthException, MultipleShowObjectsException -from sickbeard.helpers import tryInt +from sickbeard.helpers import tryInt, remove_non_release_groups from sickbeard.indexers.indexer_config import * from sickbeard.network_timezones import sb_timezone from sickbeard.sbdatetime import sbdatetime @@ -94,9 +94,13 @@ class NewznabConstants: SERVER_DEFAULT = 0 SERVER_SPOTWEB = 1 + SERVER_HYDRA1 = 2 + SERVER_HYDRA2 = 3 server_types = {SERVER_DEFAULT: 'newznab', - SERVER_SPOTWEB: 'spotweb'} + SERVER_SPOTWEB: 'spotweb', + SERVER_HYDRA1: 'NZBHydra', + SERVER_HYDRA2: 'NZBHydra 2'} def __init__(self): pass @@ -194,6 +198,8 @@ class NewznabProvider(generic.NZBProvider): def image_name(self): + if self.server_type not in (NewznabConstants.SERVER_DEFAULT, NewznabConstants.SERVER_SPOTWEB): + return 'warning16.png' return generic.GenericProvider.image_name( self, ('newznab', 'spotweb')[self.server_type == NewznabConstants.SERVER_SPOTWEB]) @@ -239,9 +245,17 @@ class NewznabProvider(generic.NZBProvider): if None is not xml_caps: server_node = xml_caps.find('.//server') if None is not server_node: - self.server_type = (NewznabConstants.SERVER_DEFAULT, NewznabConstants.SERVER_SPOTWEB)[ - NewznabConstants.server_types.get(NewznabConstants.SERVER_SPOTWEB) in - (server_node.get('type', '') or server_node.get('title', '')).lower()] + if NewznabConstants.server_types.get(NewznabConstants.SERVER_SPOTWEB) in \ + (server_node.get('type', '') or server_node.get('title', '')).lower(): + self.server_type = NewznabConstants.SERVER_SPOTWEB + elif 'nzbhydra 2' in server_node.get('title', '').lower() or \ + 'nzbhydra2' in server_node.get('url', '').lower(): + self.server_type = NewznabConstants.SERVER_HYDRA2 + elif 'nzbhydra' == server_node.get('title', '').lower().strip() or \ + server_node.get('url', '').lower().strip().endswith('nzbhydra'): + self.server_type = NewznabConstants.SERVER_HYDRA1 + else: + self.server_type = NewznabConstants.SERVER_DEFAULT tv_search = xml_caps.find('.//tv-search') if None is not tv_search: @@ -477,6 +491,7 @@ class NewznabProvider(generic.NZBProvider): def _title_and_url(self, item): title, url = None, None try: + url = str(item.findtext('link')).replace('&', '&') title = ('%s' % item.findtext('title')).strip() title = re.sub(r'\s+', '.', title) # remove indexer specific release name parts @@ -488,7 +503,8 @@ class NewznabProvider(generic.NZBProvider): if re.search(pattern, title): r_found = True title = re.sub(pattern, repl, title) - url = str(item.findtext('link')).replace('&', '&') + parts = re.findall('(.*(?:(?:h.?|x)26[45]|vp9|av1|xvid|divx)[^-]*)(.*)', title, re.I)[0] + title = '%s-%s' % (parts[0], remove_non_release_groups(parts[1].split('-')[1])) except (StandardError, Exception): pass diff --git a/sickbeard/providers/torrentz2.py b/sickbeard/providers/torrentz2.py deleted file mode 100644 index ad4c388..0000000 --- a/sickbeard/providers/torrentz2.py +++ /dev/null @@ -1,126 +0,0 @@ -# coding=utf-8 -# -# This file is part of SickGear. -# -# SickGear is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# SickGear is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with SickGear. If not, see . - -import base64 -import re -import time -import traceback -from urllib import quote_plus - -from . import generic -from sickbeard import config, logger -from sickbeard.bs4_parser import BS4Parser -from sickbeard.helpers import tryInt -from lib.unidecode import unidecode - - -class Torrentz2Provider(generic.TorrentProvider): - - def __init__(self): - generic.TorrentProvider.__init__(self, 'Torrentz2') - - self.url_home = ['https://torrentz2.eu/'] + \ - ['https://%s/' % base64.b64decode(x) for x in [''.join(x) for x in [ - [re.sub('[r\sj]+', '', x[::-1]) for x in [ - 'G d', 'yr 9', 'm jc', 'urrV', 'Hr d', 'y ro', 'n rL', '2j R']], - [re.sub('[q\sP]+', '', x[::-1]) for x in [ - 'cy 9PGd', 'Hdq uVm', 'VnLqxqo', 'vqPxmYu', 'Zlt q2Y', 'G Pd35C', '= Y']], - ]]] - - self.url_vars = {'search': 'searchA?f=%s&safe=1', 'searchv': 'verifiedA?f=%s&safe=1'} - self.url_tmpl = {'config_provider_home_uri': '%(home)s', - 'search': '%(home)s%(vars)s', 'searchv': '%(home)s%(vars)s'} - - self.proper_search_terms = '.proper.|.repack.' - self.minseed, self.minleech = 2 * [None] - self.confirmed = False - - @staticmethod - def _has_signature(data=None): - return data and re.search(r'(?i)Torrentz', data) - - def _search_provider(self, search_params, **kwargs): - - results = [] - if not self.url: - return results - - items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} - - rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': r'>>.*tv'}.iteritems()) - for mode in search_params.keys(): - for search_string in search_params[mode]: - - search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string - - search_url = self.urls['search' + ('', 'v')[self.confirmed]] % ( - quote_plus('%s NOT Torrent9 NOT nextorrent' % (search_string, 'x264')['Cache' == mode])) - - html = self.get_url(search_url) - if self.should_skip(): - return results - - cnt = len(items[mode]) - try: - if not html or self._has_no_results(html): - raise generic.HaltParseException - with BS4Parser(html, features=['html5lib', 'permissive']) as soup: - torrent_rows = soup.select('dl') - - if not len(torrent_rows): - raise generic.HaltParseException - - for tr in torrent_rows: - cells = tr.dd.find_all('span') - if 4 > len(cells): - continue - try: - if not rc['info'].search(unidecode(tr.dt.get_text().strip())): - continue - seeders, leechers, size = [tryInt(n, n) for n in [ - cells[x].get_text().strip() for x in -2, -1, -3]] - if self._reject_item(seeders, leechers): - continue - - info = tr.dt.a - title = info and info.get_text().strip() - title = title and isinstance(title, unicode) and unidecode(title) or title - download_url = info and title and self._dhtless_magnet(info['href'], title) - except (AttributeError, TypeError, ValueError, IndexError): - continue - - if title and download_url: - items[mode].append((title, download_url, seeders, self._bytesizer(size))) - - except generic.HaltParseException: - time.sleep(1.1) - except (StandardError, Exception): - logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) - - self._log_search(mode, len(items[mode]) - cnt, search_url) - - results = self._sort_seeding(mode, results + items[mode]) - - return results - - def _episode_strings(self, ep_obj, **kwargs): - return super(Torrentz2Provider, self)._episode_strings( - ep_obj, date_detail=(lambda d: [x % str(d).replace('-', '.') for x in ('"%s"', '%s')]), - ep_detail=(lambda ep_dict: [x % (config.naming_ep_type[2] % ep_dict) for x in ('"%s"', '%s')]), **kwargs) - - -provider = Torrentz2Provider() diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 0b14052..65ab198 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -11,10 +11,13 @@ description: | icon: gui/slick/images/sickgear.png confinement: strict +grade: stable adopt-info: sickgear apps: sickgear: + environment: + LC_CTYPE: C.UTF-8 command: python "$SNAP/sickgear/sickgear.py" --datadir="$SNAP_COMMON/data" # daemon: simple plugs: [network, network-bind, removable-media] @@ -44,7 +47,7 @@ parts: override-pull: | snapcraftctl pull - [ -z $(grep -o 'xx:xx:xx' CHANGES.md) ] && grade=stable || grade=devel + [ -z $(grep -o "xx:xx:xx" CHANGES.md) ] && grade=stable || grade=devel snapcraftctl set-grade "$grade" version="$(sed -n '1 s/###[^0-9]\{1,\}\([0-9]\{1,\}\.[0-9]\{1,\}\.[0-9]\{1,\}\).*/\1/p' CHANGES.md)" diff --git a/tests/all_tests.py b/tests/all_tests.py index 34eb3df..f9a54e0 100644 --- a/tests/all_tests.py +++ b/tests/all_tests.py @@ -19,7 +19,10 @@ # along with SickGear. If not, see . from __future__ import print_function -if __name__ == '__main__': +if '__main__' == __name__: + import warnings + warnings.filterwarnings('ignore', module=r'.*fuz.*', message='.*Sequence.*') + import glob import unittest import sys @@ -32,10 +35,24 @@ if __name__ == '__main__': print('==================') print('STARTING - ALL TESTS') print('==================') - print('this will include') - for includedfiles in test_file_strings: - print('- ' + includedfiles) - text_runner = unittest.TextTestRunner().run(testSuite) - if not text_runner.wasSuccessful(): - sys.exit(-1) + test_individually = False + + if not test_individually: + print('this will include') + for includedfiles in test_file_strings: + print('- ' + includedfiles) + + text_runner = unittest.TextTestRunner().run(testSuite) + if not text_runner.wasSuccessful(): + sys.exit(-1) + else: + complete_success = True + for file_string in module_strings: + testSuite = unittest.TestSuite([unittest.defaultTestLoader.loadTestsFromName(file_string)]) + print('- running ' + file_string) + test_runner = unittest.TextTestRunner().run(testSuite) + if complete_success and not test_runner.wasSuccessful(): + complete_success = False + if not complete_success: + sys.exit(-1) diff --git a/tests/network_timezone_tests.py b/tests/network_timezone_tests.py index 22c7a09..09dd672 100644 --- a/tests/network_timezone_tests.py +++ b/tests/network_timezone_tests.py @@ -1,3 +1,8 @@ +import warnings +warnings.filterwarnings('ignore', module=r'.*fuz.*', message='.*Sequence.*') +warnings.filterwarnings('ignore', module=r'.*connectionpool.*', message='.*certificate verification.*') +warnings.filterwarnings('ignore', module=r'.*zoneinfo.*', message='.*no such file.*') + import unittest import test_lib as test import os.path diff --git a/tests/newznab_data/hydra1_caps.xml b/tests/newznab_data/hydra1_caps.xml new file mode 100644 index 0000000..7f928c1 --- /dev/null +++ b/tests/newznab_data/hydra1_caps.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/newznab_data/hydra2_caps.xml b/tests/newznab_data/hydra2_caps.xml new file mode 100644 index 0000000..7ea1a91 --- /dev/null +++ b/tests/newznab_data/hydra2_caps.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/newznab_data/newznab_caps.xml b/tests/newznab_data/newznab_caps.xml new file mode 100644 index 0000000..0656f2e --- /dev/null +++ b/tests/newznab_data/newznab_caps.xml @@ -0,0 +1,177 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/newznab_data/nntmux_caps.xml b/tests/newznab_data/nntmux_caps.xml new file mode 100644 index 0000000..8e7f536 --- /dev/null +++ b/tests/newznab_data/nntmux_caps.xml @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/newznab_data/nzedb_caps.xml b/tests/newznab_data/nzedb_caps.xml new file mode 100644 index 0000000..46ff8c3 --- /dev/null +++ b/tests/newznab_data/nzedb_caps.xml @@ -0,0 +1,93 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/newznab_data/spotweb_caps.xml b/tests/newznab_data/spotweb_caps.xml new file mode 100644 index 0000000..b649c09 --- /dev/null +++ b/tests/newznab_data/spotweb_caps.xml @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/newznab_tests.py b/tests/newznab_tests.py new file mode 100644 index 0000000..1124283 --- /dev/null +++ b/tests/newznab_tests.py @@ -0,0 +1,350 @@ +# coding=utf-8 +from __future__ import print_function +import warnings +warnings.filterwarnings('ignore', module=r'.*fuz.*', message='.*Sequence.*') +warnings.filterwarnings('ignore', module=r'.*dateutil.*', message='.*Unicode.*') + +import datetime +import os.path +import random +import sys +import test_lib as test +import unittest + +sys.path.insert(1, os.path.abspath('..')) +sys.path.insert(1, os.path.abspath('../lib')) + +try: + from lxml import etree +except ImportError: + try: + import xml.etree.cElementTree as etree + except ImportError: + import xml.etree.ElementTree as etree + +from lib.dateutil import parser +from sickbeard.indexers.indexer_config import * +from sickbeard.network_timezones import sb_timezone +from sickbeard.providers import newznab + +import sickbeard + +sickbeard.SYS_ENCODING = 'UTF-8' + +DEBUG = VERBOSE = False + +item_parse_test_cases = [ + (('Show.Name.S02E01.720p.HDTV.x264-GROUP', 'https://test.h/test/hhhh'), + ('Show.Name.S02E01.720p.HDTV.x264-GROUP', 'https://test.h/test/hhhh')), + (('Show.Name.S02E02.720p.HDTV.x264-GROUP-JUNK', 'https://test.h/test/hhhh'), + ('Show.Name.S02E02.720p.HDTV.x264-GROUP', 'https://test.h/test/hhhh')), + (('Show.Name.S02E03.720p.HDTV.x264-GROUP[JUNK]', 'https://test.h'), + ('Show.Name.S02E03.720p.HDTV.x264-GROUP', 'https://test.h')), + (('Show.Name.S02E04.720p.HDTV.x264-GROUP-JUNK-JUNK', 'https://test.h'), + ('Show.Name.S02E04.720p.HDTV.x264-GROUP', 'https://test.h')), + (('Show.Name.S02E05.720p.HDTV.x264-GROUP-JUNK[JUNK]', 'https://test.h'), + ('Show.Name.S02E05.720p.HDTV.x264-GROUP', 'https://test.h')), + ((u'Show.Name.S02E06.720p.HDTV.x264-GROUP-JUNK[JUNK帝]', 'https://test.h'), + (u'Show.Name.S02E06.720p.HDTV.x264-GROUP', 'https://test.h')), + ((u'Show.Name.S02E07-EpName帝.720p.HDTV.x264-GROUP帝-JUNK[JUNK帝]', 'https://test.h'), + (u'Show.Name.S02E07-EpName帝.720p.HDTV.x264-GROUP帝', 'https://test.h')), + ((u'[grp 帝] Show Name - 11 [1024x576 h264 AAC ger-sub][123456].mp4', 'https://test.h'), + (u'[grp.帝].Show.Name.-.11.[1024x576.h264.AAC.ger-sub][123456]', 'https://test.h')), +] + +size_test_cases = [ + ((1000, 'ad87987dadf7987987'), (1000, 'ad87987dadf7987987')), + ((1254105454, 'ffdds7766dgdzhghdzghdgg'), (1254105454, 'ffdds7766dgdzhghdzghdgg')), + ((-1, ''), (-1, None)) +] + +pubdate_test_cases = [ + 'Sat, 28 Jul 2018 07:33:06 +0000', + 'Sun, 10 Sep 2017 23:11:09 +0200' +] + +caps_test_cases = [ + { + 'name': 'newznab', + 'data_files': {'caps': 'newznab_caps.xml'}, + 'caps': { + INDEXER_TVDB: 'tvdbid', INDEXER_TVRAGE: 'rid', INDEXER_TVMAZE: 'tvmazeid', + newznab.NewznabConstants.SEARCH_EPISODE: 'ep', newznab.NewznabConstants.SEARCH_SEASON: 'season', + newznab.NewznabConstants.SEARCH_TEXT: 'q' + }, + 'cats': { + newznab.NewznabConstants.CAT_SD: ['5030'], newznab.NewznabConstants.CAT_SPORT: ['5060'], + newznab.NewznabConstants.CAT_ANIME: ['5070'], newznab.NewznabConstants.CAT_UHD: ['5045'], + newznab.NewznabConstants.CAT_HD: ['5040'] + }, + 'all_cats': [ + {'id': '5070', 'name': 'Anime'}, {'id': '5185', 'name': 'Docu HD'}, + {'id': '5180', 'name': 'Docu SD'}, {'id': '5020', 'name': 'Foreign'}, + {'id': '5040', 'name': 'HD'}, {'id': '5200', 'name': 'HEVC'}, + {'id': '5050', 'name': 'Other'}, {'id': '5030', 'name': 'SD'}, + {'id': '5060', 'name': 'Sport'}, {'id': '5045', 'name': 'UHD'} + ], + 'limits': 100, + 'server_type': newznab.NewznabConstants.SERVER_DEFAULT + }, { + 'name': 'nzedb', + 'data_files': {'caps': 'nzedb_caps.xml'}, + 'caps': { + INDEXER_TVDB: 'tvdbid', INDEXER_TVRAGE: 'rid', INDEXER_TVMAZE: 'tvmazeid', INDEXER_TRAKT: 'traktid', + INDEXER_IMDB: 'imdbid', INDEXER_TMDB: 'tmdbid', + newznab.NewznabConstants.SEARCH_EPISODE: 'ep', newznab.NewznabConstants.SEARCH_SEASON: 'season', + newznab.NewznabConstants.SEARCH_TEXT: 'q' + }, + 'cats': { + newznab.NewznabConstants.CAT_SD: ['5030'], newznab.NewznabConstants.CAT_SPORT: ['5060'], + newznab.NewznabConstants.CAT_ANIME: ['5070'], newznab.NewznabConstants.CAT_UHD: ['5045'], + newznab.NewznabConstants.CAT_HD: ['5040'], newznab.NewznabConstants.CAT_WEBDL: ['5010'] + }, + 'all_cats': [ + {'id': '5070', 'name': 'Anime'}, {'id': '5080', 'name': 'Documentary'}, + {'id': '5020', 'name': 'Foreign'}, {'id': '5040', 'name': 'HD'}, + {'id': '5999', 'name': 'Other'}, {'id': '5030', 'name': 'SD'}, + {'id': '5060', 'name': 'Sport'}, {'id': '5045', 'name': 'UHD'}, + {'id': '5010', 'name': 'WEB-DL'} + ], + 'limits': 100, + 'server_type': newznab.NewznabConstants.SERVER_DEFAULT + }, { + 'name': 'nntmux', + 'data_files': {'caps': 'nntmux_caps.xml'}, + 'caps': { + INDEXER_TVDB: 'tvdbid', INDEXER_TVRAGE: 'rid', INDEXER_TVMAZE: 'tvmazeid', INDEXER_TRAKT: 'traktid', + INDEXER_IMDB: 'imdbid', INDEXER_TMDB: 'tmdbid', + newznab.NewznabConstants.SEARCH_EPISODE: 'ep', newznab.NewznabConstants.SEARCH_SEASON: 'season', + newznab.NewznabConstants.SEARCH_TEXT: 'q' + }, + 'cats': { + newznab.NewznabConstants.CAT_SD: ['5030'], newznab.NewznabConstants.CAT_SPORT: ['5060'], + newznab.NewznabConstants.CAT_ANIME: ['5070'], newznab.NewznabConstants.CAT_UHD: ['5045'], + newznab.NewznabConstants.CAT_HD: ['5040'], newznab.NewznabConstants.CAT_WEBDL: ['5010'] + }, + 'all_cats': [ + {'id': '5070', 'name': 'Anime'}, {'id': '5080', 'name': 'Documentary'}, + {'id': '5020', 'name': 'Foreign'}, {'id': '5040', 'name': 'HD'}, {'id': '5999', 'name': 'Other'}, + {'id': '5030', 'name': 'SD'}, {'id': '5060', 'name': 'Sport'}, {'id': '5045', 'name': 'UHD'}, + {'id': '5010', 'name': 'WEB-DL'}, {'id': '5090', 'name': 'X265'} + ], + 'limits': 100, + 'server_type': newznab.NewznabConstants.SERVER_DEFAULT + }, { + 'name': 'spotweb', + 'data_files': {'caps': 'spotweb_caps.xml'}, + 'caps': { + INDEXER_TVRAGE: 'rid', INDEXER_TVMAZE: 'tvmazeid', + newznab.NewznabConstants.SEARCH_EPISODE: 'ep', newznab.NewznabConstants.SEARCH_SEASON: 'season', + newznab.NewznabConstants.SEARCH_TEXT: 'q' + }, + 'cats': { + newznab.NewznabConstants.CAT_SD: ['5030'], newznab.NewznabConstants.CAT_SPORT: ['5060'], + newznab.NewznabConstants.CAT_ANIME: ['5070'], newznab.NewznabConstants.CAT_HD: ['5040'] + }, + 'all_cats': [ + {'id': '5020', 'name': 'Foreign'}, {'id': '5030', 'name': 'SD'}, {'id': '5040', 'name': 'HD'}, + {'id': '5050', 'name': 'Other'}, {'id': '5060', 'name': 'Sport'}, {'id': '5070', 'name': 'Anime'} + ], + 'limits': 500, + 'server_type': newznab.NewznabConstants.SERVER_SPOTWEB + }, { + 'name': 'NZBHydra', + 'data_files': {'caps': 'hydra1_caps.xml'}, + 'caps': { + INDEXER_TVDB: 'tvdbid', INDEXER_TVRAGE: 'rid', + newznab.NewznabConstants.SEARCH_EPISODE: 'ep', newznab.NewznabConstants.SEARCH_SEASON: 'season', + newznab.NewznabConstants.SEARCH_TEXT: 'q' + }, + 'cats': { + newznab.NewznabConstants.CAT_SD: ['5030'], newznab.NewznabConstants.CAT_SPORT: ['5060'], + newznab.NewznabConstants.CAT_ANIME: ['5070'], newznab.NewznabConstants.CAT_HD: ['5040'] + }, + 'all_cats': [ + {'id': '5020', 'name': 'Foreign'}, {'id': '5030', 'name': 'SD'}, {'id': '5040', 'name': 'HD'}, + {'id': '5050', 'name': 'Other'}, {'id': '5060', 'name': 'Sport'}, {'id': '5070', 'name': 'Anime'}, + {'id': '5080', 'name': 'Documentary'} + ], + 'limits': 100, + 'server_type': newznab.NewznabConstants.SERVER_HYDRA1 + }, { + 'name': 'NZBHydra 2', + 'data_files': {'caps': 'hydra2_caps.xml'}, + 'caps': { + INDEXER_TVDB: 'tvdbid', INDEXER_TVRAGE: 'rid', INDEXER_TVMAZE: 'tvmazeid', INDEXER_TRAKT: 'traktid', + newznab.NewznabConstants.SEARCH_EPISODE: 'ep', newznab.NewznabConstants.SEARCH_SEASON: 'season', + newznab.NewznabConstants.SEARCH_TEXT: 'q' + }, + 'cats': { + newznab.NewznabConstants.CAT_SD: ['5030'], newznab.NewznabConstants.CAT_SPORT: ['5060'], + newznab.NewznabConstants.CAT_ANIME: ['5070'], newznab.NewznabConstants.CAT_HD: ['5040'] + }, + 'all_cats': [ + {'id': '5070', 'name': 'Anime'}, {'id': '5040', 'name': 'TV HD'}, {'id': '5030', 'name': 'TV SD'} + ], + 'limits': 100, + 'server_type': newznab.NewznabConstants.SERVER_HYDRA2 + }, +] + + +class FakeNewznabProvider(newznab.NewznabProvider): + + def __init__(self, *args, **kwargs): + self.FILEDIR = os.path.join(test.TESTDIR, 'newznab_data') + self._data_files_dict = ({}, kwargs['data_files_dict'])[isinstance(kwargs.get('data_files_dict'), dict)] + if 'data_files_dict' in kwargs: + del kwargs['data_files_dict'] + super(FakeNewznabProvider, self).__init__(*args, **kwargs) + + def _read_data(self, filename): + try: + f = os.path.join(self.FILEDIR, filename) + if os.path.isfile(f): + with open(f, 'r') as d: + return d.read().decode('UTF-8') + except (StandardError, Exception): + return + + # simulate Data from provider + def get_url(self, url, skip_auth=False, use_tmr_limit=True, *args, **kwargs): + data = None + if '/api?' in url: + api_parameter = url[url.find('/api?') + 5:] + if 't=caps' in api_parameter and 'caps' in self._data_files_dict: + data = self._read_data(self._data_files_dict['caps']) + return data + + +class BasicTests(test.SickbeardTestDBCase): + + ns = {'newznab': 'http://www.newznab.com/DTD/2010/feeds/attributes/', 'atom': 'http://www.w3.org/2005/Atom'} + ns_parsed = dict((k, '{%s}' % v) for (k, v) in ns.items()) + + @staticmethod + def _create_item(title, link, size=-1, uuid='', ids=None, pubdate=None): + item = etree.Element('item', nsmap=BasicTests.ns) + title_item = etree.Element('title') + title_item.text = title + link_item = etree.Element('link') + link_item.text = link + item.append(title_item) + item.append(link_item) + if -1 != size: + size_item = etree.Element('{%s}attr' % BasicTests.ns['newznab'], nsmap=BasicTests.ns) + size_item.set('name', 'size') + size_item.set('value', '%s' % size) + item.append(size_item) + if uuid: + uuid_item = etree.Element('{%s}attr' % BasicTests.ns['newznab'], nsmap=BasicTests.ns) + uuid_item.set('name', 'guid') + uuid_item.set('value', '%s' % uuid) + item.append(uuid_item) + if ids: + for a, b in ids.iteritems(): + ids_item = etree.Element('{%s}attr' % BasicTests.ns['newznab'], nsmap=BasicTests.ns) + ids_item.set('name', a) + ids_item.set('value', '%s' % b) + item.append(ids_item) + if pubdate: + pubdate_item = etree.Element('pubDate') + pubdate_item.text = pubdate + item.append(pubdate_item) + return item + + def test_title_and_url(self): + + if VERBOSE: + print('Running tests') + + newznab_provider = newznab.NewznabProvider('test', '') + + for cur_test, cur_expected in item_parse_test_cases: + item = self._create_item(cur_test[0], cur_test[1]) + result = newznab_provider._title_and_url(item) + self.assertEqual(cur_expected, result) + + def test_get_size_uid(self): + + newznab_provider = newznab.NewznabProvider('test', '') + + for cur_test, cur_expected in size_test_cases: + item = self._create_item('Show.Name.S01E01.x264-Group', 'http://test.h', cur_test[0], cur_test[1]) + result = newznab_provider.get_size_uid(item, name_space=BasicTests.ns_parsed) + self.assertEqual(cur_expected, result) + + def test_parse_ids(self): + ids_test_cases = [] + for k in newznab.NewznabConstants.providerToIndexerMapping.iterkeys(): + rand_id = random.randrange(1, 99999999) + ids_test_cases.append(({k: rand_id}, {newznab.NewznabConstants.providerToIndexerMapping[k]: rand_id})) + + all_case = {} + all_case_ex = {} + for k in newznab.NewznabConstants.providerToIndexerMapping.iterkeys(): + rand_id = random.randrange(1, 99999999) + all_case.update({k: rand_id}) + all_case_ex.update({newznab.NewznabConstants.providerToIndexerMapping[k]: rand_id}) + + ids_test_cases.append((all_case, all_case_ex)) + + newznab_provider = newznab.NewznabProvider('test', '') + + for cur_test, cur_expected in ids_test_cases: + item = self._create_item('Show.Name.S01E01.x264-Group', 'https://test.h', ids=cur_test) + result = newznab_provider.cache.parse_ids(item, BasicTests.ns_parsed) + self.assertEqual(cur_expected, result) + + @staticmethod + def _parse_pub_date(date_str): + parsed_date = None + try: + if date_str: + p = parser.parse(date_str, fuzzy=True) + try: + p = p.astimezone(sb_timezone) + except (StandardError, Exception): + pass + if isinstance(p, datetime.datetime): + parsed_date = p.replace(tzinfo=None) + except (StandardError, Exception): + pass + + return parsed_date + + def test_parse_pub_date(self): + newznab_provider = newznab.NewznabProvider('test', '') + + for cur_test in pubdate_test_cases: + item = self._create_item('Show.Name.S01E01.x264-Group', 'https://test.h', pubdate=cur_test) + result = newznab_provider._parse_pub_date(item) + cur_expected = self._parse_pub_date(cur_test) + self.assertEqual(cur_expected, result) + + +class FakeProviderTests(test.SickbeardTestDBCase): + + def test_caps(self): + self.longMessage = True + for cur_test in caps_test_cases: + newznab_provider = FakeNewznabProvider('test', 'https://fake.fake/', data_files_dict=cur_test['data_files']) + newznab_provider.enabled = True + newznab_provider.get_caps() + msg = 'Test case: %s' % cur_test['name'] + self.assertEqual(cur_test['server_type'], newznab_provider.server_type, msg=msg) + self.assertEqual(cur_test['limits'], newznab_provider.limits, msg=msg) + self.assertEqual(cur_test['caps'], newznab_provider.caps, msg=msg) + self.assertEqual(cur_test['cats'], newznab_provider.cats, msg=msg) + self.assertEqual(cur_test['all_cats'], newznab_provider.all_cats, msg=msg) + + +if __name__ == '__main__': + if len(sys.argv) > 1: + suite = unittest.TestLoader().loadTestsFromName('newznab_tests.BasicTests.test_' + sys.argv[1]) + else: + suite = unittest.TestLoader().loadTestsFromTestCase(BasicTests) + unittest.TextTestRunner(verbosity=2).run(suite) + + suite = unittest.TestLoader().loadTestsFromTestCase(FakeProviderTests) + unittest.TextTestRunner(verbosity=2).run(FakeNewznabProvider) diff --git a/tests/scene_helpers_tests.py b/tests/scene_helpers_tests.py index ebce80f..dbed765 100644 --- a/tests/scene_helpers_tests.py +++ b/tests/scene_helpers_tests.py @@ -1,4 +1,7 @@ # coding=utf-8 +import warnings +warnings.filterwarnings('ignore', module=r'.*fuz.*', message='.*Sequence.*') +warnings.filterwarnings('ignore', module=r'.*connectionpool.*', message='.*certificate verification.*') import unittest import test_lib as test diff --git a/tests/search_tests.py b/tests/search_tests.py index c4a2aad..c852ee7 100644 --- a/tests/search_tests.py +++ b/tests/search_tests.py @@ -1,41 +1,45 @@ -import unittest - -from sickbeard import properFinder - -import sickbeard -import test_lib as test - -sickbeard.SYS_ENCODING = 'UTF-8' - - -class ProperTests(test.SickbeardTestDBCase): - def check_webdl_type(self, cases): - for c in cases: - self.assertEqual(properFinder.get_webdl_type(*c[0]), c[1]) - - def check_get_codec(self, cases): - for c in cases: - self.assertEqual(properFinder._get_codec(c[0]), c[1]) - - def test_webdl_type(self): - self.check_webdl_type([ - (('1080p.WEB.x264', 'The.Show.Name.S04E10.1080p.WEB.x264-GROUP'), 'webrip'), - (('720p.WEB-DL.DD5.1.H.264', 'The.Show.Name.720p.WEB-DL.DD5.1.H.264-GROUP'), 'webdl'), - (('1080p.AMZN.WEB-DL.DD5.1.H.264', 'The.Show.Name.1080p.AMZN.WEB-DL.DD5.1.H.264-GROUP'), 'Amazon'), - ]) - - def test_get_codec(self): - self.check_get_codec([ - ('1080p.WEB.x264', '264'), - ('720p.WEB.h264', '264'), - ('HDTV.XviD', 'xvid'), - ('720p.HEVC.x265', 'hevc'), - ('1080p.HEVC.AC3', 'hevc'), - ('10Bit.1080p.DD5.1.H.265', 'hevc'), - ('720p.DD5.1.Widescreen.x265', 'hevc'), - ]) - - -if __name__ == '__main__': - suite = unittest.TestLoader().loadTestsFromTestCase(ProperTests) - unittest.TextTestRunner(verbosity=2).run(suite) +import warnings +warnings.filterwarnings('ignore', module=r'.*fuz.*', message='.*Sequence.*') +warnings.filterwarnings('ignore', module=r'.*connectionpool.*', message='.*certificate verification.*') + +import unittest + +from sickbeard import properFinder + +import sickbeard +import test_lib as test + +sickbeard.SYS_ENCODING = 'UTF-8' + + +class ProperTests(test.SickbeardTestDBCase): + def check_webdl_type(self, cases): + for c in cases: + self.assertEqual(properFinder.get_webdl_type(*c[0]), c[1]) + + def check_get_codec(self, cases): + for c in cases: + self.assertEqual(properFinder._get_codec(c[0]), c[1]) + + def test_webdl_type(self): + self.check_webdl_type([ + (('1080p.WEB.x264', 'The.Show.Name.S04E10.1080p.WEB.x264-GROUP'), 'webrip'), + (('720p.WEB-DL.DD5.1.H.264', 'The.Show.Name.720p.WEB-DL.DD5.1.H.264-GROUP'), 'webdl'), + (('1080p.AMZN.WEB-DL.DD5.1.H.264', 'The.Show.Name.1080p.AMZN.WEB-DL.DD5.1.H.264-GROUP'), 'Amazon'), + ]) + + def test_get_codec(self): + self.check_get_codec([ + ('1080p.WEB.x264', '264'), + ('720p.WEB.h264', '264'), + ('HDTV.XviD', 'xvid'), + ('720p.HEVC.x265', 'hevc'), + ('1080p.HEVC.AC3', 'hevc'), + ('10Bit.1080p.DD5.1.H.265', 'hevc'), + ('720p.DD5.1.Widescreen.x265', 'hevc'), + ]) + + +if __name__ == '__main__': + suite = unittest.TestLoader().loadTestsFromTestCase(ProperTests) + unittest.TextTestRunner(verbosity=2).run(suite)