From 7738a3143eb1a1deb7aa579fd99702b9743d67c1 Mon Sep 17 00:00:00 2001 From: JackDandy Date: Mon, 16 Dec 2019 04:06:37 +0000 Subject: [PATCH] Fix TL provider - replace user/pass with digest auth method. Change improve TL and IPT provider recent search performance to process new items since the previous cycle. Change log a tip for TL and IPT users who have not improved on the default site setting "Torrents per page". Add recommended.txt file with recommended libs that can be installed via: python -m pip install -r recommended.txt Fix saving .nfo metadata where the file name contains unicode on certain Linux OS configurations. --- CHANGES.md | 11 +++++- recommended.txt | 4 ++ sickbeard/helpers.py | 8 ++-- sickbeard/providers/generic.py | 16 ++++++-- sickbeard/providers/iptorrents.py | 57 +++++++++++++++++++++------ sickbeard/providers/torrentday.py | 11 ++---- sickbeard/providers/torrentleech.py | 77 +++++++++++++++++++++++++++++-------- 7 files changed, 141 insertions(+), 43 deletions(-) create mode 100644 recommended.txt diff --git a/CHANGES.md b/CHANGES.md index 1806374..38cfa44 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,13 @@ -### 0.20.12 (2019-12-09 16:30:00 UTC) +### 0.20.13 (2019-12-16 04:00:00 UTC) + +* Fix TL provider - replace user/pass with digest auth method +* Change improve TL and IPT provider recent search performance to process new items since the previous cycle +* Change log a tip for TL and IPT users who have not improved on the default site setting "Torrents per page" +* Add recommended.txt file with recommended libs that can be installed via: python -m pip install -r recommended.txt +* Fix saving .nfo metadata where the file name contains unicode on certain Linux OS configurations + + +### 0.20.12 (2019-12-09 16:30:00 UTC) * Fix using multiple hostnames with config General/Interface/"Allowed browser hostnames" * Add config General/Interface/"Allow IP use for connections" diff --git a/recommended.txt b/recommended.txt new file mode 100644 index 0000000..f292a1d --- /dev/null +++ b/recommended.txt @@ -0,0 +1,4 @@ +lxml>=4.4.2 +regex>=2019.11.1 +python-Levenshtein>=0.12.0 +scandir>=1.10.0; python_version < '3.0' diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index 2ddb6df..9c2a7c0 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -1754,7 +1754,7 @@ def write_file(filepath, data, raw=False, xmltree=False, utf8=False, raise_excep if make_dirs(ek.ek(os.path.dirname, filepath), False): try: if raw: - with io.FileIO(filepath, 'wb') as fh: + with ek.ek(io.FileIO, filepath, 'wb') as fh: for chunk in data.iter_content(chunk_size=1024): if chunk: fh.write(chunk) @@ -1764,17 +1764,17 @@ def write_file(filepath, data, raw=False, xmltree=False, utf8=False, raise_excep w_mode = 'w' if utf8: w_mode = 'a' - with io.FileIO(filepath, 'wb') as fh: + with ek.ek(io.FileIO, filepath, 'wb') as fh: fh.write(codecs.BOM_UTF8) if xmltree: - with io.FileIO(filepath, w_mode) as fh: + with ek.ek(io.FileIO, filepath, w_mode) as fh: if utf8: data.write(fh, encoding='utf-8') else: data.write(fh) else: - with io.FileIO(filepath, w_mode) as fh: + with ek.ek(io.FileIO, filepath, w_mode) as fh: fh.write(data) chmodAsParent(filepath) diff --git a/sickbeard/providers/generic.py b/sickbeard/providers/generic.py index d3125c2..d37d85d 100644 --- a/sickbeard/providers/generic.py +++ b/sickbeard/providers/generic.py @@ -29,7 +29,7 @@ import time import urlparse import threading import socket -from urllib import quote_plus +from urllib import quote, quote_plus import zlib from base64 import b16encode, b32decode, b64decode @@ -805,7 +805,14 @@ class GenericProvider(object): def _link(self, url, url_tmpl=None): - url = url and str(url).strip().replace('&', '&') or '' + if url: + try: + url = url.encode('utf-8') + except (BaseException, Exception): + pass + url = quote(url).strip().replace('&', '&') + if not url: + url = '' return url if re.match('(?i)(https?://|magnet:)', url) \ else (url_tmpl or self.urls.get('get', (getattr(self, 'url', '') or getattr(self, 'url_base')) + '%s')) % url.lstrip('/') @@ -1124,7 +1131,7 @@ class GenericProvider(object): """ return '' - def _log_search(self, mode='Cache', count=0, url='url missing'): + def _log_search(self, mode='Cache', count=0, url='url missing', log_setting_hint=False): """ Simple function to log the result of a search types except propers :param count: count of successfully processed items @@ -1133,6 +1140,9 @@ class GenericProvider(object): if 'Propers' != mode: self.log_result(mode, count, url) + if log_setting_hint: + logger.log('Perfomance tip: change "Torrents per Page" to 100 at the site/Settings page') + def log_result(self, mode='Cache', count=0, url='url missing'): """ Simple function to log the result of any search diff --git a/sickbeard/providers/iptorrents.py b/sickbeard/providers/iptorrents.py index 38f6f35..5798943 100644 --- a/sickbeard/providers/iptorrents.py +++ b/sickbeard/providers/iptorrents.py @@ -39,7 +39,7 @@ class IPTorrentsProvider(generic.TorrentProvider): 'RqHEa', 'LvEoDc0', 'Zvex2', 'LuF2', 'NXdu Vn', 'XZwQxeWY1', 'Yu42bzJ', 'tgG92']], ]]]) - self.url_vars = {'login': 't', 'search': 't?%s;q=%s;qf=ti%s%s#torrents'} + self.url_vars = {'login': 't', 'search': 't?%s;q=%s;qf=ti%s%s;p=%s#torrents'} self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'login': '%(home)s%(vars)s', 'search': '%(home)s%(vars)s'} @@ -54,7 +54,7 @@ class IPTorrentsProvider(generic.TorrentProvider): return super(IPTorrentsProvider, self)._authorised( logged_in=(lambda y='': all( ['IPTorrents' in y, 'type="password"' not in y[0:2048], self.has_all_cookies()] + - [(self.session.cookies.get(x) or 'sg!no!pw') in self.digest for x in 'uid', 'pass'])), + [(self.session.cookies.get(x, domain='') or 'sg!no!pw') in self.digest for x in ('uid', 'pass')])), failed_msg=(lambda y=None: u'Invalid cookie details for %s. Check settings')) @staticmethod @@ -67,22 +67,43 @@ class IPTorrentsProvider(generic.TorrentProvider): if not self._authorised(): return results - items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} + last_recent_search = self.last_recent_search + last_recent_search = '' if not last_recent_search else last_recent_search.replace('id-', '') - rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': 'download'}.items()) for mode in search_params.keys(): + urls = [] for search_string in search_params[mode]: + urls += [[]] search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string - # URL with 50 tv-show results, or max 150 if adjusted in IPTorrents profile - search_url = self.urls['search'] % ( - self._categories_string(mode, '%s', ';'), search_string, - (';free', '')[not self.freeleech], (';o=seeders', '')['Cache' == mode]) + for page in range((3, 5)['Cache' == mode])[1:]: + # URL with 50 tv-show results, or max 150 if adjusted in IPTorrents profile + urls[-1] += [self.urls['search'] % ( + self._categories_string(mode, '%s', ';'), search_string, + (';free', '')[not self.freeleech], (';o=seeders', '')['Cache' == mode], page)] + results += self._search_urls(mode, last_recent_search, urls) + last_recent_search = '' + + return results + + def _search_urls(self, mode, last_recent_search, urls): + + results = [] + items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} + + rc = dict((k, re.compile('(?i)' + v)) for (k, v) in dict( + info='detail', get='download', id=r'download.*?/([\d]+)').items()) + lrs_found = False + lrs_new = True + for search_urls in urls: # this intentionally iterates once to preserve indentation + for search_url in search_urls: html = self.get_url(search_url) if self.should_skip(): return results cnt = len(items[mode]) + cnt_search = 0 + log_settings_hint = False try: if not html or self._has_no_results(html): raise generic.HaltParseException @@ -94,23 +115,33 @@ class IPTorrentsProvider(generic.TorrentProvider): if 2 > len(tbl_rows): raise generic.HaltParseException + if 'Cache' == mode and 100 > len(tbl_rows): + log_settings_hint = True + head = None for tr in tbl_rows[1:]: cells = tr.find_all('td') if 5 > len(cells): continue + cnt_search += 1 try: head = head if None is not head else self._header_row( tr, header_strip='(?i)(?:leechers|seeders|size);') seeders, leechers = [tryInt(tr.find('td', class_='t_' + x).get_text().strip()) - for x in 'seeders', 'leechers'] + for x in ('seeders', 'leechers')] if self._reject_item(seeders, leechers): continue + dl = tr.find('a', href=rc['get'])['href'] + dl_id = rc['id'].findall(dl)[0] + lrs_found = dl_id == last_recent_search + if lrs_found: + break + info = tr.find('a', href=rc['info']) title = (info.attrs.get('title') or info.get_text()).strip() size = cells[head['size']].get_text().strip() - download_url = self._link(tr.find('a', href=rc['get'])['href']) + download_url = self._link(dl) except (AttributeError, TypeError, ValueError): continue @@ -121,7 +152,11 @@ class IPTorrentsProvider(generic.TorrentProvider): pass except (BaseException, Exception): logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) - self._log_search(mode, len(items[mode]) - cnt, search_url) + self._log_search(mode, len(items[mode]) - cnt, search_url, log_settings_hint) + + if self.is_search_finished(mode, items, cnt_search, rc['id'], last_recent_search, lrs_new, lrs_found): + break + lrs_new = False results = self._sort_seeding(mode, results + items[mode]) diff --git a/sickbeard/providers/torrentday.py b/sickbeard/providers/torrentday.py index 5b2de68..a780eff 100644 --- a/sickbeard/providers/torrentday.py +++ b/sickbeard/providers/torrentday.py @@ -22,7 +22,6 @@ import time from . import generic from sickbeard.bs4_parser import BS4Parser from sickbeard.helpers import tryInt, anon_url -from sickbeard import logger class TorrentDayProvider(generic.TorrentProvider): @@ -54,7 +53,7 @@ class TorrentDayProvider(generic.TorrentProvider): return super(TorrentDayProvider, self)._authorised( logged_in=(lambda y='': all( ['RSS URL' in y, self.has_all_cookies()] + - [(self.session.cookies.get(x) or 'sg!no!pw') in self.digest for x in 'uid', 'pass'])), + [(self.session.cookies.get(x, domain='') or 'sg!no!pw') in self.digest for x in ('uid', 'pass')])), failed_msg=(lambda y=None: u'Invalid cookie details for %s. Check settings')) @staticmethod @@ -88,7 +87,7 @@ class TorrentDayProvider(generic.TorrentProvider): items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} - rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'get': 'download', 'id': r'download.*?/([\d]+)'}.items()) + rc = dict((k, re.compile('(?i)' + v)) for (k, v) in dict(get='download', id=r'download.*?/([\d]+)').items()) lrs_found = False lrs_new = True for search_urls in urls: # this intentionally iterates once to preserve indentation @@ -124,7 +123,7 @@ class TorrentDayProvider(generic.TorrentProvider): head = head if None is not head else self._header_row( tr, header_strip='(?i)(?:leechers|seeders|size);') seeders, leechers, size = [tryInt(n, n) for n in [ - cells[head[x]].get_text().strip() for x in 'seed', 'leech', 'size']] + cells[head[x]].get_text().strip() for x in ('seed', 'leech', 'size')]] if self._reject_item(seeders, leechers): continue @@ -146,9 +145,7 @@ class TorrentDayProvider(generic.TorrentProvider): except (BaseException, Exception): time.sleep(1.1) - self._log_search(mode, len(items[mode]) - cnt, search_url) - if log_settings_hint: - logger.log('Perfomance tip: change "Torrents per Page" to 100 at the TD site/Settings page') + self._log_search(mode, len(items[mode]) - cnt, search_url, log_settings_hint) if self.is_search_finished(mode, items, cnt_search, rc['id'], last_recent_search, lrs_new, lrs_found): break diff --git a/sickbeard/providers/torrentleech.py b/sickbeard/providers/torrentleech.py index 1c54823..822084f 100644 --- a/sickbeard/providers/torrentleech.py +++ b/sickbeard/providers/torrentleech.py @@ -27,24 +27,29 @@ from lib.unidecode import unidecode class TorrentLeechProvider(generic.TorrentProvider): def __init__(self): - generic.TorrentProvider.__init__(self, 'TorrentLeech', cache_update_freq=15) + generic.TorrentProvider.__init__(self, 'TorrentLeech') self.url_base = 'https://v4.torrentleech.org/' self.urls = {'config_provider_home_uri': self.url_base, - 'login_action': self.url_base, - 'browse': self.url_base + 'torrents/browse/index/categories/%(cats)s', - 'search': self.url_base + 'torrents/browse/index/query/%(query)s/categories/%(cats)s'} + 'login': self.url_base, + 'browse': self.url_base + 'torrents/browse/index/categories/%(cats)s/%(x)s', + 'search': self.url_base + 'torrents/browse/index/query/%(query)s/categories/%(cats)s/%(x)s'} self.categories = {'shows': [2, 26, 27, 32], 'anime': [7, 34, 35]} self.url = self.urls['config_provider_home_uri'] - - self.username, self.password, self.minseed, self.minleech = 4 * [None] + self.digest, self.minseed, self.minleech, self.freeleech = 4 * [None] def _authorised(self, **kwargs): - return super(TorrentLeechProvider, self)._authorised(logged_in=(lambda y=None: self.has_all_cookies(pre='tl')), - post_params={'remember_me': 'on', 'form_tmpl': True}) + return super(TorrentLeechProvider, self)._authorised( + logged_in=(lambda y='': all( + ['TorrentLeech' in y, 'type="password"' not in y[0:4096], self.has_all_cookies(pre='tl')])), + failed_msg=(lambda y=None: u'Invalid cookie details for %s. Check settings')) + + @staticmethod + def _has_signature(data=None): + return generic.TorrentProvider._has_signature(data) or (data and re.search(r'(?i) len(tbl_rows): raise generic.HaltParseException + if 'Cache' == mode and 100 > len(tbl_rows): + log_settings_hint = True + head = None for tr in tbl_rows[1:]: cells = tr.find_all('td') if 6 > len(cells): continue + cnt_search += 1 try: head = head if None is not head else self._header_row(tr) seeders, leechers = [tryInt(n) for n in [ - tr.find('td', class_=x).get_text().strip() for x in 'seeders', 'leechers']] + tr.find('td', class_=x).get_text().strip() for x in ('seeders', 'leechers')]] if self._reject_item(seeders, leechers): continue + dl = tr.find('a', href=rc['get'])['href'] + dl_id = rc['id'].findall(dl)[0] + lrs_found = dl_id == last_recent_search + if lrs_found: + break + info = tr.find('td', class_='name').a title = (info.attrs.get('title') or info.get_text()).strip() size = cells[head['size']].get_text().strip() - download_url = self._link(tr.find('a', href=rc['get'])['href']) + download_url = self._link(dl) except (AttributeError, TypeError, ValueError): continue @@ -103,7 +139,11 @@ class TorrentLeechProvider(generic.TorrentProvider): pass except (BaseException, Exception): logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) - self._log_search(mode, len(items[mode]) - cnt, search_url) + self._log_search(mode, len(items[mode]) - cnt, search_url, log_settings_hint) + + if self.is_search_finished(mode, items, cnt_search, rc['id'], last_recent_search, lrs_new, lrs_found): + break + lrs_new = False results = self._sort_seeding(mode, results + items[mode]) @@ -113,5 +153,8 @@ class TorrentLeechProvider(generic.TorrentProvider): return super(TorrentLeechProvider, self)._episode_strings(ep_obj, sep_date='|', **kwargs) + def ui_string(self, key): + return 'torrentleech_digest' == key and self._valid_home() and 'use... \'tluid=xx; tlpass=yy\'' or '' + provider = TorrentLeechProvider()