Fix TL provider - replace user/pass with digest auth method.

Change improve TL and IPT provider recent search performance to process new items since the previous cycle.
Change log a tip for TL and IPT users who have not improved on the default site setting "Torrents per page".
Add recommended.txt file with recommended libs that can be installed via: python -m pip install -r recommended.txt
Fix saving .nfo metadata where the file name contains unicode on certain Linux OS configurations.

CHANGES.md

@@ -1,4 +1,13 @@
-### 0.20.12 (2019-12-09 16:30:00 UTC)
+### 0.20.13 (2019-12-16 04:00:00 UTC)
+
+* Fix TL provider - replace user/pass with digest auth method
+* Change improve TL and IPT provider recent search performance to process new items since the previous cycle
+* Change log a tip for TL and IPT users who have not improved on the default site setting "Torrents per page"
+* Add recommended.txt file with recommended libs that can be installed via: python -m pip install -r recommended.txt
+* Fix saving .nfo metadata where the file name contains unicode on certain Linux OS configurations
+
+
+### 0.20.12 (2019-12-09 16:30:00 UTC)
 
 * Fix using multiple hostnames with config General/Interface/"Allowed browser hostnames"
 * Add config General/Interface/"Allow IP use for connections"

recommended.txt

@@ -0,0 +1,4 @@
+lxml>=4.4.2
+regex>=2019.11.1
+python-Levenshtein>=0.12.0
+scandir>=1.10.0; python_version < '3.0'

sickbeard/helpers.py

@@ -1754,7 +1754,7 @@ def write_file(filepath, data, raw=False, xmltree=False, utf8=False, raise_excep
     if make_dirs(ek.ek(os.path.dirname, filepath), False):
         try:
             if raw:
-                with io.FileIO(filepath, 'wb') as fh:
+                with ek.ek(io.FileIO, filepath, 'wb') as fh:
                     for chunk in data.iter_content(chunk_size=1024):
                         if chunk:
                             fh.write(chunk)
@@ -1764,17 +1764,17 @@ def write_file(filepath, data, raw=False, xmltree=False, utf8=False, raise_excep
                 w_mode = 'w'
                 if utf8:
                     w_mode = 'a'
-                    with io.FileIO(filepath, 'wb') as fh:
+                    with ek.ek(io.FileIO, filepath, 'wb') as fh:
                         fh.write(codecs.BOM_UTF8)
 
                 if xmltree:
-                    with io.FileIO(filepath, w_mode) as fh:
+                    with ek.ek(io.FileIO, filepath, w_mode) as fh:
                         if utf8:
                             data.write(fh, encoding='utf-8')
                         else:
                             data.write(fh)
                 else:
-                    with io.FileIO(filepath, w_mode) as fh:
+                    with ek.ek(io.FileIO, filepath, w_mode) as fh:
                         fh.write(data)
 
             chmodAsParent(filepath)

sickbeard/providers/generic.py

@@ -29,7 +29,7 @@ import time
 import urlparse
 import threading
 import socket
-from urllib import quote_plus
+from urllib import quote, quote_plus
 import zlib
 from base64 import b16encode, b32decode, b64decode
 
@@ -805,7 +805,14 @@ class GenericProvider(object):
 
     def _link(self, url, url_tmpl=None):
 
-        url = url and str(url).strip().replace('&', '&') or ''
+        if url:
+            try:
+                url = url.encode('utf-8')
+            except (BaseException, Exception):
+                pass
+            url = quote(url).strip().replace('&', '&')
+        if not url:
+            url = ''
         return url if re.match('(?i)(https?://|magnet:)', url) \
             else (url_tmpl or self.urls.get('get', (getattr(self, 'url', '') or
                                                     getattr(self, 'url_base')) + '%s')) % url.lstrip('/')
@@ -1124,7 +1131,7 @@ class GenericProvider(object):
         """
         return ''
 
-    def _log_search(self, mode='Cache', count=0, url='url missing'):
+    def _log_search(self, mode='Cache', count=0, url='url missing', log_setting_hint=False):
         """
         Simple function to log the result of a search types except propers
         :param count: count of successfully processed items
@@ -1133,6 +1140,9 @@ class GenericProvider(object):
         if 'Propers' != mode:
             self.log_result(mode, count, url)
 
+        if log_setting_hint:
+            logger.log('Perfomance tip: change "Torrents per Page" to 100 at the site/Settings page')
+
     def log_result(self, mode='Cache', count=0, url='url missing'):
         """
         Simple function to log the result of any search

sickbeard/providers/iptorrents.py

@@ -39,7 +39,7 @@ class IPTorrentsProvider(generic.TorrentProvider):
                                  'RqHEa', 'LvEoDc0', 'Zvex2', 'LuF2', 'NXdu Vn', 'XZwQxeWY1', 'Yu42bzJ', 'tgG92']],
                          ]]])
 
-        self.url_vars = {'login': 't', 'search': 't?%s;q=%s;qf=ti%s%s#torrents'}
+        self.url_vars = {'login': 't', 'search': 't?%s;q=%s;qf=ti%s%s;p=%s#torrents'}
         self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'login': '%(home)s%(vars)s',
                          'search': '%(home)s%(vars)s'}
 
@@ -54,7 +54,7 @@ class IPTorrentsProvider(generic.TorrentProvider):
         return super(IPTorrentsProvider, self)._authorised(
             logged_in=(lambda y='': all(
                 ['IPTorrents' in y, 'type="password"' not in y[0:2048], self.has_all_cookies()] +
-                [(self.session.cookies.get(x) or 'sg!no!pw') in self.digest for x in 'uid', 'pass'])),
+                [(self.session.cookies.get(x, domain='') or 'sg!no!pw') in self.digest for x in ('uid', 'pass')])),
             failed_msg=(lambda y=None: u'Invalid cookie details for %s. Check settings'))
 
     @staticmethod
@@ -67,22 +67,43 @@ class IPTorrentsProvider(generic.TorrentProvider):
         if not self._authorised():
             return results
 
-        items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
+        last_recent_search = self.last_recent_search
+        last_recent_search = '' if not last_recent_search else last_recent_search.replace('id-', '')
 
-        rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': 'download'}.items())
         for mode in search_params.keys():
+            urls = []
             for search_string in search_params[mode]:
+                urls += [[]]
                 search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
-                # URL with 50 tv-show results, or max 150 if adjusted in IPTorrents profile
-                search_url = self.urls['search'] % (
-                    self._categories_string(mode, '%s', ';'), search_string,
-                    (';free', '')[not self.freeleech], (';o=seeders', '')['Cache' == mode])
+                for page in range((3, 5)['Cache' == mode])[1:]:
+                    # URL with 50 tv-show results, or max 150 if adjusted in IPTorrents profile
+                    urls[-1] += [self.urls['search'] % (
+                        self._categories_string(mode, '%s', ';'), search_string,
+                        (';free', '')[not self.freeleech], (';o=seeders', '')['Cache' == mode], page)]
+                results += self._search_urls(mode, last_recent_search, urls)
+                last_recent_search = ''
+
+            return results
+
+    def _search_urls(self, mode, last_recent_search, urls):
+
+        results = []
+        items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
+
+        rc = dict((k, re.compile('(?i)' + v)) for (k, v) in dict(
+            info='detail', get='download', id=r'download.*?/([\d]+)').items())
+        lrs_found = False
+        lrs_new = True
+        for search_urls in urls:  # this intentionally iterates once to preserve indentation
+            for search_url in search_urls:
 
                 html = self.get_url(search_url)
                 if self.should_skip():
                     return results
 
                 cnt = len(items[mode])
+                cnt_search = 0
+                log_settings_hint = False
                 try:
                     if not html or self._has_no_results(html):
                         raise generic.HaltParseException
@@ -94,23 +115,33 @@ class IPTorrentsProvider(generic.TorrentProvider):
                         if 2 > len(tbl_rows):
                             raise generic.HaltParseException
 
+                        if 'Cache' == mode and 100 > len(tbl_rows):
+                            log_settings_hint = True
+
                         head = None
                         for tr in tbl_rows[1:]:
                             cells = tr.find_all('td')
                             if 5 > len(cells):
                                 continue
+                            cnt_search += 1
                             try:
                                 head = head if None is not head else self._header_row(
                                     tr, header_strip='(?i)(?:leechers|seeders|size);')
                                 seeders, leechers = [tryInt(tr.find('td', class_='t_' + x).get_text().strip())
-                                                     for x in 'seeders', 'leechers']
+                                                     for x in ('seeders', 'leechers')]
                                 if self._reject_item(seeders, leechers):
                                     continue
 
+                                dl = tr.find('a', href=rc['get'])['href']
+                                dl_id = rc['id'].findall(dl)[0]
+                                lrs_found = dl_id == last_recent_search
+                                if lrs_found:
+                                    break
+
                                 info = tr.find('a', href=rc['info'])
                                 title = (info.attrs.get('title') or info.get_text()).strip()
                                 size = cells[head['size']].get_text().strip()
-                                download_url = self._link(tr.find('a', href=rc['get'])['href'])
+                                download_url = self._link(dl)
                             except (AttributeError, TypeError, ValueError):
                                 continue
 
@@ -121,7 +152,11 @@ class IPTorrentsProvider(generic.TorrentProvider):
                     pass
                 except (BaseException, Exception):
                     logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
-                self._log_search(mode, len(items[mode]) - cnt, search_url)
+                self._log_search(mode, len(items[mode]) - cnt, search_url, log_settings_hint)
+
+                if self.is_search_finished(mode, items, cnt_search, rc['id'], last_recent_search, lrs_new, lrs_found):
+                    break
+                lrs_new = False
 
             results = self._sort_seeding(mode, results + items[mode])
 

sickbeard/providers/torrentday.py

@@ -22,7 +22,6 @@ import time
 from . import generic
 from sickbeard.bs4_parser import BS4Parser
 from sickbeard.helpers import tryInt, anon_url
-from sickbeard import logger
 
 
 class TorrentDayProvider(generic.TorrentProvider):
@@ -54,7 +53,7 @@ class TorrentDayProvider(generic.TorrentProvider):
         return super(TorrentDayProvider, self)._authorised(
             logged_in=(lambda y='': all(
                 ['RSS URL' in y, self.has_all_cookies()] +
-                [(self.session.cookies.get(x) or 'sg!no!pw') in self.digest for x in 'uid', 'pass'])),
+                [(self.session.cookies.get(x, domain='') or 'sg!no!pw') in self.digest for x in ('uid', 'pass')])),
             failed_msg=(lambda y=None: u'Invalid cookie details for %s. Check settings'))
 
     @staticmethod
@@ -88,7 +87,7 @@ class TorrentDayProvider(generic.TorrentProvider):
 
         items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
 
-        rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'get': 'download', 'id': r'download.*?/([\d]+)'}.items())
+        rc = dict((k, re.compile('(?i)' + v)) for (k, v) in dict(get='download', id=r'download.*?/([\d]+)').items())
         lrs_found = False
         lrs_new = True
         for search_urls in urls:  # this intentionally iterates once to preserve indentation
@@ -124,7 +123,7 @@ class TorrentDayProvider(generic.TorrentProvider):
                                 head = head if None is not head else self._header_row(
                                     tr, header_strip='(?i)(?:leechers|seeders|size);')
                                 seeders, leechers, size = [tryInt(n, n) for n in [
-                                    cells[head[x]].get_text().strip() for x in 'seed', 'leech', 'size']]
+                                    cells[head[x]].get_text().strip() for x in ('seed', 'leech', 'size')]]
                                 if self._reject_item(seeders, leechers):
                                     continue
 
@@ -146,9 +145,7 @@ class TorrentDayProvider(generic.TorrentProvider):
                 except (BaseException, Exception):
                     time.sleep(1.1)
 
-                self._log_search(mode, len(items[mode]) - cnt, search_url)
-                if log_settings_hint:
-                    logger.log('Perfomance tip: change "Torrents per Page" to 100 at the TD site/Settings page')
+                self._log_search(mode, len(items[mode]) - cnt, search_url, log_settings_hint)
 
                 if self.is_search_finished(mode, items, cnt_search, rc['id'], last_recent_search, lrs_new, lrs_found):
                     break

sickbeard/providers/torrentleech.py

@@ -27,24 +27,29 @@ from lib.unidecode import unidecode
 
 class TorrentLeechProvider(generic.TorrentProvider):
     def __init__(self):
-        generic.TorrentProvider.__init__(self, 'TorrentLeech', cache_update_freq=15)
+        generic.TorrentProvider.__init__(self, 'TorrentLeech')
 
         self.url_base = 'https://v4.torrentleech.org/'
         self.urls = {'config_provider_home_uri': self.url_base,
-                     'login_action': self.url_base,
-                     'browse': self.url_base + 'torrents/browse/index/categories/%(cats)s',
-                     'search': self.url_base + 'torrents/browse/index/query/%(query)s/categories/%(cats)s'}
+                     'login': self.url_base,
+                     'browse': self.url_base + 'torrents/browse/index/categories/%(cats)s/%(x)s',
+                     'search': self.url_base + 'torrents/browse/index/query/%(query)s/categories/%(cats)s/%(x)s'}
 
         self.categories = {'shows': [2, 26, 27, 32], 'anime': [7, 34, 35]}
 
         self.url = self.urls['config_provider_home_uri']
-
-        self.username, self.password, self.minseed, self.minleech = 4 * [None]
+        self.digest, self.minseed, self.minleech, self.freeleech = 4 * [None]
 
     def _authorised(self, **kwargs):
 
-        return super(TorrentLeechProvider, self)._authorised(logged_in=(lambda y=None: self.has_all_cookies(pre='tl')),
-                                                             post_params={'remember_me': 'on', 'form_tmpl': True})
+        return super(TorrentLeechProvider, self)._authorised(
+            logged_in=(lambda y='': all(
+                ['TorrentLeech' in y, 'type="password"' not in y[0:4096], self.has_all_cookies(pre='tl')])),
+            failed_msg=(lambda y=None: u'Invalid cookie details for %s. Check settings'))
+
+    @staticmethod
+    def _has_signature(data=None):
+        return generic.TorrentProvider._has_signature(data) or (data and re.search(r'(?i)<title[^<]+?leech', data))
 
     def _search_provider(self, search_params, **kwargs):
 
@@ -52,20 +57,41 @@ class TorrentLeechProvider(generic.TorrentProvider):
         if not self._authorised():
             return results
 
-        items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
-
-        rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'get': 'download'}.items())
+        last_recent_search = self.last_recent_search
+        last_recent_search = '' if not last_recent_search else last_recent_search.replace('id-', '')
         for mode in search_params.keys():
+            urls = []
             for search_string in search_params[mode]:
-                search_url = self.urls[('search', 'browse')['Cache' == mode]] % {
-                    'cats': self._categories_string(mode, '', ','),
-                    'query': isinstance(search_string, unicode) and unidecode(search_string) or search_string}
+                urls += [[]]
+                for page in range((3, 5)['Cache' == mode])[1:]:
+                    urls[-1] += [self.urls[('search', 'browse')['Cache' == mode]] % {
+                        'cats': self._categories_string(mode, '', ','),
+                        'query': isinstance(search_string, unicode) and unidecode(search_string) or search_string,
+                        'x': '%spage/%s' % (('facets/tags:FREELEECH/', '')[not self.freeleech], page)
+                    }]
+            results += self._search_urls(mode, last_recent_search, urls)
+            last_recent_search = ''
+
+        return results
+
+    def _search_urls(self, mode, last_recent_search, urls):
+
+        results = []
+        items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
+
+        rc = dict((k, re.compile('(?i)' + v)) for (k, v) in dict(get='download', id=r'download.*?/([\d]+)').items())
+        lrs_found = False
+        lrs_new = True
+        for search_urls in urls:  # this intentionally iterates once to preserve indentation
+            for search_url in search_urls:
 
                 html = self.get_url(search_url)
                 if self.should_skip():
                     return results
 
                 cnt = len(items[mode])
+                cnt_search = 0
+                log_settings_hint = False
                 try:
                     if not html or self._has_no_results(html):
                         raise generic.HaltParseException
@@ -77,22 +103,32 @@ class TorrentLeechProvider(generic.TorrentProvider):
                         if 2 > len(tbl_rows):
                             raise generic.HaltParseException
 
+                        if 'Cache' == mode and 100 > len(tbl_rows):
+                            log_settings_hint = True
+
                         head = None
                         for tr in tbl_rows[1:]:
                             cells = tr.find_all('td')
                             if 6 > len(cells):
                                 continue
+                            cnt_search += 1
                             try:
                                 head = head if None is not head else self._header_row(tr)
                                 seeders, leechers = [tryInt(n) for n in [
-                                    tr.find('td', class_=x).get_text().strip() for x in 'seeders', 'leechers']]
+                                    tr.find('td', class_=x).get_text().strip() for x in ('seeders', 'leechers')]]
                                 if self._reject_item(seeders, leechers):
                                     continue
 
+                                dl = tr.find('a', href=rc['get'])['href']
+                                dl_id = rc['id'].findall(dl)[0]
+                                lrs_found = dl_id == last_recent_search
+                                if lrs_found:
+                                    break
+
                                 info = tr.find('td', class_='name').a
                                 title = (info.attrs.get('title') or info.get_text()).strip()
                                 size = cells[head['size']].get_text().strip()
-                                download_url = self._link(tr.find('a', href=rc['get'])['href'])
+                                download_url = self._link(dl)
                             except (AttributeError, TypeError, ValueError):
                                 continue
 
@@ -103,7 +139,11 @@ class TorrentLeechProvider(generic.TorrentProvider):
                     pass
                 except (BaseException, Exception):
                     logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
-                self._log_search(mode, len(items[mode]) - cnt, search_url)
+                self._log_search(mode, len(items[mode]) - cnt, search_url, log_settings_hint)
+
+                if self.is_search_finished(mode, items, cnt_search, rc['id'], last_recent_search, lrs_new, lrs_found):
+                    break
+                lrs_new = False
 
             results = self._sort_seeding(mode, results + items[mode])
 
@@ -113,5 +153,8 @@ class TorrentLeechProvider(generic.TorrentProvider):
 
         return super(TorrentLeechProvider, self)._episode_strings(ep_obj, sep_date='|', **kwargs)
 
+    def ui_string(self, key):
+        return 'torrentleech_digest' == key and self._valid_home() and 'use... \'tluid=xx; tlpass=yy\'' or ''
+
 
 provider = TorrentLeechProvider()