Change improve TD provider recent search performance to process new items since the previous cycle.

Change log a tip for TD users who have not improved on the default site setting "Torrents per page".
Change tweak hoverover highlight on menu item Shows/History for when History is the home page.
Change update tvdb_api to 3.0.0
Change improve fetching TVDB thumbnails.
Change add new 'banner_thumb' and 'poster_thumb' direct links.
Change artwork domain to new artwork domain with fallback URLs.
Change improve handling of Plex auth failure.

CHANGES.md

@@ -1,4 +1,16 @@
-### 0.20.7 (2019-11-10 14:40:00 UTC)
+### 0.20.8 (2019-11-14 09:40:00 UTC)
+
+* Change improve TD provider recent search performance to process new items since the previous cycle
+* Change log a tip for TD users who have not improved on the default site setting "Torrents per page" 
+* Change tweak hoverover highlight on menu item Shows/History for when History is the home page
+* Change update tvdb_api to 3.0.0
+* Change improve fetching TVDB thumbnails
+* Change add new 'banner_thumb' and 'poster_thumb' direct links
+* Change artwork domain to new artwork domain with fallback URLs
+* Change improve handling of Plex auth failure
+
+                                                                                                                
+### 0.20.7 (2019-11-10 14:40:00 UTC)
 
 * Fix configured Plex notification hosts that don't start with "http"
 * Add exclude "Specials" when pruning with option edit show/Other/"Keep up to"

gui/slick/css/style.css

@@ -3929,7 +3929,7 @@ fieldset[disabled] .navbar-default .btn-link:focus{
 }
 
 .dropdown-menu li > .history{
-	width:70%;
+	width:72%;
 	display:inline-block;
 	padding-right:0
 }

lib/plex/plex.py

@@ -135,6 +135,8 @@ class Plex:
                           json=True,
                           post_data=urlencode({b'user[login]': user, b'user[password]': passw}).encode('utf-8')
                           )['user']['authentication_token']
+        except TypeError:
+            self.log('Error in response from plex.tv auth server')
         except IndexError:
             self.log('Error getting Plex Token')
 

lib/tvdb_api/tvdb_api.py

@@ -9,7 +9,7 @@ from functools import wraps
 
 __author__ = 'dbr/Ben'
 __version__ = '2.0'
-__api_version__ = '2.2.0'
+__api_version__ = '3.0.0'
 
 import os
 import time
@@ -27,6 +27,7 @@ import sickbeard
 
 from lib.dateutil.parser import parse
 from lib.cachecontrol import CacheControl, caches
+from six import integer_types
 
 from tvdb_ui import BaseUI, ConsoleUI
 from tvdb_exceptions import (
@@ -38,7 +39,7 @@ def log():
     return logging.getLogger('tvdb_api')
 
 
-def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None):
+def retry(ExceptionToCheck, tries=4, delay=3, backoff=2):
     """Retry calling the decorated function using an exponential backoff.
 
     http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
@@ -54,8 +55,6 @@ def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None):
     :param backoff: backoff multiplier e.g. value of 2 will double the delay
         each retry
     :type backoff: int
-    :param logger: logger to use. If None, print
-    :type logger: logging.Logger instance
     """
 
     def deco_retry(f):
@@ -69,10 +68,7 @@ def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None):
                     return f(*args, **kwargs)
                 except ExceptionToCheck, e:
                     msg = '%s, Retrying in %d seconds...' % (str(e), mdelay)
-                    if logger:
-                        logger.warning(msg)
-                    else:
-                        print msg
+                    log().warning(msg)
                     time.sleep(mdelay)
                     if isinstance(e, tvdb_tokenexpired) and not auth_error:
                         auth_error += 1
@@ -525,7 +521,7 @@ class Tvdb:
         self.config['url_actorsInfo'] = '%(base_url)sseries/%%s/actors' % self.config
 
         self.config['url_seriesBanner'] = '%(base_url)sseries/%%s/images/query?keyType=%%s' % self.config
-        self.config['url_artworkPrefix'] = 'https://www.thetvdb.com/banners/%s'
+        self.config['url_artworkPrefix'] = 'https://artworks.thetvdb.com/banners/%s'
 
     def get_new_token(self):
         token = sickbeard.THETVDB_V2_API_TOKEN.get('token', None)
@@ -820,14 +816,13 @@ class Tvdb:
 
                     k, v = k.lower(), v.lower() if isinstance(v, (str, unicode)) else v
                     if k == 'filename':
-                        k = 'bannerpath'
                         banners[btype][btype2][bid]['bannerpath'] = self.config['url_artworkPrefix'] % v
                     elif k == 'thumbnail':
-                        k = 'thumbnailpath'
                         banners[btype][btype2][bid]['thumbnailpath'] = self.config['url_artworkPrefix'] % v
                     elif k == 'keytype':
-                        k = 'bannertype'
-                    banners[btype][btype2][bid][k] = v
+                        banners[btype][btype2][bid]['bannertype'] = v
+                    else:
+                        banners[btype][btype2][bid][k] = v
 
         except (StandardError, Exception):
             pass
@@ -903,26 +898,40 @@ class Tvdb:
             self.shows[sid].ep_loaded = get_ep_info
 
         p = ''
+        pt = ''
         if self.config['posters_enabled']:
             poster_data = self._getetsrc(self.config['url_seriesBanner'] % (sid, 'poster'), language=language)
             if poster_data and len(poster_data.get('data', '') or '') > 0:
                 poster_data['data'] = sorted(poster_data['data'], reverse=True,
                                              key=lambda x: (x['ratingsinfo']['average'], x['ratingsinfo']['count']))
                 p = self.config['url_artworkPrefix'] % poster_data['data'][0]['filename']
+                pt = self.config['url_artworkPrefix'] % poster_data['data'][0]['thumbnail']
                 self._parse_banners(sid, poster_data['data'])
         if p:
             self._set_show_data(sid, u'poster', p)
+        if pt:
+            self._set_show_data(sid, u'poster_thumb', pt)
+        elif show_data['data']['poster']:
+            self._set_show_data(sid, u'poster_thumb',
+                                re.sub(r'\.jpg$', '_t.jpg', show_data['data']['poster'], flags=re.I))
 
         b = ''
+        bt = ''
         if self.config['banners_enabled']:
             poster_data = self._getetsrc(self.config['url_seriesBanner'] % (sid, 'series'), language=language)
             if poster_data and len(poster_data.get('data', '') or '') > 0:
                 poster_data['data'] = sorted(poster_data['data'], reverse=True,
                                              key=lambda x: (x['ratingsinfo']['average'], x['ratingsinfo']['count']))
                 b = self.config['url_artworkPrefix'] % poster_data['data'][0]['filename']
+                bt = self.config['url_artworkPrefix'] % poster_data['data'][0]['thumbnail']
                 self._parse_banners(sid, poster_data['data'])
         if b:
             self._set_show_data(sid, u'banner', b)
+        if bt:
+            self._set_show_data(sid, u'banner_thumb', bt)
+        elif show_data['data']['banner']:
+            self._set_show_data(sid, u'banner_thumb',
+                                re.sub(r'\.jpg$', '_t.jpg', show_data['data']['banner'], flags=re.I))
 
         if self.config['seasons_enabled']:
             poster_data = self._getetsrc(self.config['url_seriesBanner'] % (sid, 'season'), language=language)
@@ -939,15 +948,22 @@ class Tvdb:
                 self._parse_banners(sid, poster_data['data'])
 
         f = ''
+        ft = ''
         if self.config['fanart_enabled']:
             fanart_data = self._getetsrc(self.config['url_seriesBanner'] % (sid, 'fanart'), language=language)
             if fanart_data and len(fanart_data.get('data', '') or '') > 0:
                 fanart_data['data'] = sorted(fanart_data['data'], reverse=True,
                                              key=lambda x: (x['ratingsinfo']['average'], x['ratingsinfo']['count']))
                 f = self.config['url_artworkPrefix'] % fanart_data['data'][0]['filename']
+                ft = self.config['url_artworkPrefix'] % fanart_data['data'][0]['thumbnail']
                 self._parse_banners(sid, fanart_data['data'])
         if f:
             self._set_show_data(sid, u'fanart', f)
+        if ft:
+            self._set_show_data(sid, u'fanart_thumb', ft)
+        elif show_data[u'data'][u'fanart']:
+            self._set_show_data(sid, u'fanart_thumb',
+                                re.sub(r'\.jpg$', '_t.jpg', show_data[u'data'][u'fanart'], flags=re.I))
 
         if self.config['actors_enabled']:
             actor_data = self._getetsrc(self.config['url_actorsInfo'] % sid, language=language)
@@ -960,13 +976,27 @@ class Tvdb:
 
             page = 1
             episodes = []
-            while page is not None:
+            while page <= 400:
                 episode_data = self._getetsrc(self.config['url_epInfo'] % (sid, page), language=language)
                 if None is episode_data:
                     raise tvdb_error('Exception retrieving episodes for show')
+                if isinstance(episode_data, dict) and not episode_data.get('data', []):
+                    if 1 != page:
+                        self.not_found = False
+                    break
                 if not getattr(self, 'not_found', False) and None is not episode_data.get('data'):
                     episodes.extend(episode_data['data'])
-                page = episode_data.get('links', {}).get('next', None)
+                next_link = episode_data.get('links', {}).get('next', None)
+                # check if page is a valid following page
+                if not isinstance(next_link, integer_types) or next_link <= page:
+                    next_link = None
+                if not next_link and isinstance(episode_data, dict) \
+                        and isinstance(episode_data.get('data', []), list) and 100 > len(episode_data.get('data', [])):
+                    break
+                if next_link:
+                    page = next_link
+                else:
+                    page += 1
 
             ep_map_keys = {'absolutenumber': u'absolute_number', 'airedepisodenumber': u'episodenumber',
                            'airedseason': u'seasonnumber', 'airedseasonid': u'seasonid',

sickbeard/logger.py

@@ -62,7 +62,7 @@ class SBRotatingLogHandler(object):
 
         self.console_logging = False
         self.log_lock = threading.Lock()
-        self.log_types = ['sickbeard', 'tornado.application', 'tornado.general', 'imdbpy', 'subliminal']
+        self.log_types = ['sickbeard', 'tornado.application', 'tornado.general', 'imdbpy', 'subliminal', 'tvdb_api']
         self.log_types_null = ['tornado.access']
 
     def __del__(self):

sickbeard/metadata/generic.py

@@ -40,7 +40,8 @@ from sickbeard.exceptions import ex
 from sickbeard.indexers import indexer_config
 from sickbeard.indexers.indexer_config import INDEXER_TVDB, INDEXER_TVDB_V1
 
-from six import iteritems
+from six import iteritems, itervalues
+from collections import OrderedDict
 
 from lib.tmdb_api.tmdb_api import TMDB
 from lib.fanart.core import Request as fanartRequest
@@ -795,11 +796,39 @@ class GenericMetadata():
 
         image_urls = []
         init_url = None
+        alt_tvdb_urls = []
+
+        def build_url(s_o, image_mode):
+            _urls = []
+            _url = s_o[image_mode]
+            if _url and _url.startswith('http'):
+                if 'poster' == image_mode:
+                    _url = re.sub('posters', '_cache/posters', _url)
+                elif 'banner' == image_mode:
+                    _url = re.sub('graphical', '_cache/graphical', _url)
+                _urls.append([_url])
+
+                try:
+                    alt_url = '%swww.%s%s' % re.findall(
+                        r'(https?://)(?:artworks\.)?(thetvdb\.[^/]+/banners/[^\d]+[^.]+)(?:_t)(.*)', _url)[0][0:3]
+                    if alt_url not in _urls[0]:
+                        _urls.append([alt_url])
+                except (IndexError, Exception):
+                    try:
+                        alt_url = '%sartworks.%s_t%s' % re.findall(
+                            r'(https?://)(?:www\.)?(thetvdb\.[^/]+/banners/[^\d]+[^.]+)(.*)', _url)[0][0:3]
+                        if alt_url not in _urls[0]:
+                            _urls.append([alt_url])
+                    except (IndexError, Exception):
+                        pass
+            return _urls
+
         if 'poster_thumb' == image_type:
-            if getattr(indexer_show_obj, 'poster', None) is not None:
-                image_url = re.sub('posters', '_cache/posters', indexer_show_obj['poster'])
-                if image_url:
-                    image_urls.append(image_url)
+            if None is not getattr(indexer_show_obj, image_type, None):
+                image_urls, alt_tvdb_urls = build_url(indexer_show_obj, image_type)
+            elif None is not getattr(indexer_show_obj, 'poster', None):
+                image_urls, alt_tvdb_urls = build_url(indexer_show_obj, 'poster')
+
             for item in self._fanart_urls_from_show(show_obj, image_type, indexer_lang, True) or []:
                 image_urls.append(item[2])
             if 0 == len(image_urls):
@@ -807,10 +836,10 @@ class GenericMetadata():
                     image_urls.append(item[2])
 
         elif 'banner_thumb' == image_type:
-            if getattr(indexer_show_obj, 'banner', None) is not None:
-                image_url = re.sub('graphical', '_cache/graphical', indexer_show_obj['banner'])
-                if image_url:
-                    image_urls.append(image_url)
+            if None is not getattr(indexer_show_obj, image_type, None):
+                image_urls, alt_tvdb_urls = build_url(indexer_show_obj, image_type)
+            elif None is not getattr(indexer_show_obj, 'banner', None):
+                image_urls, alt_tvdb_urls = build_url(indexer_show_obj, 'banner')
             for item in self._fanart_urls_from_show(show_obj, image_type, indexer_lang, True) or []:
                 image_urls.append(item[2])
         else:
@@ -824,16 +853,32 @@ class GenericMetadata():
                     if 'poster' == image_type:
                         init_url = image_url
 
+            # check extra provided images in '_banners' key
+            if None is not getattr(indexer_show_obj, '_banners', None) and \
+                    isinstance(indexer_show_obj['_banners'].get(image_type, None), (list, dict)):
+                for res, value in iteritems(indexer_show_obj['_banners'][image_type]):
+                    for item in itervalues(value):
+                        image_urls.append(item['bannerpath'])
+
             if 0 == len(image_urls) or 'fanart' == image_type:
                 for item in self._tmdb_image_url(show_obj, image_type) or []:
                     image_urls.append('%s?%s' % (item[2], item[0]))
 
         image_data = len(image_urls) or None
         if image_data:
+            # remove duplicates, keeping order
+            image_urls = list(OrderedDict.fromkeys(image_urls))
             if return_links:
                 return image_urls
             else:
-                image_data = metadata_helpers.getShowImage((init_url, image_urls[0])[None is init_url], which, show_obj.name)
+                image_data = metadata_helpers.getShowImage(
+                    (init_url, image_urls[0])[None is init_url], which, show_obj.name, bool(len(alt_tvdb_urls)))
+                if None is image_data and len(alt_tvdb_urls):
+                    for url in alt_tvdb_urls:
+                        image_data = metadata_helpers.getShowImage(
+                            (init_url, url)[None is init_url], which, show_obj.name)
+                        if None is not image_data:
+                            break
 
         if None is not image_data:
             return image_data

sickbeard/metadata/helpers.py

@@ -20,7 +20,7 @@ from sickbeard import helpers
 from sickbeard import logger
 
 
-def getShowImage(url, imgNum=None, showName=None):
+def getShowImage(url, imgNum=None, showName=None, supress_log=False):
 
     if None is url:
         return None
@@ -32,6 +32,8 @@ def getShowImage(url, imgNum=None, showName=None):
 
     image_data = helpers.getURL(temp_url)
     if None is image_data:
+        if supress_log:
+            return
         logger.log('There was an error trying to retrieve the image%s, aborting' %
                    ('', ' for show: %s' % showName)[None is not showName], logger.WARNING)
         return

sickbeard/providers/generic.py

@@ -43,7 +43,7 @@ from hachoir.stream import FileInputStream
 from sickbeard import helpers, classes, logger, db, tvcache, encodingKludge as ek
 from sickbeard.common import Quality, MULTI_EP_RESULT, SEASON_RESULT, USER_AGENT
 from sickbeard.exceptions import SickBeardException, AuthException, ex
-from sickbeard.helpers import maybe_plural, remove_file_failed
+from sickbeard.helpers import maybe_plural, remove_file_failed, tryInt
 from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException
 from sickbeard.show_name_helpers import get_show_names_all_possible
 from sickbeard.sbdatetime import sbdatetime
@@ -1346,6 +1346,7 @@ class TorrentProvider(GenericProvider):
         self._reject_unverified = None
         self._reject_notfree = None
         self._reject_container = None
+        self._last_recent_search = None
 
     @property
     def url(self):
@@ -1832,3 +1833,45 @@ class TorrentProvider(GenericProvider):
                 result.after_get_data_func = self.after_get_data
 
         return result
+
+    @property
+    def last_recent_search(self):
+        if not self._last_recent_search:
+            try:
+                my_db = db.DBConnection('cache.db')
+                res = my_db.select('SELECT' + ' "datetime" FROM "lastrecentsearch" WHERE "name"=?', [self.get_id()])
+                if res:
+                    self._last_recent_search = res[0]['datetime']
+            except (BaseException, Exception):
+                pass
+        return self._last_recent_search
+
+    @last_recent_search.setter
+    def last_recent_search(self, value):
+        value = 0 if not value else 'id-%s' % value
+        try:
+            my_db = db.DBConnection('cache.db')
+            my_db.action('INSERT OR REPLACE INTO "lastrecentsearch" (name, datetime) VALUES (?,?)',
+                         [self.get_id(), value])
+        except (BaseException, Exception):
+            pass
+        self._last_recent_search = value
+
+    def is_search_finished(self, mode, items, cnt_search, rc_dlid, last_recent_search, lrs_rst, lrs_found):
+        result = True
+        if cnt_search:
+            if 'Cache' == mode:
+                if last_recent_search and lrs_rst:
+                    self.last_recent_search = None
+
+                if not self.last_recent_search:
+                    try:
+                        self.last_recent_search = tryInt(rc_dlid.findall(items[mode][0][1])[0])
+                    except IndexError:
+                        self.last_recent_search = last_recent_search
+
+                if last_recent_search and lrs_found:
+                    return result
+            if 50 == cnt_search or 100 == cnt_search:
+                result = False
+        return result

sickbeard/providers/torrentday.py

@@ -22,6 +22,7 @@ import time
 from . import generic
 from sickbeard.bs4_parser import BS4Parser
 from sickbeard.helpers import tryInt, anon_url
+from sickbeard import logger
 
 
 class TorrentDayProvider(generic.TorrentProvider):
@@ -37,7 +38,7 @@ class TorrentDayProvider(generic.TorrentProvider):
                                 '15TWd', 'hV 3c', 'lBHb', 'vNncq', 'j5ib', '=qQ02b']],
                         ]]]
 
-        self.url_vars = {'login': 'rss.php', 'search': 't?%s%s&qf=&q=%s'}
+        self.url_vars = {'login': 'rss.php', 'search': 't?%s%s&qf=&p=%s&q=%s'}
         self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'login': '%(home)s%(vars)s',
                          'search': '%(home)s%(vars)s'}
 
@@ -67,21 +68,38 @@ class TorrentDayProvider(generic.TorrentProvider):
         if not self._authorised():
             return results
 
-        items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
-
-        rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'get': 'download'}.items())
+        last_recent_search = self.last_recent_search
+        last_recent_search = '' if not last_recent_search else last_recent_search.replace('id-', '')
         for mode in search_params.keys():
+            urls = []
             for search_string in search_params[mode]:
                 search_string = '+'.join(search_string.split())
+                urls += [[]]
+                for page in range((3, 5)['Cache' == mode])[1:]:
+                    urls[-1] += [self.urls['search'] % (self._categories_string(mode, '%s=on'),
+                                                        ('&free=on', '')[not self.freeleech], page, search_string)]
+            results += self._search_urls(mode, last_recent_search, urls)
+            last_recent_search = ''
 
-                search_url = self.urls['search'] % (
-                    self._categories_string(mode, '%s=on'), ('&free=on', '')[not self.freeleech], search_string)
+        return results
+
+    def _search_urls(self, mode, last_recent_search, urls):
+        results = []
 
+        items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
+
+        rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'get': 'download', 'id': r'download.*?/([\d]+)'}.items())
+        lrs_found = False
+        lrs_new = True
+        for search_urls in urls:  # this intentionally iterates once to preserve indentation
+            for search_url in search_urls:
                 html = self.get_url(search_url)
                 if self.should_skip():
                     return results
 
                 cnt = len(items[mode])
+                cnt_search = 0
+                log_settings_hint = False
                 try:
                     if not html or self._has_no_results(html):
                         raise generic.HaltParseException
@@ -93,11 +111,15 @@ class TorrentDayProvider(generic.TorrentProvider):
                         if 2 > len(tbl_rows):
                             raise generic.HaltParseException
 
+                        if 'Cache' == mode and 100 > len(tbl_rows):
+                            log_settings_hint = True
+
                         head = None
                         for tr in tbl_rows[1:]:
                             cells = tr.find_all('td')
                             if 4 > len(cells):
                                 continue
+                            cnt_search += 1
                             try:
                                 head = head if None is not head else self._header_row(
                                     tr, header_strip='(?i)(?:leechers|seeders|size);')
@@ -107,8 +129,11 @@ class TorrentDayProvider(generic.TorrentProvider):
                                     continue
 
                                 dl = tr.find('a', href=rc['get'])['href']
-                                title = tr.find('a', href=re.compile(
-                                    '/t/%s' % re.findall('download.*?/([^/]+)', dl)[0])).get_text().strip()
+                                dl_id = rc['id'].findall(dl)[0]
+                                lrs_found = dl_id == last_recent_search
+                                if lrs_found:
+                                    break
+                                title = tr.find('a', href=re.compile('/t/%s' % dl_id)).get_text().strip()
                                 download_url = self._link(dl)
                             except (AttributeError, TypeError, ValueError, IndexError):
                                 continue
@@ -122,6 +147,12 @@ class TorrentDayProvider(generic.TorrentProvider):
                     time.sleep(1.1)
 
                 self._log_search(mode, len(items[mode]) - cnt, search_url)
+                if log_settings_hint:
+                    logger.log('Perfomance tip: change "Torrents per Page" to 100 at the TD site/Settings page')
+
+                if self.is_search_finished(mode, items, cnt_search, rc['id'], last_recent_search, lrs_new, lrs_found):
+                    break
+                lrs_new = False
 
             results = self._sort_seeding(mode, results + items[mode])