From 25337e96a2330e605c6b1b1230a8b51921c7a233 Mon Sep 17 00:00:00 2001 From: JackDandy Date: Thu, 14 Nov 2019 15:40:17 +0000 Subject: [PATCH] Change improve TD provider recent search performance to process new items since the previous cycle. Change log a tip for TD users who have not improved on the default site setting "Torrents per page". Change tweak hoverover highlight on menu item Shows/History for when History is the home page. Change update tvdb_api to 3.0.0 Change improve fetching TVDB thumbnails. Change add new 'banner_thumb' and 'poster_thumb' direct links. Change artwork domain to new artwork domain with fallback URLs. Change improve handling of Plex auth failure. --- CHANGES.md | 14 ++++++++- gui/slick/css/style.css | 2 +- lib/plex/plex.py | 2 ++ lib/tvdb_api/tvdb_api.py | 60 +++++++++++++++++++++++++++--------- sickbeard/logger.py | 2 +- sickbeard/metadata/generic.py | 65 +++++++++++++++++++++++++++++++++------ sickbeard/metadata/helpers.py | 4 ++- sickbeard/providers/generic.py | 45 ++++++++++++++++++++++++++- sickbeard/providers/torrentday.py | 47 +++++++++++++++++++++++----- 9 files changed, 203 insertions(+), 38 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 67ed0bd..db63745 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,16 @@ -### 0.20.7 (2019-11-10 14:40:00 UTC) +### 0.20.8 (2019-11-14 09:40:00 UTC) + +* Change improve TD provider recent search performance to process new items since the previous cycle +* Change log a tip for TD users who have not improved on the default site setting "Torrents per page" +* Change tweak hoverover highlight on menu item Shows/History for when History is the home page +* Change update tvdb_api to 3.0.0 +* Change improve fetching TVDB thumbnails +* Change add new 'banner_thumb' and 'poster_thumb' direct links +* Change artwork domain to new artwork domain with fallback URLs +* Change improve handling of Plex auth failure + + +### 0.20.7 (2019-11-10 14:40:00 UTC) * Fix configured Plex notification hosts that don't start with "http" * Add exclude "Specials" when pruning with option edit show/Other/"Keep up to" diff --git a/gui/slick/css/style.css b/gui/slick/css/style.css index b4744ea..a51952a 100644 --- a/gui/slick/css/style.css +++ b/gui/slick/css/style.css @@ -3929,7 +3929,7 @@ fieldset[disabled] .navbar-default .btn-link:focus{ } .dropdown-menu li > .history{ - width:70%; + width:72%; display:inline-block; padding-right:0 } diff --git a/lib/plex/plex.py b/lib/plex/plex.py index 51f7b15..667f0ea 100644 --- a/lib/plex/plex.py +++ b/lib/plex/plex.py @@ -135,6 +135,8 @@ class Plex: json=True, post_data=urlencode({b'user[login]': user, b'user[password]': passw}).encode('utf-8') )['user']['authentication_token'] + except TypeError: + self.log('Error in response from plex.tv auth server') except IndexError: self.log('Error getting Plex Token') diff --git a/lib/tvdb_api/tvdb_api.py b/lib/tvdb_api/tvdb_api.py index 526362e..1cd1a83 100644 --- a/lib/tvdb_api/tvdb_api.py +++ b/lib/tvdb_api/tvdb_api.py @@ -9,7 +9,7 @@ from functools import wraps __author__ = 'dbr/Ben' __version__ = '2.0' -__api_version__ = '2.2.0' +__api_version__ = '3.0.0' import os import time @@ -27,6 +27,7 @@ import sickbeard from lib.dateutil.parser import parse from lib.cachecontrol import CacheControl, caches +from six import integer_types from tvdb_ui import BaseUI, ConsoleUI from tvdb_exceptions import ( @@ -38,7 +39,7 @@ def log(): return logging.getLogger('tvdb_api') -def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None): +def retry(ExceptionToCheck, tries=4, delay=3, backoff=2): """Retry calling the decorated function using an exponential backoff. http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/ @@ -54,8 +55,6 @@ def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None): :param backoff: backoff multiplier e.g. value of 2 will double the delay each retry :type backoff: int - :param logger: logger to use. If None, print - :type logger: logging.Logger instance """ def deco_retry(f): @@ -69,10 +68,7 @@ def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None): return f(*args, **kwargs) except ExceptionToCheck, e: msg = '%s, Retrying in %d seconds...' % (str(e), mdelay) - if logger: - logger.warning(msg) - else: - print msg + log().warning(msg) time.sleep(mdelay) if isinstance(e, tvdb_tokenexpired) and not auth_error: auth_error += 1 @@ -525,7 +521,7 @@ class Tvdb: self.config['url_actorsInfo'] = '%(base_url)sseries/%%s/actors' % self.config self.config['url_seriesBanner'] = '%(base_url)sseries/%%s/images/query?keyType=%%s' % self.config - self.config['url_artworkPrefix'] = 'https://www.thetvdb.com/banners/%s' + self.config['url_artworkPrefix'] = 'https://artworks.thetvdb.com/banners/%s' def get_new_token(self): token = sickbeard.THETVDB_V2_API_TOKEN.get('token', None) @@ -820,14 +816,13 @@ class Tvdb: k, v = k.lower(), v.lower() if isinstance(v, (str, unicode)) else v if k == 'filename': - k = 'bannerpath' banners[btype][btype2][bid]['bannerpath'] = self.config['url_artworkPrefix'] % v elif k == 'thumbnail': - k = 'thumbnailpath' banners[btype][btype2][bid]['thumbnailpath'] = self.config['url_artworkPrefix'] % v elif k == 'keytype': - k = 'bannertype' - banners[btype][btype2][bid][k] = v + banners[btype][btype2][bid]['bannertype'] = v + else: + banners[btype][btype2][bid][k] = v except (StandardError, Exception): pass @@ -903,26 +898,40 @@ class Tvdb: self.shows[sid].ep_loaded = get_ep_info p = '' + pt = '' if self.config['posters_enabled']: poster_data = self._getetsrc(self.config['url_seriesBanner'] % (sid, 'poster'), language=language) if poster_data and len(poster_data.get('data', '') or '') > 0: poster_data['data'] = sorted(poster_data['data'], reverse=True, key=lambda x: (x['ratingsinfo']['average'], x['ratingsinfo']['count'])) p = self.config['url_artworkPrefix'] % poster_data['data'][0]['filename'] + pt = self.config['url_artworkPrefix'] % poster_data['data'][0]['thumbnail'] self._parse_banners(sid, poster_data['data']) if p: self._set_show_data(sid, u'poster', p) + if pt: + self._set_show_data(sid, u'poster_thumb', pt) + elif show_data['data']['poster']: + self._set_show_data(sid, u'poster_thumb', + re.sub(r'\.jpg$', '_t.jpg', show_data['data']['poster'], flags=re.I)) b = '' + bt = '' if self.config['banners_enabled']: poster_data = self._getetsrc(self.config['url_seriesBanner'] % (sid, 'series'), language=language) if poster_data and len(poster_data.get('data', '') or '') > 0: poster_data['data'] = sorted(poster_data['data'], reverse=True, key=lambda x: (x['ratingsinfo']['average'], x['ratingsinfo']['count'])) b = self.config['url_artworkPrefix'] % poster_data['data'][0]['filename'] + bt = self.config['url_artworkPrefix'] % poster_data['data'][0]['thumbnail'] self._parse_banners(sid, poster_data['data']) if b: self._set_show_data(sid, u'banner', b) + if bt: + self._set_show_data(sid, u'banner_thumb', bt) + elif show_data['data']['banner']: + self._set_show_data(sid, u'banner_thumb', + re.sub(r'\.jpg$', '_t.jpg', show_data['data']['banner'], flags=re.I)) if self.config['seasons_enabled']: poster_data = self._getetsrc(self.config['url_seriesBanner'] % (sid, 'season'), language=language) @@ -939,15 +948,22 @@ class Tvdb: self._parse_banners(sid, poster_data['data']) f = '' + ft = '' if self.config['fanart_enabled']: fanart_data = self._getetsrc(self.config['url_seriesBanner'] % (sid, 'fanart'), language=language) if fanart_data and len(fanart_data.get('data', '') or '') > 0: fanart_data['data'] = sorted(fanart_data['data'], reverse=True, key=lambda x: (x['ratingsinfo']['average'], x['ratingsinfo']['count'])) f = self.config['url_artworkPrefix'] % fanart_data['data'][0]['filename'] + ft = self.config['url_artworkPrefix'] % fanart_data['data'][0]['thumbnail'] self._parse_banners(sid, fanart_data['data']) if f: self._set_show_data(sid, u'fanart', f) + if ft: + self._set_show_data(sid, u'fanart_thumb', ft) + elif show_data[u'data'][u'fanart']: + self._set_show_data(sid, u'fanart_thumb', + re.sub(r'\.jpg$', '_t.jpg', show_data[u'data'][u'fanart'], flags=re.I)) if self.config['actors_enabled']: actor_data = self._getetsrc(self.config['url_actorsInfo'] % sid, language=language) @@ -960,13 +976,27 @@ class Tvdb: page = 1 episodes = [] - while page is not None: + while page <= 400: episode_data = self._getetsrc(self.config['url_epInfo'] % (sid, page), language=language) if None is episode_data: raise tvdb_error('Exception retrieving episodes for show') + if isinstance(episode_data, dict) and not episode_data.get('data', []): + if 1 != page: + self.not_found = False + break if not getattr(self, 'not_found', False) and None is not episode_data.get('data'): episodes.extend(episode_data['data']) - page = episode_data.get('links', {}).get('next', None) + next_link = episode_data.get('links', {}).get('next', None) + # check if page is a valid following page + if not isinstance(next_link, integer_types) or next_link <= page: + next_link = None + if not next_link and isinstance(episode_data, dict) \ + and isinstance(episode_data.get('data', []), list) and 100 > len(episode_data.get('data', [])): + break + if next_link: + page = next_link + else: + page += 1 ep_map_keys = {'absolutenumber': u'absolute_number', 'airedepisodenumber': u'episodenumber', 'airedseason': u'seasonnumber', 'airedseasonid': u'seasonid', diff --git a/sickbeard/logger.py b/sickbeard/logger.py index 555cf64..61a5efb 100644 --- a/sickbeard/logger.py +++ b/sickbeard/logger.py @@ -62,7 +62,7 @@ class SBRotatingLogHandler(object): self.console_logging = False self.log_lock = threading.Lock() - self.log_types = ['sickbeard', 'tornado.application', 'tornado.general', 'imdbpy', 'subliminal'] + self.log_types = ['sickbeard', 'tornado.application', 'tornado.general', 'imdbpy', 'subliminal', 'tvdb_api'] self.log_types_null = ['tornado.access'] def __del__(self): diff --git a/sickbeard/metadata/generic.py b/sickbeard/metadata/generic.py index 70bcbdf..b9abf9f 100644 --- a/sickbeard/metadata/generic.py +++ b/sickbeard/metadata/generic.py @@ -40,7 +40,8 @@ from sickbeard.exceptions import ex from sickbeard.indexers import indexer_config from sickbeard.indexers.indexer_config import INDEXER_TVDB, INDEXER_TVDB_V1 -from six import iteritems +from six import iteritems, itervalues +from collections import OrderedDict from lib.tmdb_api.tmdb_api import TMDB from lib.fanart.core import Request as fanartRequest @@ -795,11 +796,39 @@ class GenericMetadata(): image_urls = [] init_url = None + alt_tvdb_urls = [] + + def build_url(s_o, image_mode): + _urls = [] + _url = s_o[image_mode] + if _url and _url.startswith('http'): + if 'poster' == image_mode: + _url = re.sub('posters', '_cache/posters', _url) + elif 'banner' == image_mode: + _url = re.sub('graphical', '_cache/graphical', _url) + _urls.append([_url]) + + try: + alt_url = '%swww.%s%s' % re.findall( + r'(https?://)(?:artworks\.)?(thetvdb\.[^/]+/banners/[^\d]+[^.]+)(?:_t)(.*)', _url)[0][0:3] + if alt_url not in _urls[0]: + _urls.append([alt_url]) + except (IndexError, Exception): + try: + alt_url = '%sartworks.%s_t%s' % re.findall( + r'(https?://)(?:www\.)?(thetvdb\.[^/]+/banners/[^\d]+[^.]+)(.*)', _url)[0][0:3] + if alt_url not in _urls[0]: + _urls.append([alt_url]) + except (IndexError, Exception): + pass + return _urls + if 'poster_thumb' == image_type: - if getattr(indexer_show_obj, 'poster', None) is not None: - image_url = re.sub('posters', '_cache/posters', indexer_show_obj['poster']) - if image_url: - image_urls.append(image_url) + if None is not getattr(indexer_show_obj, image_type, None): + image_urls, alt_tvdb_urls = build_url(indexer_show_obj, image_type) + elif None is not getattr(indexer_show_obj, 'poster', None): + image_urls, alt_tvdb_urls = build_url(indexer_show_obj, 'poster') + for item in self._fanart_urls_from_show(show_obj, image_type, indexer_lang, True) or []: image_urls.append(item[2]) if 0 == len(image_urls): @@ -807,10 +836,10 @@ class GenericMetadata(): image_urls.append(item[2]) elif 'banner_thumb' == image_type: - if getattr(indexer_show_obj, 'banner', None) is not None: - image_url = re.sub('graphical', '_cache/graphical', indexer_show_obj['banner']) - if image_url: - image_urls.append(image_url) + if None is not getattr(indexer_show_obj, image_type, None): + image_urls, alt_tvdb_urls = build_url(indexer_show_obj, image_type) + elif None is not getattr(indexer_show_obj, 'banner', None): + image_urls, alt_tvdb_urls = build_url(indexer_show_obj, 'banner') for item in self._fanart_urls_from_show(show_obj, image_type, indexer_lang, True) or []: image_urls.append(item[2]) else: @@ -824,16 +853,32 @@ class GenericMetadata(): if 'poster' == image_type: init_url = image_url + # check extra provided images in '_banners' key + if None is not getattr(indexer_show_obj, '_banners', None) and \ + isinstance(indexer_show_obj['_banners'].get(image_type, None), (list, dict)): + for res, value in iteritems(indexer_show_obj['_banners'][image_type]): + for item in itervalues(value): + image_urls.append(item['bannerpath']) + if 0 == len(image_urls) or 'fanart' == image_type: for item in self._tmdb_image_url(show_obj, image_type) or []: image_urls.append('%s?%s' % (item[2], item[0])) image_data = len(image_urls) or None if image_data: + # remove duplicates, keeping order + image_urls = list(OrderedDict.fromkeys(image_urls)) if return_links: return image_urls else: - image_data = metadata_helpers.getShowImage((init_url, image_urls[0])[None is init_url], which, show_obj.name) + image_data = metadata_helpers.getShowImage( + (init_url, image_urls[0])[None is init_url], which, show_obj.name, bool(len(alt_tvdb_urls))) + if None is image_data and len(alt_tvdb_urls): + for url in alt_tvdb_urls: + image_data = metadata_helpers.getShowImage( + (init_url, url)[None is init_url], which, show_obj.name) + if None is not image_data: + break if None is not image_data: return image_data diff --git a/sickbeard/metadata/helpers.py b/sickbeard/metadata/helpers.py index be51dec..e433fd3 100644 --- a/sickbeard/metadata/helpers.py +++ b/sickbeard/metadata/helpers.py @@ -20,7 +20,7 @@ from sickbeard import helpers from sickbeard import logger -def getShowImage(url, imgNum=None, showName=None): +def getShowImage(url, imgNum=None, showName=None, supress_log=False): if None is url: return None @@ -32,6 +32,8 @@ def getShowImage(url, imgNum=None, showName=None): image_data = helpers.getURL(temp_url) if None is image_data: + if supress_log: + return logger.log('There was an error trying to retrieve the image%s, aborting' % ('', ' for show: %s' % showName)[None is not showName], logger.WARNING) return diff --git a/sickbeard/providers/generic.py b/sickbeard/providers/generic.py index e162334..d3125c2 100644 --- a/sickbeard/providers/generic.py +++ b/sickbeard/providers/generic.py @@ -43,7 +43,7 @@ from hachoir.stream import FileInputStream from sickbeard import helpers, classes, logger, db, tvcache, encodingKludge as ek from sickbeard.common import Quality, MULTI_EP_RESULT, SEASON_RESULT, USER_AGENT from sickbeard.exceptions import SickBeardException, AuthException, ex -from sickbeard.helpers import maybe_plural, remove_file_failed +from sickbeard.helpers import maybe_plural, remove_file_failed, tryInt from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException from sickbeard.show_name_helpers import get_show_names_all_possible from sickbeard.sbdatetime import sbdatetime @@ -1346,6 +1346,7 @@ class TorrentProvider(GenericProvider): self._reject_unverified = None self._reject_notfree = None self._reject_container = None + self._last_recent_search = None @property def url(self): @@ -1832,3 +1833,45 @@ class TorrentProvider(GenericProvider): result.after_get_data_func = self.after_get_data return result + + @property + def last_recent_search(self): + if not self._last_recent_search: + try: + my_db = db.DBConnection('cache.db') + res = my_db.select('SELECT' + ' "datetime" FROM "lastrecentsearch" WHERE "name"=?', [self.get_id()]) + if res: + self._last_recent_search = res[0]['datetime'] + except (BaseException, Exception): + pass + return self._last_recent_search + + @last_recent_search.setter + def last_recent_search(self, value): + value = 0 if not value else 'id-%s' % value + try: + my_db = db.DBConnection('cache.db') + my_db.action('INSERT OR REPLACE INTO "lastrecentsearch" (name, datetime) VALUES (?,?)', + [self.get_id(), value]) + except (BaseException, Exception): + pass + self._last_recent_search = value + + def is_search_finished(self, mode, items, cnt_search, rc_dlid, last_recent_search, lrs_rst, lrs_found): + result = True + if cnt_search: + if 'Cache' == mode: + if last_recent_search and lrs_rst: + self.last_recent_search = None + + if not self.last_recent_search: + try: + self.last_recent_search = tryInt(rc_dlid.findall(items[mode][0][1])[0]) + except IndexError: + self.last_recent_search = last_recent_search + + if last_recent_search and lrs_found: + return result + if 50 == cnt_search or 100 == cnt_search: + result = False + return result diff --git a/sickbeard/providers/torrentday.py b/sickbeard/providers/torrentday.py index 2c86d87..5b2de68 100644 --- a/sickbeard/providers/torrentday.py +++ b/sickbeard/providers/torrentday.py @@ -22,6 +22,7 @@ import time from . import generic from sickbeard.bs4_parser import BS4Parser from sickbeard.helpers import tryInt, anon_url +from sickbeard import logger class TorrentDayProvider(generic.TorrentProvider): @@ -37,7 +38,7 @@ class TorrentDayProvider(generic.TorrentProvider): '15TWd', 'hV 3c', 'lBHb', 'vNncq', 'j5ib', '=qQ02b']], ]]] - self.url_vars = {'login': 'rss.php', 'search': 't?%s%s&qf=&q=%s'} + self.url_vars = {'login': 'rss.php', 'search': 't?%s%s&qf=&p=%s&q=%s'} self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'login': '%(home)s%(vars)s', 'search': '%(home)s%(vars)s'} @@ -67,21 +68,38 @@ class TorrentDayProvider(generic.TorrentProvider): if not self._authorised(): return results - items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} - - rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'get': 'download'}.items()) + last_recent_search = self.last_recent_search + last_recent_search = '' if not last_recent_search else last_recent_search.replace('id-', '') for mode in search_params.keys(): + urls = [] for search_string in search_params[mode]: search_string = '+'.join(search_string.split()) + urls += [[]] + for page in range((3, 5)['Cache' == mode])[1:]: + urls[-1] += [self.urls['search'] % (self._categories_string(mode, '%s=on'), + ('&free=on', '')[not self.freeleech], page, search_string)] + results += self._search_urls(mode, last_recent_search, urls) + last_recent_search = '' - search_url = self.urls['search'] % ( - self._categories_string(mode, '%s=on'), ('&free=on', '')[not self.freeleech], search_string) + return results + + def _search_urls(self, mode, last_recent_search, urls): + results = [] + items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} + + rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'get': 'download', 'id': r'download.*?/([\d]+)'}.items()) + lrs_found = False + lrs_new = True + for search_urls in urls: # this intentionally iterates once to preserve indentation + for search_url in search_urls: html = self.get_url(search_url) if self.should_skip(): return results cnt = len(items[mode]) + cnt_search = 0 + log_settings_hint = False try: if not html or self._has_no_results(html): raise generic.HaltParseException @@ -93,11 +111,15 @@ class TorrentDayProvider(generic.TorrentProvider): if 2 > len(tbl_rows): raise generic.HaltParseException + if 'Cache' == mode and 100 > len(tbl_rows): + log_settings_hint = True + head = None for tr in tbl_rows[1:]: cells = tr.find_all('td') if 4 > len(cells): continue + cnt_search += 1 try: head = head if None is not head else self._header_row( tr, header_strip='(?i)(?:leechers|seeders|size);') @@ -107,8 +129,11 @@ class TorrentDayProvider(generic.TorrentProvider): continue dl = tr.find('a', href=rc['get'])['href'] - title = tr.find('a', href=re.compile( - '/t/%s' % re.findall('download.*?/([^/]+)', dl)[0])).get_text().strip() + dl_id = rc['id'].findall(dl)[0] + lrs_found = dl_id == last_recent_search + if lrs_found: + break + title = tr.find('a', href=re.compile('/t/%s' % dl_id)).get_text().strip() download_url = self._link(dl) except (AttributeError, TypeError, ValueError, IndexError): continue @@ -122,6 +147,12 @@ class TorrentDayProvider(generic.TorrentProvider): time.sleep(1.1) self._log_search(mode, len(items[mode]) - cnt, search_url) + if log_settings_hint: + logger.log('Perfomance tip: change "Torrents per Page" to 100 at the TD site/Settings page') + + if self.is_search_finished(mode, items, cnt_search, rc['id'], last_recent_search, lrs_new, lrs_found): + break + lrs_new = False results = self._sort_seeding(mode, results + items[mode])