diff --git a/CHANGES.md b/CHANGES.md index 2d1e644..3c4b401 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ ### 0.23.0 (2019-xx-xx xx:xx:xx UTC) +* Change improve search performance for backlog, manual, failed, and proper * Add overview of the last release age/date at each newznab provider to History/Layout "Connect fails" * Add "History new..." to Shows menu by clicking the number * Add db backup to the scheduled daily update diff --git a/sickbeard/properFinder.py b/sickbeard/properFinder.py index ab1c529..1ae3c30 100644 --- a/sickbeard/properFinder.py +++ b/sickbeard/properFinder.py @@ -36,20 +36,21 @@ from .history import dateFormat from .name_parser.parser import InvalidNameException, InvalidShowException, NameParser from .sgdatetime import timestamp_near -from _23 import filter_iter, list_values, map_consume, map_list +from _23 import filter_iter, filter_list, list_values, map_consume, map_list from six import string_types # noinspection PyUnreachableCode if False: # noinspection PyUnresolvedReferences - from typing import AnyStr, List, Tuple + from typing import AnyStr, Dict, List, Tuple + from .providers.generic import GenericProvider def search_propers(provider_proper_obj=None): + # type: (Dict[AnyStr, List[Proper]]) -> None """ :param provider_proper_obj: Optional dict with provider keys containing Proper objects - :type provider_proper_obj: dict :return: """ if not sickbeard.DOWNLOAD_PROPERS: @@ -214,19 +215,35 @@ def load_webdl_types(): sickbeard.WEBDL_TYPES = new_types + default_types -def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_dict=None): +def _search_provider(cur_provider, provider_propers, aired_since_shows, recent_shows, recent_anime): + # type: (GenericProvider, List, datetime.datetime, List[Tuple[int, int]], List[Tuple[int, int]]) -> None + try: + # we need to extent the referenced list from parameter to update the original var + provider_propers.extend(cur_provider.find_propers(search_date=aired_since_shows, shows=recent_shows, + anime=recent_anime)) + except AuthException as e: + logger.log('Authentication error: %s' % ex(e), logger.ERROR) + except (BaseException, Exception) as e: + logger.log('Error while searching %s, skipping: %s' % (cur_provider.name, ex(e)), logger.ERROR) + logger.log(traceback.format_exc(), logger.ERROR) + + if not provider_propers: + logger.log('No Proper releases found at [%s]' % cur_provider.name) + + +def _get_proper_list(aired_since_shows, # type: datetime.datetime + recent_shows, # type: List[Tuple[int, int]] + recent_anime, # type: List[Tuple[int, int]] + proper_dict=None # type: Dict[AnyStr, List[Proper]] + ): + # type: (...) -> List[Proper] """ :param aired_since_shows: date since aired - :type aired_since_shows: datetime.datetime :param recent_shows: list of recent shows - :type recent_shows: List[Tuple[int, int]] :param recent_anime: list of recent anime shows - :type recent_anime: List[Tuple[int, int]] :param proper_dict: dict with provider keys containing Proper objects - :type proper_dict: dict :return: list of propers - :rtype: List[sickbeard.classes.Proper] """ propers = {} # make sure the episode has been downloaded before @@ -235,31 +252,47 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_dict= my_db = db.DBConnection() # for each provider get a list of arbitrary Propers orig_thread_name = threading.currentThread().name - for cur_provider in filter_iter(lambda p: p.is_active(), sickbeard.providers.sortedProviderList()): - if not recent_anime and cur_provider.anime_only: - continue - - if None is not proper_dict: - found_propers = proper_dict.get(cur_provider.get_id(), []) - if not found_propers: + # filter provider list for: + # 1. from recent search: recent search enabled providers + # 2. native proper search: active search enabled providers + provider_list = filter_list( + lambda p: p.is_active() and (p.enable_recentsearch, p.enable_backlog)[None is proper_dict], + sickbeard.providers.sortedProviderList()) + search_threads = [] + + if None is proper_dict: + # if not a recent proper search create a thread per provider to search for Propers + proper_dict = {} + for cur_provider in provider_list: + if not recent_anime and cur_provider.anime_only: continue - else: - threading.currentThread().name = '%s :: [%s]' % (orig_thread_name, cur_provider.name) - logger.log('Searching for new PROPER releases') + provider_id = cur_provider.get_id() - try: - found_propers = cur_provider.find_propers(search_date=aired_since_shows, shows=recent_shows, - anime=recent_anime) - except AuthException as e: - logger.log('Authentication error: %s' % ex(e), logger.ERROR) - continue - except (BaseException, Exception) as e: - logger.log('Error while searching %s, skipping: %s' % (cur_provider.name, ex(e)), logger.ERROR) - logger.log(traceback.format_exc(), logger.ERROR) - continue - finally: - threading.currentThread().name = orig_thread_name + logger.log('Searching for new Proper releases at [%s]' % cur_provider.name) + proper_dict[provider_id] = [] + + search_threads.append(threading.Thread(target=_search_provider, + kwargs={'cur_provider': cur_provider, + 'provider_propers': proper_dict[provider_id], + 'aired_since_shows': aired_since_shows, + 'recent_shows': recent_shows, + 'recent_anime': recent_anime}, + name='%s :: [%s]' % (orig_thread_name, cur_provider.name))) + + search_threads[-1].start() + + # wait for all searches to finish + for cur_thread in search_threads: + cur_thread.join() + + for cur_provider in provider_list: + if not recent_anime and cur_provider.anime_only: + continue + + found_propers = proper_dict.get(cur_provider.get_id(), []) + if not found_propers: + continue # if they haven't been added by a different provider than add the Proper to the list for cur_proper in found_propers: @@ -277,7 +310,7 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_dict= cur_proper.parsed_show_obj = (cur_proper.parsed_show_obj or helpers.find_show_by_id(parse_result.show_obj.tvid_prodid)) if None is cur_proper.parsed_show_obj: - logger.log('Skip download; cannot find show with ID [%s] from %s' % + logger.log('Skip download; cannot find show with ID [%s] at %s' % (cur_proper.prodid, sickbeard.TVInfoAPI(cur_proper.tvid).name), logger.ERROR) continue @@ -462,11 +495,11 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_dict= def _download_propers(proper_list): + # type: (List[Proper]) -> None """ download propers from given list :param proper_list: proper list - :type proper_list: List[sickbeard.classes.Proper] """ verified_propers = True consumed_proper = [] @@ -560,12 +593,11 @@ def _download_propers(proper_list): def get_needed_qualites(needed=None): + # type: (sickbeard.common.NeededQualities) -> sickbeard.common.NeededQualities """ :param needed: optional needed object - :type needed: sickbeard.common.NeededQualities :return: needed object - :rtype: sickbeard.common.NeededQualities """ if not isinstance(needed, NeededQualities): needed = NeededQualities() diff --git a/sickbeard/providers/__init__.py b/sickbeard/providers/__init__.py index 3bf3e10..0956811 100755 --- a/sickbeard/providers/__init__.py +++ b/sickbeard/providers/__init__.py @@ -29,7 +29,8 @@ from six import iteritems, itervalues # noinspection PyUnreachableCode if False: - from typing import AnyStr, List + from typing import AnyStr, List, Union + from .generic import GenericProvider, NZBProvider, TorrentProvider __all__ = [ # usenet @@ -57,6 +58,12 @@ for module in __all__: def sortedProviderList(): + # type: (...) -> List[Union[GenericProvider, NZBProvider, TorrentProvider]] + """ + return sorted provider list + + :return: sorted list of providers + """ initialList = sickbeard.providerList + sickbeard.newznabProviderList + sickbeard.torrentRssProviderList providerDict = dict(zip([x.get_id() for x in initialList], initialList)) diff --git a/sickbeard/search.py b/sickbeard/search.py index ffad4d1..493e76c 100644 --- a/sickbeard/search.py +++ b/sickbeard/search.py @@ -710,6 +710,70 @@ def can_reject(release_name): return pred and (None, None) or (predb_rej or True, ', '.join(rej_urls)) +def _search_provider_thread(provider, provider_results, show_obj, ep_obj_list, manual_search, try_other_searches): + # type: (GenericProvider, Dict, TVShow, List[TVEpisode], bool, bool) -> None + """ + perform a search on a provider for specified show, episodes + + :param provider: Provider to search + :param provider_results: reference to dict to return results + :param show_obj: show to search for + :param ep_obj_list: list of episodes to search for + :param manual_search: is manual search + :param try_other_searches: try other search methods + """ + search_count = 0 + search_mode = getattr(provider, 'search_mode', 'eponly') + + while True: + search_count += 1 + + if 'eponly' == search_mode: + logger.log(u'Performing episode search for %s' % show_obj.name) + else: + logger.log(u'Performing season pack search for %s' % show_obj.name) + + try: + provider.cache._clearCache() + search_result_list = provider.find_search_results(show_obj, ep_obj_list, search_mode, manual_search, + try_other_searches=try_other_searches) + if any(search_result_list): + logger.log(', '.join(['%s %s candidate%s' % ( + len(v), (('multiep', 'season')[SEASON_RESULT == k], 'episode')['ep' in search_mode], + helpers.maybe_plural(v)) for (k, v) in iteritems(search_result_list)])) + except exceptions_helper.AuthException as e: + logger.error(u'Authentication error: %s' % ex(e)) + break + except (BaseException, Exception) as e: + logger.error(u'Error while searching %s, skipping: %s' % (provider.name, ex(e))) + logger.error(traceback.format_exc()) + break + + if len(search_result_list): + # make a list of all the results for this provider + for cur_search_result in search_result_list: + # skip non-tv crap + search_result_list[cur_search_result] = filter_list( + lambda ep_item: ep_item.show_obj == show_obj and show_name_helpers.pass_wordlist_checks( + ep_item.name, parse=False, indexer_lookup=False, show_obj=ep_item.show_obj), + search_result_list[cur_search_result]) + + if cur_search_result in provider_results: + provider_results[cur_search_result] += search_result_list[cur_search_result] + else: + provider_results[cur_search_result] = search_result_list[cur_search_result] + + break + elif not getattr(provider, 'search_fallback', False) or 2 == search_count: + break + + search_mode = '%sonly' % ('ep', 'sp')['ep' in search_mode] + logger.log(u'Falling back to %s search ...' % ('season pack', 'episode')['ep' in search_mode]) + + if not provider_results: + logger.log('No suitable result at [%s]' % provider.name) + + def search_providers( show_obj, # type: TVShow ep_obj_list, # type: List[TVEpisode] @@ -736,6 +800,7 @@ def search_providers( final_results = [] search_done = False + search_threads = [] orig_thread_name = threading.currentThread().name @@ -743,71 +808,39 @@ def search_providers( getattr(x, 'enable_backlog', None) and (not torrent_only or GenericProvider.TORRENT == x.providerType) and (not scheduled or getattr(x, 'enable_scheduled_backlog', None))] + + # create a thread for each provider to search for cur_provider in provider_list: if cur_provider.anime_only and not show_obj.is_anime: logger.log(u'%s is not an anime, skipping' % show_obj.name, logger.DEBUG) continue - threading.currentThread().name = '%s :: [%s]' % (orig_thread_name, cur_provider.name) provider_id = cur_provider.get_id() found_results[provider_id] = {} + search_threads.append(threading.Thread(target=_search_provider_thread, + kwargs={'cur_provider': cur_provider, + 'provider_results': found_results[provider_id], + 'show_obj': show_obj, + 'ep_obj_list': ep_obj_list, + 'manual_search': manual_search, + 'try_other_searches': try_other_searches}, + name='%s :: [%s]' % (orig_thread_name, cur_provider.name))) + + # start the provider search thread + search_threads[-1].start() + search_done = True - search_count = 0 - search_mode = getattr(cur_provider, 'search_mode', 'eponly') - - while True: - search_count += 1 - - if 'eponly' == search_mode: - logger.log(u'Performing episode search for %s' % show_obj.name) - else: - logger.log(u'Performing season pack search for %s' % show_obj.name) - - search_result_list = {} - try: - cur_provider.cache._clearCache() - search_result_list = cur_provider.find_search_results(show_obj, ep_obj_list, search_mode, manual_search, - try_other_searches=try_other_searches) - if any(search_result_list): - logger.log(', '.join(['%s %s candidate%s' % ( - len(v), (('multiep', 'season')[SEASON_RESULT == k], 'episode')['ep' in search_mode], - helpers.maybe_plural(v)) for (k, v) in iteritems(search_result_list)])) - except exceptions_helper.AuthException as e: - logger.log(u'Authentication error: %s' % ex(e), logger.ERROR) - break - except (BaseException, Exception) as e: - logger.log(u'Error while searching %s, skipping: %s' % (cur_provider.name, ex(e)), logger.ERROR) - logger.log(traceback.format_exc(), logger.ERROR) - break - finally: - threading.currentThread().name = orig_thread_name - - search_done = True - - if len(search_result_list): - # make a list of all the results for this provider - for cur_search_result in search_result_list: - # skip non-tv crap - search_result_list[cur_search_result] = filter_list( - lambda ep_item: ep_item.show_obj == show_obj and show_name_helpers.pass_wordlist_checks( - ep_item.name, parse=False, indexer_lookup=False, show_obj=ep_item.show_obj), - search_result_list[cur_search_result]) - - if cur_search_result in found_results: - found_results[provider_id][cur_search_result] += search_result_list[cur_search_result] - else: - found_results[provider_id][cur_search_result] = search_result_list[cur_search_result] - - break - elif not getattr(cur_provider, 'search_fallback', False) or 2 == search_count: - break + # wait for all searches to finish + for s_t in search_threads: + s_t.join() - search_mode = '%sonly' % ('ep', 'sp')['ep' in search_mode] - logger.log(u'Falling back to %s search ...' % ('season pack', 'episode')['ep' in search_mode]) + # now look in all the results + for cur_provider in provider_list: + provider_id = cur_provider.get_id() # skip to next provider if we have no results to process - if not len(found_results[provider_id]): + if provider_id not in found_results or not len(found_results[provider_id]): continue any_qualities, best_qualities = Quality.splitQuality(show_obj.quality) @@ -823,8 +856,7 @@ def search_providers( for cur_result in found_results[provider_id][cur_episode]: if Quality.UNKNOWN != cur_result.quality and highest_quality_overall < cur_result.quality: highest_quality_overall = cur_result.quality - logger.log(u'%s is the highest quality of any match' % Quality.qualityStrings[highest_quality_overall], - logger.DEBUG) + logger.debug(u'%s is the highest quality of any match' % Quality.qualityStrings[highest_quality_overall]) # see if every episode is wanted if best_season_result: @@ -1077,8 +1109,7 @@ def search_providers( break if not len(provider_list): - logger.log('No NZB/Torrent providers in Media Providers/Options are allowed for active searching', - logger.WARNING) + logger.warning('No NZB/Torrent providers in Media Providers/Options are allowed for active searching') elif not search_done: logger.log('Failed active search of %s enabled provider%s. More info in debug log.' % ( len(provider_list), helpers.maybe_plural(provider_list)), logger.ERROR)