Merge branch 'feature/AddConcurrentSearch' into develop

CHANGES.md

@@ -1,5 +1,6 @@
 ### 0.23.0 (2019-xx-xx xx:xx:xx UTC)
 
+* Change improve search performance for backlog, manual, failed, and proper
 * Add overview of the last release age/date at each newznab provider to History/Layout "Connect fails" 
 * Add "History new..." to Shows menu by clicking the number
 * Add db backup to the scheduled daily update

sickbeard/properFinder.py

@@ -36,20 +36,21 @@ from .history import dateFormat
 from .name_parser.parser import InvalidNameException, InvalidShowException, NameParser
 from .sgdatetime import timestamp_near
 
-from _23 import filter_iter, list_values, map_consume, map_list
+from _23 import filter_iter, filter_list, list_values, map_consume, map_list
 from six import string_types
 
 # noinspection PyUnreachableCode
 if False:
     # noinspection PyUnresolvedReferences
-    from typing import AnyStr, List, Tuple
+    from typing import AnyStr, Dict, List, Tuple
+    from .providers.generic import GenericProvider
 
 
 def search_propers(provider_proper_obj=None):
+    # type: (Dict[AnyStr, List[Proper]]) -> None
     """
 
     :param provider_proper_obj: Optional dict with provider keys containing Proper objects
-    :type provider_proper_obj: dict
     :return:
     """
     if not sickbeard.DOWNLOAD_PROPERS:
@@ -214,19 +215,35 @@ def load_webdl_types():
     sickbeard.WEBDL_TYPES = new_types + default_types
 
 
-def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_dict=None):
+def _search_provider(cur_provider, provider_propers, aired_since_shows, recent_shows, recent_anime):
+    # type: (GenericProvider, List, datetime.datetime, List[Tuple[int, int]], List[Tuple[int, int]]) -> None
+    try:
+        # we need to extent the referenced list from parameter to update the original var
+        provider_propers.extend(cur_provider.find_propers(search_date=aired_since_shows, shows=recent_shows,
+                                                          anime=recent_anime))
+    except AuthException as e:
+        logger.log('Authentication error: %s' % ex(e), logger.ERROR)
+    except (BaseException, Exception) as e:
+        logger.log('Error while searching %s, skipping: %s' % (cur_provider.name, ex(e)), logger.ERROR)
+        logger.log(traceback.format_exc(), logger.ERROR)
+
+    if not provider_propers:
+        logger.log('No Proper releases found at [%s]' % cur_provider.name)
+
+
+def _get_proper_list(aired_since_shows,  # type: datetime.datetime
+                     recent_shows,  # type: List[Tuple[int, int]]
+                     recent_anime,  # type:  List[Tuple[int, int]]
+                     proper_dict=None  # type:  Dict[AnyStr, List[Proper]]
+                     ):
+    # type: (...) -> List[Proper]
     """
 
     :param aired_since_shows: date since aired
-    :type aired_since_shows: datetime.datetime
     :param recent_shows: list of recent shows
-    :type recent_shows: List[Tuple[int, int]]
     :param recent_anime: list of recent anime shows
-    :type recent_anime: List[Tuple[int, int]]
     :param proper_dict: dict with provider keys containing Proper objects
-    :type proper_dict: dict
     :return: list of propers
-    :rtype: List[sickbeard.classes.Proper]
     """
     propers = {}
     # make sure the episode has been downloaded before
@@ -235,31 +252,47 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_dict=
     my_db = db.DBConnection()
     # for each provider get a list of arbitrary Propers
     orig_thread_name = threading.currentThread().name
-    for cur_provider in filter_iter(lambda p: p.is_active(), sickbeard.providers.sortedProviderList()):
-        if not recent_anime and cur_provider.anime_only:
-            continue
-
-        if None is not proper_dict:
-            found_propers = proper_dict.get(cur_provider.get_id(), [])
-            if not found_propers:
+    # filter provider list for:
+    # 1. from recent search: recent search enabled providers
+    # 2. native proper search: active search enabled providers
+    provider_list = filter_list(
+        lambda p: p.is_active() and (p.enable_recentsearch, p.enable_backlog)[None is proper_dict],
+        sickbeard.providers.sortedProviderList())
+    search_threads = []
+
+    if None is proper_dict:
+        # if not a recent proper search create a thread per provider to search for Propers
+        proper_dict = {}
+        for cur_provider in provider_list:
+            if not recent_anime and cur_provider.anime_only:
                 continue
-        else:
-            threading.currentThread().name = '%s :: [%s]' % (orig_thread_name, cur_provider.name)
 
-            logger.log('Searching for new PROPER releases')
+            provider_id = cur_provider.get_id()
 
-            try:
-                found_propers = cur_provider.find_propers(search_date=aired_since_shows, shows=recent_shows,
-                                                          anime=recent_anime)
-            except AuthException as e:
-                logger.log('Authentication error: %s' % ex(e), logger.ERROR)
-                continue
-            except (BaseException, Exception) as e:
-                logger.log('Error while searching %s, skipping: %s' % (cur_provider.name, ex(e)), logger.ERROR)
-                logger.log(traceback.format_exc(), logger.ERROR)
-                continue
-            finally:
-                threading.currentThread().name = orig_thread_name
+            logger.log('Searching for new Proper releases at [%s]' % cur_provider.name)
+            proper_dict[provider_id] = []
+
+            search_threads.append(threading.Thread(target=_search_provider,
+                                                   kwargs={'cur_provider': cur_provider,
+                                                           'provider_propers': proper_dict[provider_id],
+                                                           'aired_since_shows': aired_since_shows,
+                                                           'recent_shows': recent_shows,
+                                                           'recent_anime': recent_anime},
+                                                   name='%s :: [%s]' % (orig_thread_name, cur_provider.name)))
+
+            search_threads[-1].start()
+
+        # wait for all searches to finish
+        for cur_thread in search_threads:
+            cur_thread.join()
+
+    for cur_provider in provider_list:
+        if not recent_anime and cur_provider.anime_only:
+            continue
+
+        found_propers = proper_dict.get(cur_provider.get_id(), [])
+        if not found_propers:
+            continue
 
         # if they haven't been added by a different provider than add the Proper to the list
         for cur_proper in found_propers:
@@ -277,7 +310,7 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_dict=
             cur_proper.parsed_show_obj = (cur_proper.parsed_show_obj
                                           or helpers.find_show_by_id(parse_result.show_obj.tvid_prodid))
             if None is cur_proper.parsed_show_obj:
-                logger.log('Skip download; cannot find show with ID [%s] from %s' %
+                logger.log('Skip download; cannot find show with ID [%s] at %s' %
                            (cur_proper.prodid, sickbeard.TVInfoAPI(cur_proper.tvid).name), logger.ERROR)
                 continue
 
@@ -462,11 +495,11 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_dict=
 
 
 def _download_propers(proper_list):
+    # type: (List[Proper]) -> None
     """
     download propers from given list
 
     :param proper_list: proper list
-    :type proper_list: List[sickbeard.classes.Proper]
     """
     verified_propers = True
     consumed_proper = []
@@ -560,12 +593,11 @@ def _download_propers(proper_list):
 
 
 def get_needed_qualites(needed=None):
+    # type: (sickbeard.common.NeededQualities) -> sickbeard.common.NeededQualities
     """
 
     :param needed: optional needed object
-    :type needed: sickbeard.common.NeededQualities
     :return: needed object
-    :rtype: sickbeard.common.NeededQualities
     """
     if not isinstance(needed, NeededQualities):
         needed = NeededQualities()

sickbeard/providers/__init__.py

@@ -29,7 +29,8 @@ from six import iteritems, itervalues
 
 # noinspection PyUnreachableCode
 if False:
-    from typing import AnyStr, List
+    from typing import AnyStr, List, Union
+    from .generic import GenericProvider, NZBProvider, TorrentProvider
 
 __all__ = [
     # usenet
@@ -57,6 +58,12 @@ for module in __all__:
 
 
 def sortedProviderList():
+    # type: (...) -> List[Union[GenericProvider, NZBProvider, TorrentProvider]]
+    """
+    return sorted provider list
+
+    :return: sorted list of providers
+    """
     initialList = sickbeard.providerList + sickbeard.newznabProviderList + sickbeard.torrentRssProviderList
     providerDict = dict(zip([x.get_id() for x in initialList], initialList))
 

sickbeard/search.py

@@ -710,6 +710,70 @@ def can_reject(release_name):
     return pred and (None, None) or (predb_rej or True,  ', '.join(rej_urls))
 
 
+def _search_provider_thread(provider, provider_results, show_obj, ep_obj_list, manual_search, try_other_searches):
+    # type: (GenericProvider, Dict, TVShow, List[TVEpisode], bool, bool) -> None
+    """
+    perform a search on a provider for specified show, episodes
+
+    :param provider: Provider to search
+    :param provider_results: reference to dict to return results
+    :param show_obj: show to search for
+    :param ep_obj_list: list of episodes to search for
+    :param manual_search: is manual search
+    :param try_other_searches: try other search methods
+    """
+    search_count = 0
+    search_mode = getattr(provider, 'search_mode', 'eponly')
+
+    while True:
+        search_count += 1
+
+        if 'eponly' == search_mode:
+            logger.log(u'Performing episode search for %s' % show_obj.name)
+        else:
+            logger.log(u'Performing season pack search for %s' % show_obj.name)
+
+        try:
+            provider.cache._clearCache()
+            search_result_list = provider.find_search_results(show_obj, ep_obj_list, search_mode, manual_search,
+                                                              try_other_searches=try_other_searches)
+            if any(search_result_list):
+                logger.log(', '.join(['%s %s candidate%s' % (
+                    len(v), (('multiep', 'season')[SEASON_RESULT == k], 'episode')['ep' in search_mode],
+                    helpers.maybe_plural(v)) for (k, v) in iteritems(search_result_list)]))
+        except exceptions_helper.AuthException as e:
+            logger.error(u'Authentication error: %s' % ex(e))
+            break
+        except (BaseException, Exception) as e:
+            logger.error(u'Error while searching %s, skipping: %s' % (provider.name, ex(e)))
+            logger.error(traceback.format_exc())
+            break
+
+        if len(search_result_list):
+            # make a list of all the results for this provider
+            for cur_search_result in search_result_list:
+                # skip non-tv crap
+                search_result_list[cur_search_result] = filter_list(
+                    lambda ep_item: ep_item.show_obj == show_obj and show_name_helpers.pass_wordlist_checks(
+                        ep_item.name, parse=False, indexer_lookup=False, show_obj=ep_item.show_obj),
+                    search_result_list[cur_search_result])
+
+                if cur_search_result in provider_results:
+                    provider_results[cur_search_result] += search_result_list[cur_search_result]
+                else:
+                    provider_results[cur_search_result] = search_result_list[cur_search_result]
+
+            break
+        elif not getattr(provider, 'search_fallback', False) or 2 == search_count:
+            break
+
+        search_mode = '%sonly' % ('ep', 'sp')['ep' in search_mode]
+        logger.log(u'Falling back to %s search ...' % ('season pack', 'episode')['ep' in search_mode])
+
+    if not provider_results:
+        logger.log('No suitable result at [%s]' % provider.name)
+
+
 def search_providers(
         show_obj,  # type: TVShow
         ep_obj_list,  # type: List[TVEpisode]
@@ -736,6 +800,7 @@ def search_providers(
     final_results = []
 
     search_done = False
+    search_threads = []
 
     orig_thread_name = threading.currentThread().name
 
@@ -743,71 +808,39 @@ def search_providers(
                      getattr(x, 'enable_backlog', None) and
                      (not torrent_only or GenericProvider.TORRENT == x.providerType) and
                      (not scheduled or getattr(x, 'enable_scheduled_backlog', None))]
+
+    # create a thread for each provider to search
     for cur_provider in provider_list:
         if cur_provider.anime_only and not show_obj.is_anime:
             logger.log(u'%s is not an anime, skipping' % show_obj.name, logger.DEBUG)
             continue
 
-        threading.currentThread().name = '%s :: [%s]' % (orig_thread_name, cur_provider.name)
         provider_id = cur_provider.get_id()
 
         found_results[provider_id] = {}
+        search_threads.append(threading.Thread(target=_search_provider_thread,
+                                               kwargs={'cur_provider': cur_provider,
+                                                       'provider_results': found_results[provider_id],
+                                                       'show_obj': show_obj,
+                                                       'ep_obj_list': ep_obj_list,
+                                                       'manual_search': manual_search,
+                                                       'try_other_searches': try_other_searches},
+                                               name='%s :: [%s]' % (orig_thread_name, cur_provider.name)))
+
+        # start the provider search thread
+        search_threads[-1].start()
+        search_done = True
 
-        search_count = 0
-        search_mode = getattr(cur_provider, 'search_mode', 'eponly')
-
-        while True:
-            search_count += 1
-
-            if 'eponly' == search_mode:
-                logger.log(u'Performing episode search for %s' % show_obj.name)
-            else:
-                logger.log(u'Performing season pack search for %s' % show_obj.name)
-
-            search_result_list = {}
-            try:
-                cur_provider.cache._clearCache()
-                search_result_list = cur_provider.find_search_results(show_obj, ep_obj_list, search_mode, manual_search,
-                                                                      try_other_searches=try_other_searches)
-                if any(search_result_list):
-                    logger.log(', '.join(['%s %s candidate%s' % (
-                        len(v), (('multiep', 'season')[SEASON_RESULT == k], 'episode')['ep' in search_mode],
-                        helpers.maybe_plural(v)) for (k, v) in iteritems(search_result_list)]))
-            except exceptions_helper.AuthException as e:
-                logger.log(u'Authentication error: %s' % ex(e), logger.ERROR)
-                break
-            except (BaseException, Exception) as e:
-                logger.log(u'Error while searching %s, skipping: %s' % (cur_provider.name, ex(e)), logger.ERROR)
-                logger.log(traceback.format_exc(), logger.ERROR)
-                break
-            finally:
-                threading.currentThread().name = orig_thread_name
-
-            search_done = True
-
-            if len(search_result_list):
-                # make a list of all the results for this provider
-                for cur_search_result in search_result_list:
-                    # skip non-tv crap
-                    search_result_list[cur_search_result] = filter_list(
-                        lambda ep_item: ep_item.show_obj == show_obj and show_name_helpers.pass_wordlist_checks(
-                            ep_item.name, parse=False, indexer_lookup=False, show_obj=ep_item.show_obj),
-                        search_result_list[cur_search_result])
-
-                    if cur_search_result in found_results:
-                        found_results[provider_id][cur_search_result] += search_result_list[cur_search_result]
-                    else:
-                        found_results[provider_id][cur_search_result] = search_result_list[cur_search_result]
-
-                break
-            elif not getattr(cur_provider, 'search_fallback', False) or 2 == search_count:
-                break
+    # wait for all searches to finish
+    for s_t in search_threads:
+        s_t.join()
 
-            search_mode = '%sonly' % ('ep', 'sp')['ep' in search_mode]
-            logger.log(u'Falling back to %s search ...' % ('season pack', 'episode')['ep' in search_mode])
+    # now look in all the results
+    for cur_provider in provider_list:
+        provider_id = cur_provider.get_id()
 
         # skip to next provider if we have no results to process
-        if not len(found_results[provider_id]):
+        if provider_id not in found_results or not len(found_results[provider_id]):
             continue
 
         any_qualities, best_qualities = Quality.splitQuality(show_obj.quality)
@@ -823,8 +856,7 @@ def search_providers(
             for cur_result in found_results[provider_id][cur_episode]:
                 if Quality.UNKNOWN != cur_result.quality and highest_quality_overall < cur_result.quality:
                     highest_quality_overall = cur_result.quality
-        logger.log(u'%s is the highest quality of any match' % Quality.qualityStrings[highest_quality_overall],
-                   logger.DEBUG)
+        logger.debug(u'%s is the highest quality of any match' % Quality.qualityStrings[highest_quality_overall])
 
         # see if every episode is wanted
         if best_season_result:
@@ -1077,8 +1109,7 @@ def search_providers(
             break
 
     if not len(provider_list):
-        logger.log('No NZB/Torrent providers in Media Providers/Options are allowed for active searching',
-                   logger.WARNING)
+        logger.warning('No NZB/Torrent providers in Media Providers/Options are allowed for active searching')
     elif not search_done:
         logger.log('Failed active search of %s enabled provider%s. More info in debug log.' % (
             len(provider_list), helpers.maybe_plural(provider_list)), logger.ERROR)