Change scantree into a more multi-purpose function and use to improve performance.

Change improve list performance for file/directory browser.
Change improve import shows listing performance.
Change improve performance during show rescan process.
Change improve performance during media processing.

CHANGES.md

@@ -1,5 +1,9 @@
 ### 0.23.0 (2019-xx-xx xx:xx:xx UTC)
 
+* Change improve list performance for file/directory browser
+* Change improve import shows listing performance
+* Change improve performance during show rescan process
+* Change improve performance during media processing
 * Add config/General/Updates/Alias Process button, minimum interval for a fetch of custom names/numbering is 30 mins
 * Add Export alternatives button to edit show
 * Change season specific alt names now available not just for anime

sickbeard/browser.py

@@ -29,6 +29,7 @@ import encodingKludge as ek
 from exceptions_helper import ex
 
 from . import logger
+from .helpers import scantree
 
 # this is for the drive letter code, it only works on windows
 if 'nt' == os.name:
@@ -122,11 +123,7 @@ def get_file_list(path, include_files):
         '.git']
 
     # filter directories to protect
-    for name in ek.ek(os.listdir, path):
+    for direntry in scantree(path, exclude=hide_names, filter_kind=not include_files, recurse=False) or []:
-        if name.lower() not in hide_names:
+        result.append(dict(name=direntry.name, path=direntry.path, isFile=int(direntry.is_file(follow_symlinks=False))))
-            path_file = ek.ek(os.path.join, path, name)
-            is_dir = ek.ek(os.path.isdir, path_file)
-            if include_files or is_dir:
-                result.append({'name': name, 'path': path_file, 'isFile': (1, 0)[is_dir]})
 
     return result

sickbeard/helpers.py

@@ -28,7 +28,6 @@ import os
 import re
 import shutil
 import socket
-import stat
 import time
 import uuid
 import subprocess
@@ -360,27 +359,14 @@ def list_media_files(path):
     :param path: path
     :return: list of media files
     """
-    if not dir or not ek.ek(os.path.isdir, path):
+    result = []
-        return []
+    if path:
-
+        if '.sickgearignore' in [direntry.name for direntry in scantree(path, filter_kind=False, recurse=False)]:
-    files = []
+            logger.log('Skipping folder "%s" because it contains ".sickgearignore"' % path, logger.DEBUG)
-    file_list = ek.ek(os.listdir, path)
+        else:
-
+            result = [direntry.path for direntry in scantree(path, filter_kind=False, exclude='Extras')
-    if '.sickgearignore' in file_list:
+                      if has_media_ext(direntry.name)]
-        logger.log('Folder "%s" contains ".sickgearignore", ignoring Folder' % path, logger.DEBUG)
+    return result
-        return []
-
-    for cur_file in file_list:
-        full_cur_file = ek.ek(os.path.join, path, cur_file)  # type: AnyStr
-
-        # if it's a folder do it recursively
-        if ek.ek(os.path.isdir, full_cur_file) and not cur_file.startswith('.') and 'Extras' != cur_file:
-            files += list_media_files(full_cur_file)
-
-        elif has_media_ext(cur_file):
-            files.append(full_cur_file)
-
-    return files
 
 
 def copyFile(src_file, dest_file):
@@ -1385,23 +1371,33 @@ def cpu_sleep():
         time.sleep(cpu_presets[sickbeard.CPU_PRESET])
 
 
-def scantree(path, exclude=None, follow_symlinks=False):
+def scantree(path,  # type: AnyStr
-    # type: (AnyStr, Optional[AnyStr, List[AnyStr]], bool) -> Optional[Iterator[DirEntry], Iterable]
+             exclude=None,  # type: Optional[AnyStr, List[AnyStr]]
-    """Recursively yield DirEntry objects for given directory.
+             follow_symlinks=False,  # type: bool
-    :param path: path
+             filter_kind=None,  # type: Optional[bool]
-    :param exclude: excludes
+             recurse=True  # type: bool
-    :param follow_symlinks: follow symlinks
+             ):
+    # type: (...) -> Optional[Iterator[DirEntry], Iterable]
+    """yield DirEntry objects for given path.
+    :param path: Path to scan
+    :param exclude: Exclusions
+    :param follow_symlinks: Follow symlinks
+    :param filter_kind: None to yield everything, True only yields directories, False only yields files
+    :param recurse: Recursively scan down the tree
     :return: iter of results
     """
-    exclude = (exclude, ([exclude], [])[None is exclude])[not isinstance(exclude, list)]
+    exclude = [x.lower() for x in (exclude, ([exclude], [])[None is exclude])[not isinstance(exclude, list)]]
     for entry in ek.ek(scandir, path):
-        if entry.is_dir(follow_symlinks=follow_symlinks):
+        is_dir = entry.is_dir(follow_symlinks=follow_symlinks)
-            if entry.name not in exclude:
+        is_file = entry.is_file(follow_symlinks=follow_symlinks)
-                for subentry in scantree(entry.path):
+        if entry.name.lower() not in exclude \
+                and any([None is filter_kind, filter_kind and is_dir,
+                         not filter_kind and is_dir and recurse, not filter_kind and is_file]):
+            if recurse and is_dir:
+                for subentry in scantree(entry.path, exclude, follow_symlinks, filter_kind):
                     yield subentry
+            if any([None is filter_kind, filter_kind and is_dir, not filter_kind and is_file]):
                 yield entry
-        else:
-            yield entry
 
 
 def cleanup_cache():
@@ -1604,6 +1600,8 @@ def get_overview(ep_status, show_quality, upgrade_once, split_snatch=False):
     :type show_quality: int
     :param upgrade_once: upgrade once
     :type upgrade_once: bool
+    :param split_snatch:
+    :type split_snatch: bool
     :return: constant from classes Overview
     :rtype: int
     """

sickbeard/processTV.py

@@ -38,6 +38,7 @@ from exceptions_helper import ex, MultipleShowObjectsException
 import sickbeard
 from . import common, db, failedProcessor, helpers, logger, notifiers, postProcessor
 from .common import SNATCHED_ANY
+from .helpers import scantree
 from .history import reset_status
 from .name_parser.parser import InvalidNameException, InvalidShowException, NameParser
 from .sgdatetime import timestamp_near
@@ -137,7 +138,7 @@ class ProcessTVShow(object):
             return False
 
         # check if it's empty folder when wanted checked
-        if check_empty and ek.ek(os.listdir, folder):
+        if check_empty and len([direntry.path for direntry in scantree(folder, recurse=False)]):
             return False
 
         # try deleting folder

sickbeard/show_name_helpers.py

@@ -27,11 +27,11 @@ from exceptions_helper import ex
 
 import sickbeard
 from . import common, db, logger
-from .helpers import sanitize_scene_name
+from .helpers import sanitize_scene_name, scantree
 from .name_parser.parser import InvalidNameException, InvalidShowException, NameParser
 from .scene_exceptions import get_scene_exceptions
 
-from _23 import filter_list, map_list, quote_plus
+from _23 import map_list, quote_plus
 from six import iterkeys, itervalues
 
 # noinspection PyUnreachableCode
@@ -440,20 +440,19 @@ def determineReleaseName(dir_name=None, nzb_name=None):
         return None
 
     # try to get the release name from nzb/nfo
-    file_types = ["*.nzb", "*.nfo"]
+    file_types = ['*.nzb', '*.nfo']
 
     for search in file_types:
 
         reg_expr = re.compile(fnmatch.translate(search), re.IGNORECASE)
-        files = [file_name for file_name in ek.ek(os.listdir, dir_name) if
+        results = [direntry.name for direntry in scantree(dir_name, filter_kind=False, recurse=False)
-                 ek.ek(os.path.isfile, ek.ek(os.path.join, dir_name, file_name))]
+                   if reg_expr.search(direntry.name)]
-        results = filter_list(reg_expr.search, files)
 
         if 1 == len(results):
             found_file = ek.ek(os.path.basename, results[0])
             found_file = found_file.rpartition('.')[0]
             if pass_wordlist_checks(found_file):
-                logger.log(u"Release name (" + found_file + ") found from file (" + results[0] + ")")
+                logger.log(u'Release name (%s) found from file (%s)' % (found_file, results[0]))
                 return found_file.rpartition('.')[0]
 
     # If that fails, we try the folder
@@ -462,7 +461,7 @@ def determineReleaseName(dir_name=None, nzb_name=None):
         # NOTE: Multiple failed downloads will change the folder name.
         # (e.g., appending #s)
         # Should we handle that?
-        logger.log(u"Folder name (" + folder + ") appears to be a valid release name. Using it.")
+        logger.log(u'Folder name (%s) appears to be a valid release name. Using it.' % folder)
         return folder
 
     return None

sickbeard/webserve.py

@@ -52,7 +52,7 @@ from .anime import AniGroupList, pull_anidb_groups, short_group_names
 from .browser import folders_at_path
 from .common import ARCHIVED, DOWNLOADED, FAILED, IGNORED, SKIPPED, SNATCHED, SNATCHED_ANY, UNAIRED, UNKNOWN, WANTED, \
      SD, HD720p, HD1080p, UHD2160p, Overview, Quality, qualityPresetStrings, statusStrings
-from .helpers import has_image_ext, remove_article, starify
+from .helpers import has_image_ext, remove_article, scantree, starify
 from .indexermapper import MapStatus, map_indexers_to_show, save_mapping
 from .indexers.indexer_config import TVINFO_IMDB, TVINFO_TRAKT, TVINFO_TVDB
 from .name_parser.parser import InvalidNameException, InvalidShowException, NameParser
@@ -3644,15 +3644,13 @@ class AddShows(Home):
             try:
                 for root_dir in sickbeard.ROOT_DIRS.split('|')[1:]:
                     try:
-                        file_list = ek.ek(os.listdir, root_dir)
+                        file_list = [x for x in scantree(root_dir, filter_kind=True, recurse=False)]
                     except (BaseException, Exception):
                         continue
 
                     for cur_file in file_list:
 
-                        cur_path = ek.ek(os.path.normpath, ek.ek(os.path.join, root_dir, cur_file))
+                        cur_path = ek.ek(os.path.normpath, cur_file.path)
-                        if not ek.ek(os.path.isdir, cur_path):
-                            continue
 
                         display_one_dir = hash_dir == str(abs(hash(cur_path)))
                         if display_one_dir:
@@ -3664,20 +3662,18 @@ class AddShows(Home):
         for root_dir in root_dirs:
             if not file_list:
                 try:
-                    file_list = ek.ek(os.listdir, root_dir)
+                    file_list = [x for x in scantree(root_dir, filter_kind=True, recurse=False)]
                 except (BaseException, Exception):
                     continue
 
             for cur_file in file_list:
 
-                cur_path = ek.ek(os.path.normpath, ek.ek(os.path.join, root_dir, cur_file))
+                cur_path = ek.ek(os.path.normpath, cur_file.path)
-                if not ek.ek(os.path.isdir, cur_path):
-                    continue
 
                 highlight = hash_dir == str(abs(hash(cur_path)))
                 if display_one_dir and not highlight:
                     continue
-                cur_dir = dict(dir=cur_path, highlight=highlight, name=ek.ek(os.path.basename, cur_path),
+                cur_dir = dict(dir=cur_path, highlight=highlight, name=cur_file.name,
                                path='%s%s' % (ek.ek(os.path.dirname, cur_path), os.sep),
                                added_already=any(my_db.select(
                                    'SELECT indexer'