From d3fc24454556dd6004f13341976de38da7df2ac1 Mon Sep 17 00:00:00 2001 From: JackDandy Date: Thu, 15 Oct 2020 02:19:10 +0100 Subject: [PATCH] Change scantree into a more multi-purpose function and use to improve performance. Change improve list performance for file/directory browser. Change improve import shows listing performance. Change improve performance during show rescan process. Change improve performance during media processing. --- CHANGES.md | 4 +++ sickbeard/browser.py | 9 ++---- sickbeard/helpers.py | 66 ++++++++++++++++++++---------------------- sickbeard/processTV.py | 3 +- sickbeard/show_name_helpers.py | 15 +++++----- sickbeard/webserve.py | 16 ++++------ 6 files changed, 54 insertions(+), 59 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cb22e1c..c916234 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ ### 0.23.0 (2019-xx-xx xx:xx:xx UTC) +* Change improve list performance for file/directory browser +* Change improve import shows listing performance +* Change improve performance during show rescan process +* Change improve performance during media processing * Add config/General/Updates/Alias Process button, minimum interval for a fetch of custom names/numbering is 30 mins * Add Export alternatives button to edit show * Change season specific alt names now available not just for anime diff --git a/sickbeard/browser.py b/sickbeard/browser.py index 1472559..c229129 100644 --- a/sickbeard/browser.py +++ b/sickbeard/browser.py @@ -29,6 +29,7 @@ import encodingKludge as ek from exceptions_helper import ex from . import logger +from .helpers import scantree # this is for the drive letter code, it only works on windows if 'nt' == os.name: @@ -122,11 +123,7 @@ def get_file_list(path, include_files): '.git'] # filter directories to protect - for name in ek.ek(os.listdir, path): - if name.lower() not in hide_names: - path_file = ek.ek(os.path.join, path, name) - is_dir = ek.ek(os.path.isdir, path_file) - if include_files or is_dir: - result.append({'name': name, 'path': path_file, 'isFile': (1, 0)[is_dir]}) + for direntry in scantree(path, exclude=hide_names, filter_kind=not include_files, recurse=False) or []: + result.append(dict(name=direntry.name, path=direntry.path, isFile=int(direntry.is_file(follow_symlinks=False)))) return result diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index 439f27b..f939a99 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -28,7 +28,6 @@ import os import re import shutil import socket -import stat import time import uuid import subprocess @@ -360,27 +359,14 @@ def list_media_files(path): :param path: path :return: list of media files """ - if not dir or not ek.ek(os.path.isdir, path): - return [] - - files = [] - file_list = ek.ek(os.listdir, path) - - if '.sickgearignore' in file_list: - logger.log('Folder "%s" contains ".sickgearignore", ignoring Folder' % path, logger.DEBUG) - return [] - - for cur_file in file_list: - full_cur_file = ek.ek(os.path.join, path, cur_file) # type: AnyStr - - # if it's a folder do it recursively - if ek.ek(os.path.isdir, full_cur_file) and not cur_file.startswith('.') and 'Extras' != cur_file: - files += list_media_files(full_cur_file) - - elif has_media_ext(cur_file): - files.append(full_cur_file) - - return files + result = [] + if path: + if '.sickgearignore' in [direntry.name for direntry in scantree(path, filter_kind=False, recurse=False)]: + logger.log('Skipping folder "%s" because it contains ".sickgearignore"' % path, logger.DEBUG) + else: + result = [direntry.path for direntry in scantree(path, filter_kind=False, exclude='Extras') + if has_media_ext(direntry.name)] + return result def copyFile(src_file, dest_file): @@ -1385,23 +1371,33 @@ def cpu_sleep(): time.sleep(cpu_presets[sickbeard.CPU_PRESET]) -def scantree(path, exclude=None, follow_symlinks=False): - # type: (AnyStr, Optional[AnyStr, List[AnyStr]], bool) -> Optional[Iterator[DirEntry], Iterable] - """Recursively yield DirEntry objects for given directory. - :param path: path - :param exclude: excludes - :param follow_symlinks: follow symlinks +def scantree(path, # type: AnyStr + exclude=None, # type: Optional[AnyStr, List[AnyStr]] + follow_symlinks=False, # type: bool + filter_kind=None, # type: Optional[bool] + recurse=True # type: bool + ): + # type: (...) -> Optional[Iterator[DirEntry], Iterable] + """yield DirEntry objects for given path. + :param path: Path to scan + :param exclude: Exclusions + :param follow_symlinks: Follow symlinks + :param filter_kind: None to yield everything, True only yields directories, False only yields files + :param recurse: Recursively scan down the tree :return: iter of results """ - exclude = (exclude, ([exclude], [])[None is exclude])[not isinstance(exclude, list)] + exclude = [x.lower() for x in (exclude, ([exclude], [])[None is exclude])[not isinstance(exclude, list)]] for entry in ek.ek(scandir, path): - if entry.is_dir(follow_symlinks=follow_symlinks): - if entry.name not in exclude: - for subentry in scantree(entry.path): + is_dir = entry.is_dir(follow_symlinks=follow_symlinks) + is_file = entry.is_file(follow_symlinks=follow_symlinks) + if entry.name.lower() not in exclude \ + and any([None is filter_kind, filter_kind and is_dir, + not filter_kind and is_dir and recurse, not filter_kind and is_file]): + if recurse and is_dir: + for subentry in scantree(entry.path, exclude, follow_symlinks, filter_kind): yield subentry + if any([None is filter_kind, filter_kind and is_dir, not filter_kind and is_file]): yield entry - else: - yield entry def cleanup_cache(): @@ -1604,6 +1600,8 @@ def get_overview(ep_status, show_quality, upgrade_once, split_snatch=False): :type show_quality: int :param upgrade_once: upgrade once :type upgrade_once: bool + :param split_snatch: + :type split_snatch: bool :return: constant from classes Overview :rtype: int """ diff --git a/sickbeard/processTV.py b/sickbeard/processTV.py index 00b03ed..65eacfa 100644 --- a/sickbeard/processTV.py +++ b/sickbeard/processTV.py @@ -38,6 +38,7 @@ from exceptions_helper import ex, MultipleShowObjectsException import sickbeard from . import common, db, failedProcessor, helpers, logger, notifiers, postProcessor from .common import SNATCHED_ANY +from .helpers import scantree from .history import reset_status from .name_parser.parser import InvalidNameException, InvalidShowException, NameParser from .sgdatetime import timestamp_near @@ -137,7 +138,7 @@ class ProcessTVShow(object): return False # check if it's empty folder when wanted checked - if check_empty and ek.ek(os.listdir, folder): + if check_empty and len([direntry.path for direntry in scantree(folder, recurse=False)]): return False # try deleting folder diff --git a/sickbeard/show_name_helpers.py b/sickbeard/show_name_helpers.py index 200c0c1..9ddb822 100644 --- a/sickbeard/show_name_helpers.py +++ b/sickbeard/show_name_helpers.py @@ -27,11 +27,11 @@ from exceptions_helper import ex import sickbeard from . import common, db, logger -from .helpers import sanitize_scene_name +from .helpers import sanitize_scene_name, scantree from .name_parser.parser import InvalidNameException, InvalidShowException, NameParser from .scene_exceptions import get_scene_exceptions -from _23 import filter_list, map_list, quote_plus +from _23 import map_list, quote_plus from six import iterkeys, itervalues # noinspection PyUnreachableCode @@ -440,20 +440,19 @@ def determineReleaseName(dir_name=None, nzb_name=None): return None # try to get the release name from nzb/nfo - file_types = ["*.nzb", "*.nfo"] + file_types = ['*.nzb', '*.nfo'] for search in file_types: reg_expr = re.compile(fnmatch.translate(search), re.IGNORECASE) - files = [file_name for file_name in ek.ek(os.listdir, dir_name) if - ek.ek(os.path.isfile, ek.ek(os.path.join, dir_name, file_name))] - results = filter_list(reg_expr.search, files) + results = [direntry.name for direntry in scantree(dir_name, filter_kind=False, recurse=False) + if reg_expr.search(direntry.name)] if 1 == len(results): found_file = ek.ek(os.path.basename, results[0]) found_file = found_file.rpartition('.')[0] if pass_wordlist_checks(found_file): - logger.log(u"Release name (" + found_file + ") found from file (" + results[0] + ")") + logger.log(u'Release name (%s) found from file (%s)' % (found_file, results[0])) return found_file.rpartition('.')[0] # If that fails, we try the folder @@ -462,7 +461,7 @@ def determineReleaseName(dir_name=None, nzb_name=None): # NOTE: Multiple failed downloads will change the folder name. # (e.g., appending #s) # Should we handle that? - logger.log(u"Folder name (" + folder + ") appears to be a valid release name. Using it.") + logger.log(u'Folder name (%s) appears to be a valid release name. Using it.' % folder) return folder return None diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index dde5c51..3e17a40 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -52,7 +52,7 @@ from .anime import AniGroupList, pull_anidb_groups, short_group_names from .browser import folders_at_path from .common import ARCHIVED, DOWNLOADED, FAILED, IGNORED, SKIPPED, SNATCHED, SNATCHED_ANY, UNAIRED, UNKNOWN, WANTED, \ SD, HD720p, HD1080p, UHD2160p, Overview, Quality, qualityPresetStrings, statusStrings -from .helpers import has_image_ext, remove_article, starify +from .helpers import has_image_ext, remove_article, scantree, starify from .indexermapper import MapStatus, map_indexers_to_show, save_mapping from .indexers.indexer_config import TVINFO_IMDB, TVINFO_TRAKT, TVINFO_TVDB from .name_parser.parser import InvalidNameException, InvalidShowException, NameParser @@ -3644,15 +3644,13 @@ class AddShows(Home): try: for root_dir in sickbeard.ROOT_DIRS.split('|')[1:]: try: - file_list = ek.ek(os.listdir, root_dir) + file_list = [x for x in scantree(root_dir, filter_kind=True, recurse=False)] except (BaseException, Exception): continue for cur_file in file_list: - cur_path = ek.ek(os.path.normpath, ek.ek(os.path.join, root_dir, cur_file)) - if not ek.ek(os.path.isdir, cur_path): - continue + cur_path = ek.ek(os.path.normpath, cur_file.path) display_one_dir = hash_dir == str(abs(hash(cur_path))) if display_one_dir: @@ -3664,20 +3662,18 @@ class AddShows(Home): for root_dir in root_dirs: if not file_list: try: - file_list = ek.ek(os.listdir, root_dir) + file_list = [x for x in scantree(root_dir, filter_kind=True, recurse=False)] except (BaseException, Exception): continue for cur_file in file_list: - cur_path = ek.ek(os.path.normpath, ek.ek(os.path.join, root_dir, cur_file)) - if not ek.ek(os.path.isdir, cur_path): - continue + cur_path = ek.ek(os.path.normpath, cur_file.path) highlight = hash_dir == str(abs(hash(cur_path))) if display_one_dir and not highlight: continue - cur_dir = dict(dir=cur_path, highlight=highlight, name=ek.ek(os.path.basename, cur_path), + cur_dir = dict(dir=cur_path, highlight=highlight, name=cur_file.name, path='%s%s' % (ek.ek(os.path.dirname, cur_path), os.sep), added_already=any(my_db.select( 'SELECT indexer'