diff --git a/CHANGES.md b/CHANGES.md index b0f6eb3..beb29bb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,6 +20,7 @@ * Change enable image caching on browse pages * Change update sceneNameCache after scene names are updated * Change add core dedicated base class tvinfo_base to unify future info sources +* Add exclude ignore words and exclude required words to settings/Search, Edit and View show [develop changelog] diff --git a/gui/slick/interfaces/default/apiBuilder.tmpl b/gui/slick/interfaces/default/apiBuilder.tmpl index 8cdea88..dd9e7c7 100644 --- a/gui/slick/interfaces/default/apiBuilder.tmpl +++ b/gui/slick/interfaces/default/apiBuilder.tmpl @@ -191,18 +191,28 @@ addList("setwords", "Optional Param", "", "addwords"); addList("setwords", "$cur_show_obj.name", "&indexerid=$cur_show_obj.prodid&indexer=$cur_show_obj.tvid", "addwords"); #end for -addList("addwords", "Optional Param", "", "removewords"); +addList("addwords", "Add (Optional)", "", "removewords"); addList("addwords", "ignore1", "&add=ignore1", "removewords"); addList("addwords", "ignore2, ignore3", "&add=ignore2|ignore3", "removewords"); -addList("removewords", "Optional Param", "", "useregex"); +addList("removewords", "Remove (Optional)", "", "useregex"); addList("removewords", "ignore1", "&remove=ignore1", "useregex"); addList("removewords", "ignore2", "&remove=ignore2", "useregex"); addList("removewords", "ignore2, ignore3", "&remove=ignore2|ignore3", "useregex"); -addOption("useregex", "Optional Param", "", 1); -addOption("useregex", "as Regex", "®ex=1"); -addOption("useregex", "as Words", "®ex=0"); +addList("useregex", "Optional Param", "", "excludeadd"); +addList("useregex", "as Regex", "®ex=1", "excludeadd"); +addList("useregex", "as Words", "®ex=0", "excludeadd"); + +addList("excludeadd", "Add Exclude (Optional)", "", "excluderemove"); +addList("excludeadd", "ignore1", "&add_exclude=ignore1", "excluderemove"); +addList("excludeadd", "ignore2", "&add_exclude=ignore2", "excluderemove"); +addList("excludeadd", "ignore2, ignore3", "&add_exclude=ignore2|ignore3", "excluderemove"); + +addList("excluderemove", "Remove Exclude (Optional)", "", ""); +addList("excluderemove", "ignore1", "&remove_exclude=ignore1", ""); +addList("excluderemove", "ignore2", "&remove_exclude=ignore2", ""); +addList("excluderemove", "ignore2, ignore3", "&remove_exclude=ignore2|ignore3", ""); addOption("listrequiredwords", "Optional Param", "", 1); #for $cur_show_obj in $sortedShowList: diff --git a/gui/slick/interfaces/default/config_search.tmpl b/gui/slick/interfaces/default/config_search.tmpl index 09d58e3..4e91969 100755 --- a/gui/slick/interfaces/default/config_search.tmpl +++ b/gui/slick/interfaces/default/config_search.tmpl @@ -1,6 +1,6 @@ #import sickbeard #from sickbeard import clients -#from sickbeard.helpers import starify +#from sickbeard.helpers import starify, generate_word_str <% def sg_var(varname, default=False): return getattr(sickbeard, varname, default) %>#slurp# <% def sg_str(varname, default=''): return getattr(sickbeard, varname, default) %>#slurp# ## @@ -143,7 +143,7 @@ @@ -165,7 +177,7 @@ diff --git a/gui/slick/interfaces/default/displayShow.tmpl b/gui/slick/interfaces/default/displayShow.tmpl index 4665c77..01dcd95 100644 --- a/gui/slick/interfaces/default/displayShow.tmpl +++ b/gui/slick/interfaces/default/displayShow.tmpl @@ -2,7 +2,7 @@ #from sickbeard import TVInfoAPI, indexermapper, network_timezones #from sickbeard.common import Overview, qualityPresets, qualityPresetStrings, \ Quality, statusStrings, WANTED, SKIPPED, ARCHIVED, IGNORED, FAILED, DOWNLOADED -#from sickbeard.helpers import anon_url, get_size, human, maybe_plural +#from sickbeard.helpers import anon_url, get_size, human, maybe_plural, generate_word_str #from sickbeard.indexers.indexer_config import TVINFO_TVDB, TVINFO_IMDB #from six import iteritems <% def sg_var(varname, default=False): return getattr(sickbeard, varname, default) %>#slurp# @@ -348,10 +348,16 @@ Scene names #end if #if $show_obj.rls_ignore_words - Ignored words + Ignored words #end if #if $show_obj.rls_require_words - Required words + Required words +#end if +#if $show_obj.rls_global_exclude_ignore + Excluded global ignored words +#end if +#if $show_obj.rls_global_exclude_require + Excluded global required words #end if #if $show_obj.flatten_folders or $sg_var('NAMING_FORCE_FOLDERS') Flat folders diff --git a/gui/slick/interfaces/default/editShow.tmpl b/gui/slick/interfaces/default/editShow.tmpl index 5544ae8..ee21f6e 100644 --- a/gui/slick/interfaces/default/editShow.tmpl +++ b/gui/slick/interfaces/default/editShow.tmpl @@ -1,8 +1,8 @@ #import sickbeard #import lib.adba as adba #from sickbeard import (anime, common, helpers, scene_exceptions) +#from sickbeard.helpers import anon_url, generate_word_str #from lib import exceptions_helper as exceptions -#from sickbeard.helpers import anon_url #from sickbeard.tv import TVidProdid <% def sg_var(varname, default=False): return getattr(sickbeard, varname, default) %>#slurp# <% def sg_str(varname, default=''): return getattr(sickbeard, varname, default) %>#slurp# @@ -131,24 +131,75 @@ +#if $sickbeard.IGNORE_WORDS: +
+ +
+#end if +
+#if $sickbeard.REQUIRE_WORDS: +
+ +
+#end if + +
#set $qualities = $common.Quality.splitQuality(int($show_obj.quality)) #set global $any_qualities = $qualities[0] diff --git a/sickbeard/__init__.py b/sickbeard/__init__.py index 22ed12a..018c35a 100755 --- a/sickbeard/__init__.py +++ b/sickbeard/__init__.py @@ -539,10 +539,14 @@ SG_EXTRA_SCRIPTS = [] GIT_PATH = None -IGNORE_WORDS = 'regex:^(?=.*?\bspanish\b)((?!spanish.?princess).)*$, ' + \ - 'core2hd, hevc, MrLss, reenc, x265, danish, deutsch, dutch, flemish, french, ' + \ - 'german, italian, nordic, norwegian, portuguese, swedish, turkish' -REQUIRE_WORDS = '' +IGNORE_WORDS = { + '^(?=.*?\bspanish\b)((?!spanish.?princess).)*$', + 'core2hd', 'hevc', 'MrLss', 'reenc', 'x265', 'danish', 'deutsch', 'dutch', 'flemish', 'french', + 'german', 'italian', 'nordic', 'norwegian', 'portuguese', 'spanish', 'swedish', 'turkish' +} +IGNORE_WORDS_REGEX = True +REQUIRE_WORDS = set() +REQUIRE_WORDS_REGEX = False WANTEDLIST_CACHE = None @@ -645,6 +649,7 @@ def init_stage_1(console_logging): # Search Settings/Episode global DOWNLOAD_PROPERS, PROPERS_WEBDL_ONEGRP, WEBDL_TYPES, RECENTSEARCH_FREQUENCY, \ BACKLOG_DAYS, BACKLOG_NOFULL, BACKLOG_FREQUENCY, USENET_RETENTION, IGNORE_WORDS, REQUIRE_WORDS, \ + IGNORE_WORDS, IGNORE_WORDS_REGEX, REQUIRE_WORDS, REQUIRE_WORDS_REGEX, \ ALLOW_HIGH_PRIORITY, SEARCH_UNAIRED, UNAIRED_RECENT_SEARCH_ONLY # Search Settings/NZB search global USE_NZBS, NZB_METHOD, NZB_DIR, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ @@ -1215,8 +1220,8 @@ def init_stage_1(console_logging): GIT_PATH = check_setting_str(CFG, 'General', 'git_path', '') - IGNORE_WORDS = check_setting_str(CFG, 'General', 'ignore_words', IGNORE_WORDS) - REQUIRE_WORDS = check_setting_str(CFG, 'General', 'require_words', REQUIRE_WORDS) + IGNORE_WORDS, IGNORE_WORDS_REGEX = helpers.split_word_str(check_setting_str(CFG, 'General', 'ignore_words', IGNORE_WORDS)) + REQUIRE_WORDS, REQUIRE_WORDS_REGEX = helpers.split_word_str(check_setting_str(CFG, 'General', 'require_words', REQUIRE_WORDS)) CALENDAR_UNPROTECTED = bool(check_setting_int(CFG, 'General', 'calendar_unprotected', 0)) @@ -1812,8 +1817,8 @@ def save_config(): new_config['General']['extra_scripts'] = '|'.join(EXTRA_SCRIPTS) new_config['General']['sg_extra_scripts'] = '|'.join(SG_EXTRA_SCRIPTS) new_config['General']['git_path'] = GIT_PATH - new_config['General']['ignore_words'] = IGNORE_WORDS - new_config['General']['require_words'] = REQUIRE_WORDS + new_config['General']['ignore_words'] = helpers.generate_word_str(IGNORE_WORDS, IGNORE_WORDS_REGEX) + new_config['General']['require_words'] = helpers.generate_word_str(REQUIRE_WORDS, REQUIRE_WORDS_REGEX) new_config['General']['calendar_unprotected'] = int(CALENDAR_UNPROTECTED) default_not_zero = ('enable_recentsearch', 'enable_backlog', 'enable_scheduled_backlog', 'use_after_get_data') diff --git a/sickbeard/config.py b/sickbeard/config.py index c13bd44..8497e32 100644 --- a/sickbeard/config.py +++ b/sickbeard/config.py @@ -783,25 +783,11 @@ class ConfigMigrator(object): else: sickbeard.SHOWLIST_TAGVIEW = 'default' - @staticmethod - def _migrate_v12(): + def _migrate_v12(self): # add words to ignore list and insert spaces to improve the ui config readability words_to_add = ['hevc', 'reenc', 'x265', 'danish', 'deutsch', 'flemish', 'italian', 'nordic', 'norwegian', 'portuguese', 'spanish', 'turkish'] - config_words = sickbeard.IGNORE_WORDS.split(',') - new_list = [] - for new_word in words_to_add: - add_word = True - for ignore_word in config_words: - ignored = ignore_word.strip().lower() - if ignored and ignored not in new_list: - new_list += [ignored] - if re.search(r'(?i)%s' % new_word, ignored): - add_word = False - if add_word: - new_list += [new_word] - - sickbeard.IGNORE_WORDS = ', '.join(sorted(new_list)) + self.add_ignore_words(words_to_add) @staticmethod def _migrate_v13(): @@ -861,25 +847,24 @@ class ConfigMigrator(object): if not isinstance(removelist, list): removelist = ([removelist], [])[None is removelist] - words = sickbeard.IGNORE_WORDS.split(',') + wordlist - - new_list = [] + new_list = set() dedupe = [] - using_regex = '' - for ignore_word in words: + using_regex = False + for ignore_word in list(sickbeard.IGNORE_WORDS) + wordlist: # words: word = ignore_word.strip() if word.startswith('regex:'): word = word.lstrip('regex:').strip() - using_regex = 'regex:' + using_regex = True # 'regex:' if word: check_word = word.lower() if check_word not in dedupe and check_word not in removelist: dedupe += [check_word] if 'spanish' in check_word: word = re.sub(r'(?i)(portuguese)\|spanish(\|swedish)', r'\1\2', word) - new_list += [word] + new_list.add(word) - sickbeard.IGNORE_WORDS = '%s%s' % (using_regex, ', '.join(new_list)) + sickbeard.IGNORE_WORDS = new_list + sickbeard.IGNORE_WORDS_REGEX = using_regex def _migrate_v17(self): diff --git a/sickbeard/databases/mainDB.py b/sickbeard/databases/mainDB.py index 5379ca0..4af35cf 100644 --- a/sickbeard/databases/mainDB.py +++ b/sickbeard/databases/mainDB.py @@ -28,7 +28,7 @@ import encodingKludge as ek from six import iteritems MIN_DB_VERSION = 9 # oldest db version we support migrating from -MAX_DB_VERSION = 20011 +MAX_DB_VERSION = 20012 TEST_BASE_VERSION = None # the base production db version, only needed for TEST db versions (>=100000) @@ -1642,3 +1642,31 @@ class AddIndexerToTables(db.SchemaUpgrade): self.setDBVersion(20011) return self.checkDBVersion() + + +# 20011 -> 20012 +class AddShowExludeGlobals(db.SchemaUpgrade): + def execute(self): + + if not self.hasColumn('tv_shows', 'rls_global_exclude_ignore'): + logger.log('Adding rls_global_exclude_ignore, rls_global_exclude_require to tv_shows') + + db.backup_database('sickbeard.db', self.checkDBVersion()) + self.addColumn('tv_shows', 'rls_global_exclude_ignore', data_type='TEXT', default='') + self.addColumn('tv_shows', 'rls_global_exclude_require', data_type='TEXT', default='') + + if self.hasTable('tv_shows_exclude_backup'): + self.connection.mass_action([['UPDATE tv_shows SET rls_global_exclude_ignore = ' + '(SELECT te.rls_global_exclude_ignore FROM tv_shows_exclude_backup te WHERE ' + 'te.show_id = tv_shows.show_id AND te.indexer = tv_shows.indexer), ' + 'rls_global_exclude_require = (SELECT te.rls_global_exclude_require FROM ' + 'tv_shows_exclude_backup te WHERE te.show_id = tv_shows.show_id AND ' + 'te.indexer = tv_shows.indexer) WHERE EXISTS (SELECT 1 FROM ' + 'tv_shows_exclude_backup WHERE tv_shows.show_id = ' + 'tv_shows_exclude_backup.show_id AND ' + 'tv_shows.indexer = tv_shows_exclude_backup.indexer)'], + ['DROP TABLE tv_shows_exclude_backup'] + ]) + + self.setDBVersion(20012) + return self.checkDBVersion() diff --git a/sickbeard/db.py b/sickbeard/db.py index fb9810c..44475be 100644 --- a/sickbeard/db.py +++ b/sickbeard/db.py @@ -618,6 +618,7 @@ def MigrationCode(my_db): 20008: sickbeard.mainDB.AddWatched, 20009: sickbeard.mainDB.AddPrune, 20010: sickbeard.mainDB.AddIndexerToTables, + 20011: sickbeard.mainDB.AddShowExludeGlobals, # 20002: sickbeard.mainDB.AddCoolSickGearFeature3, } diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index e6bb544..e1ef82f 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -68,7 +68,7 @@ from sg_helpers import chmod_as_parent, clean_data, get_system_temp_dir, \ # noinspection PyUnreachableCode if False: # noinspection PyUnresolvedReferences - from typing import Any, AnyStr, Dict, NoReturn, Iterable, Iterator, List, Optional, Tuple, Union + from typing import Any, AnyStr, Dict, NoReturn, Iterable, Iterator, List, Optional, Set, Tuple, Union from .tv import TVShow # the following workaround hack resolves a pyc resolution bug from .name_cache import retrieveNameFromCache @@ -2028,3 +2028,43 @@ def parse_imdb_id(string): pass return result + + +def generate_word_str(words, regex=False, join_chr=','): + # type: (Set[AnyStr], bool, AnyStr) -> AnyStr + """ + combine a list or set to a string with optional prefix 'regex:' + + :param words: list or set of words + :type words: set + :param regex: prefix regex: ? + :type regex: bool + :param join_chr: character(s) used for join words + :type join_chr: basestring + :return: combined string + :rtype: basestring + """ + return '%s%s' % (('', 'regex:')[True is regex], join_chr.join(words)) + + +def split_word_str(word_list): + # type: (AnyStr) -> Tuple[Set[AnyStr], bool] + """ + split string into set and boolean regex + + :param word_list: string with words + :type word_list: basestring + :return: set of words, is it regex + :rtype: (set, bool) + """ + try: + if word_list.startswith('regex:'): + rx = True + word_list = word_list.replace('regex:', '') + else: + rx = False + s = set(w.strip() for w in word_list.split(',') if w.strip()) + except (BaseException, Exception): + rx = False + s = set() + return s, rx diff --git a/sickbeard/properFinder.py b/sickbeard/properFinder.py index 16e0db3..122b45b 100644 --- a/sickbeard/properFinder.py +++ b/sickbeard/properFinder.py @@ -292,13 +292,14 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_dict= logger.DEBUG) continue - if not show_name_helpers.pass_wordlist_checks(cur_proper.name, parse=False, indexer_lookup=False): + if not show_name_helpers.pass_wordlist_checks(cur_proper.name, parse=False, indexer_lookup=False, + show_obj=cur_proper.parsed_show_obj): logger.log('Ignored unwanted Proper [%s]' % cur_proper.name, logger.DEBUG) continue re_x = dict(re_prefix='.*', re_suffix='.*') result = show_name_helpers.contains_any(cur_proper.name, cur_proper.parsed_show_obj.rls_ignore_words, - **re_x) + rx=cur_proper.parsed_show_obj.rls_ignore_words_regex, **re_x) if None is not result and result: logger.log('Ignored Proper containing ignore word [%s]' % cur_proper.name, logger.DEBUG) continue diff --git a/sickbeard/search.py b/sickbeard/search.py index f4957c3..bd58eca 100644 --- a/sickbeard/search.py +++ b/sickbeard/search.py @@ -216,12 +216,14 @@ def pass_show_wordlist_checks(name, show_obj): :return: passed check """ re_extras = dict(re_prefix='.*', re_suffix='.*') - result = show_name_helpers.contains_any(name, show_obj.rls_ignore_words, **re_extras) + result = show_name_helpers.contains_any(name, show_obj.rls_ignore_words, rx=show_obj.rls_ignore_words_regex, + **re_extras) if None is not result and result: logger.log(u'Ignored: %s for containing ignore word' % name) return False - result = show_name_helpers.contains_any(name, show_obj.rls_require_words, **re_extras) + result = show_name_helpers.contains_any(name, show_obj.rls_require_words, rx=show_obj.rls_require_words_regex, + **re_extras) if None is not result and not result: logger.log(u'Ignored: %s for not containing any required word match' % name) return False @@ -797,8 +799,8 @@ def search_providers( for cur_search_result in search_result_list: # skip non-tv crap search_result_list[cur_search_result] = filter_list( - lambda ep_item: show_name_helpers.pass_wordlist_checks( - ep_item.name, parse=False, indexer_lookup=False) and ep_item.show_obj == show_obj, + lambda ep_item: ep_item.show_obj == show_obj and show_name_helpers.pass_wordlist_checks( + ep_item.name, parse=False, indexer_lookup=False, show_obj=ep_item.show_obj), search_result_list[cur_search_result]) if cur_search_result in found_results: @@ -881,8 +883,8 @@ def search_providers( individual_results = nzbSplitter.splitResult(best_season_result) for cur_result in filter_iter( - lambda r: show_name_helpers.pass_wordlist_checks( - r.name, parse=False, indexer_lookup=False) and r.show_obj == show_obj, individual_results): + lambda r: r.show_obj == show_obj and show_name_helpers.pass_wordlist_checks( + r.name, parse=False, indexer_lookup=False, show_obj=r.show_obj), individual_results): ep_num = None if 1 == len(cur_result.ep_obj_list): ep_num = cur_result.ep_obj_list[0].episode @@ -1032,7 +1034,8 @@ def search_providers( if name: if not pass_show_wordlist_checks(name, show_obj): continue - if not show_name_helpers.pass_wordlist_checks(name, indexer_lookup=False): + if not show_name_helpers.pass_wordlist_checks(name, indexer_lookup=False, + show_obj=show_obj): logger.log('Ignored: %s (debug log has detail)' % name) continue best_result.name = name diff --git a/sickbeard/show_name_helpers.py b/sickbeard/show_name_helpers.py index 487018d..dea217e 100644 --- a/sickbeard/show_name_helpers.py +++ b/sickbeard/show_name_helpers.py @@ -18,6 +18,7 @@ import datetime import fnmatch import os +import copy import re # noinspection PyPep8Naming @@ -35,21 +36,32 @@ from six import iterkeys, itervalues # noinspection PyUnreachableCode if False: - from typing import AnyStr, List, Union + from typing import AnyStr, List, Optional, Set, Union + from .tv import TVShow + # noinspection PyUnresolvedReferences + from re import Pattern -def pass_wordlist_checks(name, # type: AnyStr +def pass_wordlist_checks(name, # type: AnyStr parse=True, # type: bool - indexer_lookup=True # type: bool + indexer_lookup=True, # type: bool + show_obj=None # type: TVShow ): # type: (...) -> bool """ Filters out non-english and just all-around stupid releases by comparing the word list contents at boundaries or the end of name. :param name: the release name to check - :param parse: try to parse release name - :param indexer_lookup: try to look up on tvinfo source + :type name: basestring + :param parse: parse release name + :type parse: bool + :param indexer_lookup: use indexer lookup during paring + :type indexer_lookup: bool + :param show_obj: TVShow object + :type show_obj: TVShow + :return: True if the release name is OK, False if it's bad. + :rtype: bool """ if parse: @@ -63,21 +75,43 @@ def pass_wordlist_checks(name, # type: AnyStr logger.log(err_msg + 'show', logger.DEBUG) return False - word_list = ['sub(bed|ed|pack|s)', '(dk|fin|heb|kor|nor|nordic|pl|swe)sub(bed|ed|s)?', + word_list = {'sub(bed|ed|pack|s)', '(dk|fin|heb|kor|nor|nordic|pl|swe)sub(bed|ed|s)?', '(dir|sample|sub|nfo)fix', 'sample', '(dvd)?extras', - 'dub(bed)?'] + 'dub(bed)?'} # if any of the bad strings are in the name then say no if sickbeard.IGNORE_WORDS: - word_list = ','.join([sickbeard.IGNORE_WORDS] + word_list) + word_list.update(sickbeard.IGNORE_WORDS) + + req_word_list = copy.copy(sickbeard.REQUIRE_WORDS) - result = contains_any(name, word_list) + result = None + if show_obj: + if show_obj.rls_ignore_words and isinstance(show_obj.rls_ignore_words, set): + if sickbeard.IGNORE_WORDS_REGEX == show_obj.rls_ignore_words_regex: + word_list.update(show_obj.rls_ignore_words) + else: + result = contains_any(name, show_obj.rls_ignore_words, rx=show_obj.rls_ignore_words_regex) + if show_obj.rls_global_exclude_ignore and isinstance(show_obj.rls_global_exclude_ignore, set): + word_list = word_list - show_obj.rls_global_exclude_ignore + + result = result or contains_any(name, word_list, rx=sickbeard.IGNORE_WORDS_REGEX) if None is not result and result: logger.log(u'Ignored: %s for containing ignore word' % name, logger.DEBUG) return False + result = None + if show_obj: + if show_obj.rls_require_words and isinstance(show_obj.rls_require_words, set): + if sickbeard.REQUIRE_WORDS_REGEX == show_obj.rls_require_words_regex: + req_word_list.update(show_obj.rls_require_words) + else: + result = not_contains_any(name, show_obj.rls_require_words, rx=show_obj.rls_require_words_regex) + if show_obj.rls_global_exclude_require and isinstance(show_obj.rls_global_exclude_require, set): + req_word_list = req_word_list - show_obj.rls_global_exclude_require + # if any of the good strings aren't in the name then say no - result = not_contains_any(name, sickbeard.REQUIRE_WORDS) + result = result or not_contains_any(name, req_word_list, rx=sickbeard.REQUIRE_WORDS_REGEX) if None is not result and result: logger.log(u'Ignored: %s for not containing required word match' % name, logger.DEBUG) return False @@ -86,33 +120,45 @@ def pass_wordlist_checks(name, # type: AnyStr def not_contains_any(subject, # type: AnyStr - lookup_words, # type: Union[AnyStr, List[AnyStr]] + lookup_words, # type: Union[AnyStr, Set[AnyStr]] + rx=None, **kwargs ): # type: (...) -> bool - return contains_any(subject, lookup_words, invert=True, **kwargs) + return contains_any(subject, lookup_words, invert=True, rx=rx, **kwargs) def contains_any(subject, # type: AnyStr - lookup_words, # type: Union[AnyStr, List[AnyStr]] + lookup_words, # type: Union[AnyStr, Set[AnyStr]] invert=False, # type: bool + rx=None, **kwargs - ): # type: (...) -> Union[bool, None] + ): # type: (...) -> Optional[bool] """ Check if subject does or does not contain a match from a list or string of regular expression lookup words - word: word to test existence of - re_prefix: insert string to all lookup words - re_suffix: append string to all lookup words + :param subject: word to test existence of + :type subject: basestring + :param lookup_words: List or comma separated string of words to search + :type lookup_words: Union(list, set, basestring) + :param re_prefix: insert string to all lookup words + :type re_prefix: basestring + :param re_suffix: append string to all lookup words + :type re_suffix: basestring + :param invert: invert function logic "contains any" into "does not contain any" + :type invert: bool + :param rx: lookup_words are regex + :type rx: Union(NoneType, bool) - :param subject: + :return: None if no checking was done. True for first match found, or if invert is False, :param lookup_words: List or comma separated string of words to search :param invert: invert function logic "contains any" into "does not contain any" :param kwargs: :return: None if no checking was done. True for first match found, or if invert is False, then True for first pattern that does not match, or False + :rtype: Union(NoneType, bool) """ - compiled_words = compile_word_list(lookup_words, **kwargs) + compiled_words = compile_word_list(lookup_words, rx=rx, **kwargs) if subject and compiled_words: for rc_filter in compiled_words: match = rc_filter.search(subject) @@ -125,19 +171,25 @@ def contains_any(subject, # type: AnyStr return None -def compile_word_list(lookup_words, # type: AnyStr +def compile_word_list(lookup_words, # type: Union[AnyStr, Set[AnyStr]] re_prefix=r'(^|[\W_])', # type: AnyStr - re_suffix=r'($|[\W_])' # type: AnyStr - ): # type: (...) -> List[AnyStr] + re_suffix=r'($|[\W_])', # type: AnyStr + rx=None + ): # type: (...) -> List[Pattern[AnyStr]] result = [] if lookup_words: - search_raw = isinstance(lookup_words, list) - if not search_raw: - search_raw = not lookup_words.startswith('regex:') - lookup_words = lookup_words[(6, 0)[search_raw]:].split(',') - lookup_words = [x.strip() for x in lookup_words] - for word in [x for x in lookup_words if x]: + if None is rx: + search_raw = isinstance(lookup_words, list) + if not search_raw: + # noinspection PyUnresolvedReferences + search_raw = not lookup_words.startswith('regex:') + # noinspection PyUnresolvedReferences + lookup_words = lookup_words[(6, 0)[search_raw]:].split(',') + lookup_words = [x.strip() for x in lookup_words if x.strip()] + else: + search_raw = not rx + for word in lookup_words: try: # !0 == regex and subject = s / 'what\'s the "time"' / what\'s\ the\ \"time\" subject = search_raw and re.escape(word) or re.sub(r'([\" \'])', r'\\\1', word) diff --git a/sickbeard/show_updater.py b/sickbeard/show_updater.py index ed244f5..c82b587 100644 --- a/sickbeard/show_updater.py +++ b/sickbeard/show_updater.py @@ -25,6 +25,35 @@ from exceptions_helper import ex import sickbeard from . import db, failed_history, logger, network_timezones, properFinder, ui +# noinspection PyUnreachableCode +if False: + from sickbeard.tv import TVShow + + +def clean_ignore_require_words(): + """ + removes duplicate ignore/require words from shows and global lists + """ + try: + for s in sickbeard.showList: # type: TVShow + # test before set to prevent dirty setter from setting unchanged shows to dirty + if s.rls_ignore_words - sickbeard.IGNORE_WORDS != s.rls_ignore_words: + s.rls_ignore_words -= sickbeard.IGNORE_WORDS + if 0 == len(s.rls_ignore_words): + s.rls_ignore_words_regex = False + if s.rls_require_words - sickbeard.REQUIRE_WORDS != s.rls_require_words: + s.rls_require_words -= sickbeard.REQUIRE_WORDS + if 0 == len(s.rls_require_words): + s.rls_require_words_regex = False + if s.rls_global_exclude_ignore & sickbeard.IGNORE_WORDS != s.rls_global_exclude_ignore: + s.rls_global_exclude_ignore &= sickbeard.IGNORE_WORDS + if s.rls_global_exclude_require & sickbeard.REQUIRE_WORDS != s.rls_global_exclude_require: + s.rls_global_exclude_require &= sickbeard.REQUIRE_WORDS + if s.dirty: + s.save_to_db() + except (BaseException, Exception): + pass + class ShowUpdater(object): def __init__(self): @@ -87,6 +116,13 @@ class ShowUpdater(object): logger.log('image cache cleanup error', logger.ERROR) logger.log(traceback.format_exc(), logger.ERROR) + # cleanup ignore and require lists + try: + clean_ignore_require_words() + except Exception: + logger.log('ignore, require words cleanup error', logger.ERROR) + logger.log(traceback.format_exc(), logger.ERROR) + # cleanup manual search history sickbeard.search_queue.remove_old_fifo(sickbeard.search_queue.MANUAL_SEARCH_HISTORY) diff --git a/sickbeard/tv.py b/sickbeard/tv.py index 90f09ca..8ddfe55 100644 --- a/sickbeard/tv.py +++ b/sickbeard/tv.py @@ -1206,8 +1206,13 @@ class TVShow(TVShowBase): self._last_update_indexer = sql_result[0]['last_update_indexer'] - self._rls_ignore_words = sql_result[0]['rls_ignore_words'] - self._rls_require_words = sql_result[0]['rls_require_words'] + self._rls_ignore_words, self._rls_ignore_words_regex = helpers.split_word_str(sql_result[0]['rls_ignore_words']) + + self._rls_require_words, self._rls_require_words_regex = helpers.split_word_str(sql_result[0]['rls_require_words']) + + self._rls_global_exclude_ignore = helpers.split_word_str(sql_result[0]['rls_global_exclude_ignore'])[0] + + self._rls_global_exclude_require = helpers.split_word_str(sql_result[0]['rls_global_exclude_require'])[0] if not self._imdbid: imdbid = sql_result[0]['imdb_id'] or '' @@ -1806,8 +1811,10 @@ class TVShow(TVShowBase): startyear=self.startyear, lang=self.lang, imdb_id=self.imdbid, last_update_indexer=self.last_update_indexer, - rls_ignore_words=self.rls_ignore_words, - rls_require_words=self.rls_require_words, + rls_ignore_words=helpers.generate_word_str(self.rls_ignore_words, self.rls_ignore_words_regex), + rls_require_words=helpers.generate_word_str(self.rls_require_words, self.rls_require_words_regex), + rls_global_exclude_ignore=','.join(self.rls_global_exclude_ignore), + rls_global_exclude_require=','.join(self.rls_global_exclude_require), overview=self.overview, prune=self.prune, tag=self.tag) diff --git a/sickbeard/tv_base.py b/sickbeard/tv_base.py index 6fb4516..9257a18 100644 --- a/sickbeard/tv_base.py +++ b/sickbeard/tv_base.py @@ -80,14 +80,50 @@ class TVShowBase(LegacyTVShow, TVBase): self._sports = 0 self._anime = 0 self._scene = 0 - self._rls_ignore_words = '' - self._rls_require_words = '' + self._rls_ignore_words = set() + self._rls_require_words = set() self._overview = '' self._prune = 0 self._tag = '' + self._rls_ignore_words_regex = False + self._rls_require_words_regex = False + self._rls_global_exclude_ignore = set() + self._rls_global_exclude_require = set() # name = property(lambda self: self._name, dirty_setter('_name')) @property + def rls_ignore_words_regex(self): + return self._rls_ignore_words_regex + + @rls_ignore_words_regex.setter + def rls_ignore_words_regex(self, val): + self.dirty_setter('_rls_ignore_words_regex')(self, val) + + @property + def rls_require_words_regex(self): + return self._rls_require_words_regex + + @rls_require_words_regex.setter + def rls_require_words_regex(self, val): + self.dirty_setter('_rls_require_words_regex')(self, val) + + @property + def rls_global_exclude_ignore(self): + return self._rls_global_exclude_ignore + + @rls_global_exclude_ignore.setter + def rls_global_exclude_ignore(self, val): + self.dirty_setter('_rls_global_exclude_ignore', set)(self, val) + + @property + def rls_global_exclude_require(self): + return self._rls_global_exclude_require + + @rls_global_exclude_require.setter + def rls_global_exclude_require(self, val): + self.dirty_setter('_rls_global_exclude_require', set)(self, val) + + @property def name(self): return self._name @@ -285,7 +321,7 @@ class TVShowBase(LegacyTVShow, TVBase): @rls_ignore_words.setter def rls_ignore_words(self, *arg): - self.dirty_setter('_rls_ignore_words')(self, *arg) + self.dirty_setter('_rls_ignore_words', set)(self, *arg) # rls_require_words = property(lambda self: self._rls_require_words, dirty_setter('_rls_require_words')) @property @@ -294,7 +330,7 @@ class TVShowBase(LegacyTVShow, TVBase): @rls_require_words.setter def rls_require_words(self, *arg): - self.dirty_setter('_rls_require_words')(self, *arg) + self.dirty_setter('_rls_require_words', set)(self, *arg) # overview = property(lambda self: self._overview, dirty_setter('_overview')) @property diff --git a/sickbeard/tvcache.py b/sickbeard/tvcache.py index 7a39421..e9ffda9 100644 --- a/sickbeard/tvcache.py +++ b/sickbeard/tvcache.py @@ -407,15 +407,16 @@ class TVCache(object): # for each cache entry for cur_result in sql_result: - # skip non-tv crap - if not show_name_helpers.pass_wordlist_checks(cur_result['name'], parse=False, indexer_lookup=False): - continue - # get the show object, or if it's not one of our shows then ignore it show_obj = helpers.find_show_by_id({int(cur_result['indexer']): int(cur_result['indexerid'])}) if not show_obj: continue + # skip non-tv crap + if not show_name_helpers.pass_wordlist_checks(cur_result['name'], parse=False, indexer_lookup=False, + show_obj=show_obj): + continue + # skip if provider is anime only and show is not anime if self.provider.anime_only and not show_obj.is_anime: logger.log(u'' + str(show_obj.name) + ' is not an anime, skipping', logger.DEBUG) diff --git a/sickbeard/webapi.py b/sickbeard/webapi.py index 6fa5bea..4607f18 100644 --- a/sickbeard/webapi.py +++ b/sickbeard/webapi.py @@ -25,6 +25,7 @@ from random import randint import datetime import glob +import copy try: import json except ImportError: @@ -52,6 +53,7 @@ from .indexers.indexer_config import * from tvinfo_base.exceptions import * from .scene_numbering import set_scene_numbering_helper from .search_backlog import FORCED_BACKLOG +from .show_updater import clean_ignore_require_words from .sgdatetime import SGDatetime from .tv import TVEpisode, TVShow, TVidProdid from .webserve import AddShows @@ -2809,21 +2811,22 @@ class CMD_SickGearListIgnoreWords(ApiCall): if self.tvid and self.prodid: my_db = db.DBConnection() sql_result = my_db.select( - 'SELECT show_name, rls_ignore_words' + 'SELECT show_name, rls_ignore_words, rls_global_exclude_ignore' ' FROM tv_shows' ' WHERE indexer = ? AND indexer_id = ?', [self.tvid, self.prodid]) if sql_result: ignore_words = sql_result[0]['rls_ignore_words'] return_data = {'type': 'show', 'indexer': self.tvid, 'indexerid': self.prodid, - 'show name': sql_result[0]['show_name']} + 'show name': sql_result[0]['show_name'], + 'global_exclude_ignore': sql_result[0]['rls_global_exclude_ignore']} return_type = '%s:' % sql_result[0]['show_name'] else: return _responds(RESULT_FAILURE, msg='Show not found.') elif (None is self.tvid) != (None is self.prodid): return _responds(RESULT_FAILURE, msg='You must supply indexer + indexerid.') else: - ignore_words = sickbeard.IGNORE_WORDS + ignore_words = helpers.generate_word_str(sickbeard.IGNORE_WORDS, sickbeard.IGNORE_WORDS_REGEX) return_data = {'type': 'global'} return_type = 'Global' @@ -2838,6 +2841,8 @@ class CMD_SickGearSetIgnoreWords(ApiCall): "indexer": {"desc": "indexer of a show"}, "add": {"desc": "add words to list"}, "remove": {"desc": "remove words from list"}, + "add_exclude": {"desc": "add global exclude words"}, + "remove_exclude": {"desc": "remove global exclude words"}, "regex": {"desc": "interpret ALL (including existing) ignore words as regex"}, } } @@ -2850,14 +2855,23 @@ class CMD_SickGearSetIgnoreWords(ApiCall): [i for i in indexer_api.TVInfoAPI().sources]) self.add, args = self.check_params(args, kwargs, "add", None, False, "list", []) self.remove, args = self.check_params(args, kwargs, "remove", None, False, "list", []) + self.add_exclude, args = self.check_params(args, kwargs, "add_exclude", None, False, "list", []) + self.remove_exclude, args = self.check_params(args, kwargs, "remove_exclude", None, False, "list", []) self.regex, args = self.check_params(args, kwargs, "regex", None, False, "bool", []) # super, missing, help ApiCall.__init__(self, handler, args, kwargs) def run(self): """ set ignore words """ - if not self.add and not self.remove: - return _responds(RESULT_FAILURE, msg="No words to add/remove provided") + if (not self.add and not self.remove and not self.add_exclude and not self.remove_exclude) or \ + ((self.add_exclude or self.remove_exclude) and not (self.tvid and self.prodid)): + return _responds(RESULT_FAILURE, msg=('No indexer, indexerid provided', + 'No words to add/remove provided')[None is not self.tvid and + None is not self.prodid]) + + use_regex = None + return_type = '' + ignore_list = set() def _create_ignore_words(): _use_regex = ignore_words.startswith('regex:') @@ -2883,35 +2897,63 @@ class CMD_SickGearSetIgnoreWords(ApiCall): if not show_obj: return _responds(RESULT_FAILURE, msg="Show not found") - my_db = db.DBConnection() - sql_result = my_db.select('SELECT show_name, rls_ignore_words' - ' FROM tv_shows' - ' WHERE indexer = ? AND indexer_id = ?', - [self.tvid, self.prodid]) - - ignore_words = '' - if sql_result: - ignore_words = sql_result[0]['rls_ignore_words'] - return_data = {'type': 'show', 'indexer': self.tvid, 'indexerid': self.prodid, - 'show name': sql_result[0]['show_name']} - return_type = '%s:' % sql_result[0]['show_name'] + 'show name': show_obj.name} - use_regex, ignore_list, new_ignore_words = _create_ignore_words() - my_db.action('UPDATE tv_shows SET rls_ignore_words = ? WHERE indexer = ? AND indexer_id = ?', - [new_ignore_words, self.tvid, self.prodid]) + my_db = db.DBConnection() + if self.add or self.remove: + sql_results = my_db.select('SELECT show_name, rls_ignore_words FROM tv_shows WHERE indexer = ? AND ' + 'indexer_id = ?', [self.tvid, self.prodid]) + + ignore_words = '' + if sql_results: + ignore_words = sql_results[0]['rls_ignore_words'] + return_type = '%s:' % sql_results[0]['show_name'] + + use_regex, ignore_list, new_ignore_words = _create_ignore_words() + my_db.action('UPDATE tv_shows SET rls_ignore_words = ? WHERE indexer = ? AND indexer_id = ?', + [new_ignore_words, self.tvid, self.prodid]) + show_obj.rls_ignore_words, show_obj.rls_ignore_words_regex = helpers.split_word_str(new_ignore_words) + + if self.add_exclude or self.remove_exclude: + sql_results = my_db.select('SELECT rls_global_exclude_ignore FROM tv_shows WHERE indexer = ? AND ' + 'indexer_id = ?', [self.tvid, self.prodid]) + + exclude_ignore = set() + if sql_results: + exclude_ignore = helpers.split_word_str(sql_results[0]['rls_global_exclude_ignore'])[0] + exclude_ignore = {i for i in exclude_ignore if i not in sickbeard.IGNORE_WORDS} + if self.add_exclude: + for a in self.add_exclude: + if a not in sickbeard.IGNORE_WORDS: + exclude_ignore.add(a) + if self.remove_exclude: + for r in self.remove_exclude: + try: + exclude_ignore.remove(r) + except KeyError: + pass + + my_db.action('UPDATE tv_shows SET rls_global_exclude_ignore = ? WHERE indexer = ? AND indexer_id = ?', + [helpers.generate_word_str(exclude_ignore), self.tvid, self.prodid]) + show_obj.rls_global_exclude_ignore = copy.copy(exclude_ignore) + return_data['global exclude ignore'] = exclude_ignore elif (None is self.tvid) != (None is self.prodid): return _responds(RESULT_FAILURE, msg='You must supply indexer + indexerid.') else: - ignore_words = sickbeard.IGNORE_WORDS + ignore_words = helpers.generate_word_str(sickbeard.IGNORE_WORDS, sickbeard.IGNORE_WORDS_REGEX) use_regex, ignore_list, new_ignore_words = _create_ignore_words() - sickbeard.IGNORE_WORDS = new_ignore_words + sickbeard.IGNORE_WORDS, sickbeard.IGNORE_WORDS_REGEX = helpers.split_word_str(new_ignore_words) sickbeard.save_config() return_data = {'type': 'global'} return_type = 'Global' - return_data['use regex'] = use_regex + if None is not use_regex: + return_data['use regex'] = use_regex + elif None is not self.regex: + return_data['use regex'] = self.regex return_data['ignore words'] = ignore_list + clean_ignore_require_words() return _responds(RESULT_SUCCESS, data=return_data, msg="%s set ignore words" % return_type) @@ -2936,21 +2978,22 @@ class CMD_SickGearListRequireWords(ApiCall): if self.tvid and self.prodid: my_db = db.DBConnection() sql_result = my_db.select( - 'SELECT show_name, rls_require_words' + 'SELECT show_name, rls_require_words, rls_global_exclude_require' ' FROM tv_shows' ' WHERE indexer = ? AND indexer_id = ?', [self.tvid, self.prodid]) if sql_result: required_words = sql_result[0]['rls_require_words'] return_data = {'type': 'show', 'indexer': self.tvid, 'indexerid': self.prodid, - 'show name': sql_result[0]['show_name']} + 'show name': sql_result[0]['show_name'], + 'global_exclude_require': sql_result[0]['rls_global_exclude_require']} return_type = '%s:' % sql_result[0]['show_name'] else: return _responds(RESULT_FAILURE, msg='Show not found.') elif (None is self.tvid) != (None is self.prodid): return _responds(RESULT_FAILURE, msg='You must supply indexer + indexerid.') else: - required_words = sickbeard.REQUIRE_WORDS + required_words = helpers.generate_word_str(sickbeard.REQUIRE_WORDS, sickbeard.REQUIRE_WORDS_REGEX) return_data = {'type': 'global'} return_type = 'Global' @@ -2966,6 +3009,8 @@ class CMD_SickGearSetRequrieWords(ApiCall): "indexer": {"desc": "indexer of a show"}, "add": {"desc": "add words to list"}, "remove": {"desc": "remove words from list"}, + "add_exclude": {"desc": "add global exclude words"}, + "remove_exclude": {"desc": "remove global exclude words"}, "regex": {"desc": "interpret ALL (including existing) ignore words as regex"}, } } @@ -2978,14 +3023,23 @@ class CMD_SickGearSetRequrieWords(ApiCall): [i for i in indexer_api.TVInfoAPI().sources]) self.add, args = self.check_params(args, kwargs, "add", None, False, "list", []) self.remove, args = self.check_params(args, kwargs, "remove", None, False, "list", []) + self.add_exclude, args = self.check_params(args, kwargs, "add_exclude", None, False, "list", []) + self.remove_exclude, args = self.check_params(args, kwargs, "remove_exclude", None, False, "list", []) self.regex, args = self.check_params(args, kwargs, "regex", None, False, "bool", []) # super, missing, help ApiCall.__init__(self, handler, args, kwargs) def run(self): """ set require words """ - if not self.add and not self.remove: - return _responds(RESULT_FAILURE, msg="No words to add/remove provided") + if (not self.add and not self.remove and not self.add_exclude and not self.remove_exclude) or \ + ((self.add_exclude or self.remove_exclude) and not (self.tvid and self.prodid)): + return _responds(RESULT_FAILURE, msg=('No indexer, indexerid provided', + 'No words to add/remove provided')[None is not self.tvid and + None is not self.prodid]) + + use_regex = None + return_type = '' + required_list = set() def _create_required_words(): _use_regex = requried_words.startswith('regex:') @@ -3011,36 +3065,65 @@ class CMD_SickGearSetRequrieWords(ApiCall): if not show_obj: return _responds(RESULT_FAILURE, msg="Show not found") - my_db = db.DBConnection() - sql_result = my_db.select( - 'SELECT show_name, rls_require_words' - ' FROM tv_shows' - ' WHERE indexer = ? AND indexer_id = ?', - [self.tvid, self.prodid]) - - requried_words = '' - if sql_result: - requried_words = sql_result[0]['rls_require_words'] - return_data = {'type': 'show', 'indexer': self.tvid, 'indexerid': self.prodid, - 'show name': sql_result[0]['show_name']} - return_type = '%s:' % sql_result[0]['show_name'] + 'show name': show_obj.name} - use_regex, required_list, new_required_words = _create_required_words() - my_db.action('UPDATE tv_shows SET rls_require_words = ? WHERE indexer = ? AND indexer_id = ?', - [new_required_words, self.tvid, self.prodid]) + my_db = db.DBConnection() + if self.add or self.remove: + sql_result = my_db.select('SELECT show_name, rls_require_words FROM tv_shows WHERE indexer = ? AND ' + 'indexer_id = ?', [self.tvid, self.prodid]) + + requried_words = '' + if sql_result: + requried_words = sql_result[0]['rls_require_words'] + return_type = '%s:' % sql_result[0]['show_name'] + + use_regex, required_list, new_required_words = _create_required_words() + my_db.action('UPDATE tv_shows SET rls_require_words = ? WHERE indexer = ? AND indexer_id = ?', + [new_required_words, self.tvid, self.prodid]) + + show_obj.rls_require_words, show_obj.rls_require_words_regex = helpers.split_word_str( + new_required_words) + + if self.add_exclude or self.remove_exclude: + sql_result = my_db.select('SELECT rls_global_exclude_require FROM tv_shows WHERE indexer = ? AND ' + 'indexer_id = ?', [self.tvid, self.prodid]) + + exclude_require = set() + if sql_result: + exclude_require = helpers.split_word_str(sql_result[0]['rls_global_exclude_require'])[0] + exclude_require = {r for r in exclude_require if r not in sickbeard.REQUIRE_WORDS} + if self.add_exclude: + for a in self.add_exclude: + if a not in sickbeard.REQUIRE_WORDS: + exclude_require.add(a) + if self.remove_exclude: + for r in self.remove_exclude: + try: + exclude_require.remove(r) + except KeyError: + pass + my_db.action( + 'UPDATE tv_shows SET rls_global_exclude_require = ? WHERE indexer = ? AND indexer_id = ?', + [helpers.generate_word_str(exclude_require), self.tvid, self.prodid]) + show_obj.rls_global_exclude_require = copy.copy(exclude_require) + return_data['global exclude require'] = exclude_require elif (None is self.tvid) != (None is self.prodid): return _responds(RESULT_FAILURE, msg='You must supply indexer + indexerid.') else: - requried_words = sickbeard.REQUIRE_WORDS + requried_words = helpers.generate_word_str(sickbeard.REQUIRE_WORDS, sickbeard.REQUIRE_WORDS_REGEX) use_regex, required_list, new_required_words = _create_required_words() - sickbeard.REQUIRE_WORDS = new_required_words + sickbeard.REQUIRE_WORDS, sickbeard.REQUIRE_WORDS_REGEX = helpers.split_word_str(new_required_words) sickbeard.save_config() return_data = {'type': 'global'} return_type = 'Global' - return_data['use regex'] = use_regex + if None is not use_regex: + return_data['use regex'] = use_regex + elif None is not self.regex: + return_data['use regex'] = self.regex return_data['required words'] = required_list + clean_ignore_require_words() return _responds(RESULT_SUCCESS, data=return_data, msg="%s set requried words" % return_type) @@ -3138,8 +3221,10 @@ class CMD_SickGearShow(ApiCall): showDict["status"] = show_obj.status showDict["scenenumbering"] = show_obj.is_scene showDict["upgrade_once"] = show_obj.upgrade_once - showDict["ignorewords"] = show_obj.rls_ignore_words - showDict["requirewords"] = show_obj.rls_require_words + showDict["ignorewords"] = helpers.generate_word_str(show_obj.rls_ignore_words, show_obj.rls_ignore_words_regex) + showDict["global_exclude_ignore"] = helpers.generate_word_str(show_obj.rls_global_exclude_ignore) + showDict["requirewords"] = helpers.generate_word_str(show_obj.rls_require_words, show_obj.rls_require_words_regex) + showDict["global_exclude_require"] = helpers.generate_word_str(show_obj.rls_global_exclude_require) if self.overview: showDict["overview"] = show_obj.overview showDict["prune"] = show_obj.prune @@ -4397,8 +4482,10 @@ class CMD_SickGearShows(ApiCall): "subtitles": cur_show_obj.subtitles, "scenenumbering": cur_show_obj.is_scene, "upgrade_once": cur_show_obj.upgrade_once, - "ignorewords": cur_show_obj.rls_ignore_words, - "requirewords": cur_show_obj.rls_require_words, + "ignorewords": helpers.generate_word_str(cur_show_obj.rls_ignore_words, cur_show_obj.rls_ignore_words_regex), + "global_exclude_ignore": helpers.generate_word_str(cur_show_obj.rls_global_exclude_ignore), + "requirewords": helpers.generate_word_str(cur_show_obj.rls_require_words, cur_show_obj.rls_require_words_regex), + "global_exclude_require": helpers.generate_word_str(cur_show_obj.rls_global_exclude_require), "prune": cur_show_obj.prune, "tag": cur_show_obj.tag, "imdb_id": cur_show_obj.imdbid, diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index 38f2cf5..089b346 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -59,6 +59,7 @@ from .scene_numbering import get_scene_absolute_numbering_for_show, get_scene_nu get_xem_absolute_numbering_for_show, get_xem_numbering_for_show, set_scene_numbering_helper from .search_backlog import FORCED_BACKLOG from .sgdatetime import SGDatetime +from .show_updater import clean_ignore_require_words from .trakt_helpers import build_config, trakt_collection_remove_account from .tv import TVidProdid @@ -2444,7 +2445,8 @@ class Home(MainHandler): flatten_folders=None, paused=None, direct_call=False, air_by_date=None, sports=None, dvdorder=None, tvinfo_lang=None, subs=None, upgrade_once=None, rls_ignore_words=None, rls_require_words=None, anime=None, blacklist=None, whitelist=None, - scene=None, prune=None, tag=None, quality_preset=None, reset_fanart=None, **kwargs): + scene=None, prune=None, tag=None, quality_preset=None, reset_fanart=None, + rls_global_exclude_ignore=None, rls_global_exclude_require=None, **kwargs): any_qualities = any_qualities if None is not any_qualities else [] best_qualities = best_qualities if None is not best_qualities else [] @@ -2612,8 +2614,29 @@ class Home(MainHandler): if not direct_call: show_obj.lang = infosrc_lang show_obj.dvdorder = dvdorder - show_obj.rls_ignore_words = rls_ignore_words.strip() - show_obj.rls_require_words = rls_require_words.strip() + new_ignore_words, new_i_regex = helpers.split_word_str(rls_ignore_words.strip()) + new_ignore_words -= sickbeard.IGNORE_WORDS + if 0 == len(new_ignore_words): + new_i_regex = False + show_obj.rls_ignore_words, show_obj.rls_ignore_words_regex = new_ignore_words, new_i_regex + new_require_words, new_r_regex = helpers.split_word_str(rls_require_words.strip()) + new_require_words -= sickbeard.REQUIRE_WORDS + if 0 == len(new_require_words): + new_r_regex = False + show_obj.rls_require_words, show_obj.rls_require_words_regex = new_require_words, new_r_regex + if isinstance(rls_global_exclude_ignore, list): + show_obj.rls_global_exclude_ignore = set(r for r in rls_global_exclude_ignore if '.*' != r) + elif isinstance(rls_global_exclude_ignore, string_types) and '.*' != rls_global_exclude_ignore: + show_obj.rls_global_exclude_ignore = {rls_global_exclude_ignore} + else: + show_obj.rls_global_exclude_ignore = set() + if isinstance(rls_global_exclude_require, list): + show_obj.rls_global_exclude_require = set(r for r in rls_global_exclude_require if '.*' != r) + elif isinstance(rls_global_exclude_require, string_types) and '.*' != rls_global_exclude_require: + show_obj.rls_global_exclude_require = {rls_global_exclude_require} + else: + show_obj.rls_global_exclude_require = set() + clean_ignore_require_words() # if we change location clear the db of episodes, change it, write to db, and rescan # noinspection PyProtectedMember @@ -6494,11 +6517,17 @@ class ConfigSearch(Config): t = PageTemplate(web_handler=self, file='config_search.tmpl') t.submenu = self.config_menu('Search') t.using_rls_ignore_words = [(cur_so.tvid_prodid, cur_so.name) for cur_so in sickbeard.showList - if cur_so.rls_ignore_words and cur_so.rls_ignore_words.strip()] + if cur_so.rls_ignore_words and cur_so.rls_ignore_words] t.using_rls_ignore_words.sort(key=lambda x: x[1], reverse=False) t.using_rls_require_words = [(cur_so.tvid_prodid, cur_so.name) for cur_so in sickbeard.showList - if cur_so.rls_require_words and cur_so.rls_require_words.strip()] + if cur_so.rls_require_words and cur_so.rls_require_words] t.using_rls_require_words.sort(key=lambda x: x[1], reverse=False) + t.using_exclude_ignore_words = [(cur_so.tvid_prodid, cur_so.name) + for cur_so in sickbeard.showList if cur_so.rls_global_exclude_ignore] + t.using_exclude_ignore_words.sort(key=lambda x: x[1], reverse=False) + t.using_exclude_require_words = [(cur_so.tvid_prodid, cur_so.name) + for cur_so in sickbeard.showList if cur_so.rls_global_exclude_require] + t.using_exclude_require_words.sort(key=lambda x: x[1], reverse=False) t.using_regex = False try: from sickbeard.name_parser.parser import regex @@ -6550,8 +6579,10 @@ class ConfigSearch(Config): sickbeard.TORRENT_METHOD = torrent_method sickbeard.USENET_RETENTION = config.to_int(usenet_retention, default=500) - sickbeard.IGNORE_WORDS = ignore_words if ignore_words else '' - sickbeard.REQUIRE_WORDS = require_words if require_words else '' + sickbeard.IGNORE_WORDS, sickbeard.IGNORE_WORDS_REGEX = helpers.split_word_str(ignore_words if ignore_words else '') + sickbeard.REQUIRE_WORDS, sickbeard.REQUIRE_WORDS_REGEX = helpers.split_word_str(require_words if require_words else '') + + clean_ignore_require_words() config.schedule_download_propers(config.checkbox_to_value(download_propers)) sickbeard.PROPERS_WEBDL_ONEGRP = config.checkbox_to_value(propers_webdl_onegrp) diff --git a/sickgear.py b/sickgear.py index c6c230d..389ed6a 100755 --- a/sickgear.py +++ b/sickgear.py @@ -503,6 +503,11 @@ class SickGear(object): # Build from the DB to start with sickbeard.classes.loading_msg.message = 'Loading shows from db' self.load_shows_from_db() + if not db.DBConnection().has_flag('ignore_require_cleaned'): + from sickbeard.show_updater import clean_ignore_require_words + sickbeard.classes.loading_msg.message = 'Cleaning ignore/require words lists' + clean_ignore_require_words() + db.DBConnection().set_flag('ignore_require_cleaned') # Fire up all our threads sickbeard.classes.loading_msg.message = 'Starting threads' diff --git a/tests/all_tests.py b/tests/all_tests.py index 1e6c47d..8e19db3 100644 --- a/tests/all_tests.py +++ b/tests/all_tests.py @@ -26,7 +26,9 @@ if '__main__' == __name__: import glob import sys import unittest + import os + sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib'))) test_file_strings = [x for x in glob.glob('*_tests.py') if x not in __file__] module_strings = [file_string[0:len(file_string) - 3] for file_string in test_file_strings] suites = [unittest.defaultTestLoader.loadTestsFromName(file_string) for file_string in module_strings] diff --git a/tests/ignore_and_require_words_tests.py b/tests/ignore_and_require_words_tests.py index 5b50f15..9e04fe9 100644 --- a/tests/ignore_and_require_words_tests.py +++ b/tests/ignore_and_require_words_tests.py @@ -3,56 +3,72 @@ import sys import unittest import sickbeard -from sickbeard import show_name_helpers +from sickbeard import show_name_helpers, helpers sys.path.insert(1, os.path.abspath('..')) +class TVShow(object): + def __init__(self, ei=set(), er=set(), i=set(), r=set(), ir=False, rr=False): + self.rls_global_exclude_ignore = ei + self.rls_global_exclude_require = er + self.rls_ignore_words = i + self.rls_ignore_words_regex = ir + self.rls_require_words = r + self.rls_require_words_regex = rr + + class TestCase(unittest.TestCase): cases_pass_wordlist_checks = [ - ('[GroupName].Show.Name.-.%02d.[null]', '', '', True), - - ('[GroupName].Show.Name.-.%02d.[ignore]', '', 'required', False), - ('[GroupName].Show.Name.-.%02d.[required]', '', 'required', True), - ('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', 'GroupName', True), - ('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', '[GroupName]', True), - ('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', 'Show.Name', True), - ('[GroupName].Show.Name.-.%02d.[required]', 'not_ignored', 'required', True), - ('[GroupName].Show.Name.-.%02d.[required]', '[not_ignored]', '[required]', True), - - ('[GroupName].Show.Name.-.%02d.[ignore]', '[ignore]', '', False), - ('[GroupName].Show.Name.-.%02d.[required]', '[GroupName]', 'required', False), - ('[GroupName].Show.Name.-.%02d.[required]', 'GroupName', 'required', False), - ('[GroupName].Show.Name.-.%02d.[ignore]', 'ignore', 'GroupName', False), - ('[GroupName].Show.Name.-.%02d.[required]', 'Show.Name', 'required', False), - - ('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: no_ignore', '', True), - ('[GroupName].Show.Name.-.%02d.[480p]', 'ignore', r'regex: \d?\d80p', True), - ('[GroupName].Show.Name.-.%02d.[480p]', 'ignore', r'regex: \[\d?\d80p\]', True), - ('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: ignore', '', False), - ('[GroupName].Show.Name.-.%02d.[ignore]', r'regex: \[ignore\]', '', False), - ('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: ignore', 'required', False), + ('[GroupName].Show.Name.-.%02d.[null]', '', '', True, TVShow()), + + ('[GroupName].Show.Name.-.%02d.[ignore]', '', 'required', False, TVShow()), + ('[GroupName].Show.Name.-.%02d.[required]', '', 'required', True, TVShow()), + ('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', 'GroupName', True, TVShow()), + ('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', '[GroupName]', True, TVShow()), + ('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', 'Show.Name', True, TVShow()), + ('[GroupName].Show.Name.-.%02d.[required]', 'not_ignored', 'required', True, TVShow()), + ('[GroupName].Show.Name.-.%02d.[required]', '[not_ignored]', '[required]', True, TVShow()), + + ('[GroupName].Show.Name.-.%02d.[ignore]', '[ignore]', '', False, TVShow()), + ('[GroupName].Show.Name.-.%02d.[required]', '[GroupName]', 'required', False, TVShow()), + ('[GroupName].Show.Name.-.%02d.[required]', 'GroupName', 'required', False, TVShow()), + ('[GroupName].Show.Name.-.%02d.[ignore]', 'ignore', 'GroupName', False, TVShow()), + ('[GroupName].Show.Name.-.%02d.[required]', 'Show.Name', 'required', False, TVShow()), + + ('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: no_ignore', '', True, TVShow()), + ('[GroupName].Show.Name.-.%02d.[480p]', 'ignore', r'regex: \d?\d80p', True, TVShow()), + ('[GroupName].Show.Name.-.%02d.[480p]', 'ignore', r'regex: \[\d?\d80p\]', True, TVShow()), + ('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: ignore', '', False, TVShow()), + ('[GroupName].Show.Name.-.%02d.[ignore]', r'regex: \[ignore\]', '', False, TVShow()), + ('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: ignore', 'required', False, TVShow()), # The following test is True because a boundary is added to each regex not overridden with the prefix param - ('[GroupName].Show.ONEONE.-.%02d.[required]', 'regex: (one(two)?)', '', True), - ('[GroupName].Show.ONETWO.-.%02d.[required]', 'regex: ((one)?two)', 'required', False), - ('[GroupName].Show.TWO.-.%02d.[required]', 'regex: ((one)?two)', 'required', False), + ('[GroupName].Show.ONEONE.-.%02d.[required]', 'regex: (one(two)?)', '', True, TVShow()), + ('[GroupName].Show.ONETWO.-.%02d.[required]', 'regex: ((one)?two)', 'required', False, TVShow()), + ('[GroupName].Show.TWO.-.%02d.[required]', 'regex: ((one)?two)', 'required', False, TVShow()), + + ('[GroupName].Show.TWO.-.%02d.[required]', '[GroupName]', '', True, TVShow(ei={'[GroupName]'})), + ('[GroupName].Show.TWO.-.%02d.[something]', '[GroupName]', 'required', False, TVShow(er={'required'})), + + ('[GroupName].Show.TWO.-.%02d.[required]-[GroupName]', '', '', False, TVShow(i={'[GroupName]'})), + ('[GroupName].Show.TWO.-.%02d.[something]-required', '', '', True, TVShow(r={'required'})), ('The.Spanish.Princess.-.%02d', - r'regex:^(?:(?=.*?\bspanish\b)((?!spanish.?princess).)*|.*princess.*?spanish.*)$, ignore', '', True), + r'regex:^(?:(?=.*?\bspanish\b)((?!spanish.?princess).)*|.*princess.*?spanish.*)$, ignore', '', True, TVShow()), ('Spanish.Princess.Spanish.-.%02d', - r'regex:^(?:(?=.*?\bspanish\b)((?!spanish.?princess).)*|.*princess.*?spanish.*)$, ignore', '', False) + r'regex:^(?:(?=.*?\bspanish\b)((?!spanish.?princess).)*|.*princess.*?spanish.*)$, ignore', '', False, TVShow()) ] cases_contains = [ - ('[GroupName].Show.Name.-.%02d.[illegal_regex]', 'regex:??illegal_regex', False), + ('[GroupName].Show.Name.-.%02d.[illegal_regex]', 'regex:??illegal_regex', None), ('[GroupName].Show.Name.-.%02d.[480p]', 'regex:(480|1080)p', True), ('[GroupName].Show.Name.-.%02d.[contains]', r'regex:\[contains\]', True), ('[GroupName].Show.Name.-.%02d.[contains]', '[contains]', True), ('[GroupName].Show.Name.-.%02d.[contains]', 'contains', True), ('[GroupName].Show.Name.-.%02d.[contains]', '[not_contains]', False), - ('[GroupName].Show.Name.-.%02d.[null]', '', False) + ('[GroupName].Show.Name.-.%02d.[null]', '', None) ] cases_not_contains = [ @@ -61,7 +77,7 @@ class TestCase(unittest.TestCase): ('[GroupName].Show.Name.-.%02d.[contains]', '[contains]', False), ('[GroupName].Show.Name.-.%02d.[contains]', 'contains', False), ('[GroupName].Show.Name.-.%02d.[not_contains]', '[blah_blah]', True), - ('[GroupName].Show.Name.-.%02d.[null]', '', False) + ('[GroupName].Show.Name.-.%02d.[null]', '', None) ] def test_pass_wordlist_checks(self): @@ -69,11 +85,21 @@ class TestCase(unittest.TestCase): isolated = [] test_cases = (self.cases_pass_wordlist_checks, isolated)[len(isolated)] - for case_num, (name, ignore_list, require_list, expected_result) in enumerate(test_cases): + for case_num, (name, ignore_list, require_list, expected_result, show_obj) in enumerate(test_cases): name = name if '%02d' not in name else name % case_num - sickbeard.IGNORE_WORDS = ignore_list - sickbeard.REQUIRE_WORDS = require_list - self.assertEqual(expected_result, show_name_helpers.pass_wordlist_checks(name, False), + if ignore_list.startswith('regex:'): + sickbeard.IGNORE_WORDS_REGEX = True + ignore_list = ignore_list.replace('regex:', '') + else: + sickbeard.IGNORE_WORDS_REGEX = False + sickbeard.IGNORE_WORDS = set(i.strip() for i in ignore_list.split(',') if i.strip()) + if require_list.startswith('regex:'): + sickbeard.REQUIRE_WORDS_REGEX = True + require_list = require_list.replace('regex:', '') + else: + sickbeard.REQUIRE_WORDS_REGEX = False + sickbeard.REQUIRE_WORDS = set(r.strip() for r in require_list.split(',') if r.strip()) + self.assertEqual(expected_result, show_name_helpers.pass_wordlist_checks(name, False, show_obj=show_obj), 'Expected %s with test: "%s" with ignore: "%s", require: "%s"' % (expected_result, name, ignore_list, require_list)) @@ -83,16 +109,17 @@ class TestCase(unittest.TestCase): test_cases = (self.cases_contains, isolated)[len(isolated)] for case_num, (name, csv_words, expected_result) in enumerate(test_cases): + s_words, s_regex = helpers.split_word_str(csv_words) name = name if '%02d' not in name else name % case_num - self.assertEqual(expected_result, self.call_contains_any(name, csv_words), + self.assertEqual(expected_result, self.call_contains_any(name, s_words, rx=s_regex), 'Expected %s test: "%s" with csv_words: "%s"' % (expected_result, name, csv_words)) @staticmethod - def call_contains_any(name, csv_words): + def call_contains_any(name, csv_words, *args, **kwargs): re_extras = dict(re_prefix='.*', re_suffix='.*') - match = show_name_helpers.contains_any(name, csv_words, **re_extras) - return None is not match and match + re_extras.update(kwargs) + return show_name_helpers.contains_any(name, csv_words, *args, **re_extras) def test_not_contains_any(self): # default:[] or copy in a test case tuple to debug in isolation @@ -100,16 +127,17 @@ class TestCase(unittest.TestCase): test_cases = (self.cases_not_contains, isolated)[len(isolated)] for case_num, (name, csv_words, expected_result) in enumerate(test_cases): + s_words, s_regex = helpers.split_word_str(csv_words) name = name if '%02d' not in name else name % case_num - self.assertEqual(expected_result, self.call_not_contains_any(name, csv_words), + self.assertEqual(expected_result, self.call_not_contains_any(name, s_words, rx=s_regex), 'Expected %s test: "%s" with csv_words:"%s"' % (expected_result, name, csv_words)) @staticmethod - def call_not_contains_any(name, csv_words): + def call_not_contains_any(name, csv_words, *args, **kwargs): re_extras = dict(re_prefix='.*', re_suffix='.*') - match = show_name_helpers.not_contains_any(name, csv_words, **re_extras) - return None is not match and match + re_extras.update(kwargs) + return show_name_helpers.not_contains_any(name, csv_words, *args, **re_extras) if '__main__' == __name__: