Browse Source

Add exclude ignore words and exclude required words to settings/Search, Edit and View show.

Clean shows ignore, require list before saving global lists.
Clean require, ignore words after db upgrade.
Add global ignore, require words cleanup to show_updater.
Clean editShow ignore/require list.
Also cleanup exclude lists of shows.
Optionally restore excludes from previous downgrade.
Add lib folder to all_tests.py to fix local tests.
Add shows ignore/require words to pass_wordlist_checks if show_obj is given.
pull/1289/head
Prinz23 5 years ago
committed by JackDandy
parent
commit
c69504ae67
  1. 1
      CHANGES.md
  2. 20
      gui/slick/interfaces/default/apiBuilder.tmpl
  3. 30
      gui/slick/interfaces/default/config_search.tmpl
  4. 12
      gui/slick/interfaces/default/displayShow.tmpl
  5. 57
      gui/slick/interfaces/default/editShow.tmpl
  6. 21
      sickbeard/__init__.py
  7. 33
      sickbeard/config.py
  8. 30
      sickbeard/databases/mainDB.py
  9. 1
      sickbeard/db.py
  10. 42
      sickbeard/helpers.py
  11. 5
      sickbeard/properFinder.py
  12. 17
      sickbeard/search.py
  13. 108
      sickbeard/show_name_helpers.py
  14. 36
      sickbeard/show_updater.py
  15. 15
      sickbeard/tv.py
  16. 44
      sickbeard/tv_base.py
  17. 9
      sickbeard/tvcache.py
  18. 189
      sickbeard/webapi.py
  19. 45
      sickbeard/webserve.py
  20. 5
      sickgear.py
  21. 2
      tests/all_tests.py
  22. 114
      tests/ignore_and_require_words_tests.py

1
CHANGES.md

@ -20,6 +20,7 @@
* Change enable image caching on browse pages
* Change update sceneNameCache after scene names are updated
* Change add core dedicated base class tvinfo_base to unify future info sources
* Add exclude ignore words and exclude required words to settings/Search, Edit and View show
[develop changelog]

20
gui/slick/interfaces/default/apiBuilder.tmpl

@ -191,18 +191,28 @@ addList("setwords", "Optional Param", "", "addwords");
addList("setwords", "$cur_show_obj.name", "&indexerid=$cur_show_obj.prodid&indexer=$cur_show_obj.tvid", "addwords");
#end for
addList("addwords", "Optional Param", "", "removewords");
addList("addwords", "Add (Optional)", "", "removewords");
addList("addwords", "ignore1", "&add=ignore1", "removewords");
addList("addwords", "ignore2, ignore3", "&add=ignore2|ignore3", "removewords");
addList("removewords", "Optional Param", "", "useregex");
addList("removewords", "Remove (Optional)", "", "useregex");
addList("removewords", "ignore1", "&remove=ignore1", "useregex");
addList("removewords", "ignore2", "&remove=ignore2", "useregex");
addList("removewords", "ignore2, ignore3", "&remove=ignore2|ignore3", "useregex");
addOption("useregex", "Optional Param", "", 1);
addOption("useregex", "as Regex", "&regex=1");
addOption("useregex", "as Words", "&regex=0");
addList("useregex", "Optional Param", "", "excludeadd");
addList("useregex", "as Regex", "&regex=1", "excludeadd");
addList("useregex", "as Words", "&regex=0", "excludeadd");
addList("excludeadd", "Add Exclude (Optional)", "", "excluderemove");
addList("excludeadd", "ignore1", "&add_exclude=ignore1", "excluderemove");
addList("excludeadd", "ignore2", "&add_exclude=ignore2", "excluderemove");
addList("excludeadd", "ignore2, ignore3", "&add_exclude=ignore2|ignore3", "excluderemove");
addList("excluderemove", "Remove Exclude (Optional)", "", "");
addList("excluderemove", "ignore1", "&remove_exclude=ignore1", "");
addList("excluderemove", "ignore2", "&remove_exclude=ignore2", "");
addList("excluderemove", "ignore2, ignore3", "&remove_exclude=ignore2|ignore3", "");
addOption("listrequiredwords", "Optional Param", "", 1);
#for $cur_show_obj in $sortedShowList:

30
gui/slick/interfaces/default/config_search.tmpl

@ -1,6 +1,6 @@
#import sickbeard
#from sickbeard import clients
#from sickbeard.helpers import starify
#from sickbeard.helpers import starify, generate_word_str
<% def sg_var(varname, default=False): return getattr(sickbeard, varname, default) %>#slurp#
<% def sg_str(varname, default=''): return getattr(sickbeard, varname, default) %>#slurp#
##
@ -143,7 +143,7 @@
<label>
<span class="component-title">Ignore result with any word</span>
<span class="component-desc">
<input type="text" name="ignore_words" value="$sickbeard.IGNORE_WORDS" class="form-control input-sm input350"><p>(opt: start 'regex:')</p>
<input type="text" name="ignore_words" value="$generate_word_str($sickbeard.IGNORE_WORDS, $sickbeard.IGNORE_WORDS_REGEX)" class="form-control input-sm input350"><p>(opt: start 'regex:')</p>
<p class="clear-left note">ignore search result <em class="grey-text">if its title contains any</em> of these comma seperated words</p>
</span>
<span class="component-title">Shows with custom ignores</span>
@ -158,6 +158,18 @@
<p style="line-height:1.2em;margin-top:7px">...will list here when in use</p>
#end if
</span>
<span class="component-title" style="clear:both">Shows with exclude ignores</span>
<span class="component-desc">
#set $shows = []
#for $show in $using_exclude_ignore_words
#set void = $shows.append('<a href="%s/home/edit-show?tvid_prodid=%s" style="vertical-align:middle">%s</a>' % ($sbRoot, $show[0], $show[1]))
#end for
#if len($using_exclude_ignore_words)
<p style="margin-top:6px">#echo ', '.join($shows)#</p>
#else
<p style="margin-top:7px">...will list here when in use</p>
#end if
</span>
</label>
</div>
@ -165,7 +177,7 @@
<label>
<span class="component-title">Require all these words</span>
<span class="component-desc">
<input type="text" name="require_words" value="$sickbeard.REQUIRE_WORDS" class="form-control input-sm input350"><p>(opt: start 'regex:')</p>
<input type="text" name="require_words" value="$generate_word_str($sickbeard.REQUIRE_WORDS, $sickbeard.REQUIRE_WORDS_REGEX)" class="form-control input-sm input350"><p>(opt: start 'regex:')</p>
<p class="clear-left note">ignore search result <em class="grey-text">unless its title contains all</em> of these comma seperated words</p>
</span>
<span class="component-title">Shows with custom requires</span>
@ -180,6 +192,18 @@
<p style="line-height:1.2em;margin-top:7px">...will list here when in use</p>
#end if
</span>
<span class="component-title" style="clear:both">Shows with exclude requires</span>
<span class="component-desc">
#set $shows = []
#for $show in $using_exclude_require_words
#set void = $shows.append('<a href="%s/home/edit-show?tvid_prodid=%s" style="vertical-align:middle">%s</a>' % ($sbRoot, $show[0], $show[1]))
#end for
#if len($using_exclude_require_words)
<p style="margin-top:6px">#echo ', '.join($shows)#</p>
#else
<p style="margin-top:7px">...will list here when in use</p>
#end if
</span>
</label>
</div>

12
gui/slick/interfaces/default/displayShow.tmpl

@ -2,7 +2,7 @@
#from sickbeard import TVInfoAPI, indexermapper, network_timezones
#from sickbeard.common import Overview, qualityPresets, qualityPresetStrings, \
Quality, statusStrings, WANTED, SKIPPED, ARCHIVED, IGNORED, FAILED, DOWNLOADED
#from sickbeard.helpers import anon_url, get_size, human, maybe_plural
#from sickbeard.helpers import anon_url, get_size, human, maybe_plural, generate_word_str
#from sickbeard.indexers.indexer_config import TVINFO_TVDB, TVINFO_IMDB
#from six import iteritems
<% def sg_var(varname, default=False): return getattr(sickbeard, varname, default) %>#slurp#
@ -348,10 +348,16 @@
<span class="label addQTip" title="$exceptions_string.replace(', ', '<br />')">Scene names</span>
#end if
#if $show_obj.rls_ignore_words
<span class="label addQTip" title="#echo $show_obj.rls_ignore_words.replace(',', '<br />')#">Ignored words</span>
<span class="label addQTip" title="#echo $generate_word_str($show_obj.rls_ignore_words, $show_obj.rls_ignore_words_regex, join_chr='<br />')#">Ignored words</span>
#end if
#if $show_obj.rls_require_words
<span class="label addQTip" title="#echo $show_obj.rls_require_words.replace(',', '<br />')#">Required words</span>
<span class="label addQTip" title="#echo $generate_word_str($show_obj.rls_require_words, $show_obj.rls_require_words_regex, join_chr='<br />')#">Required words</span>
#end if
#if $show_obj.rls_global_exclude_ignore
<span class="label addQTip" title="#echo $generate_word_str($show_obj.rls_global_exclude_ignore, join_chr='<br />')#">Excluded global ignored words</span>
#end if
#if $show_obj.rls_global_exclude_require
<span class="label addQTip" title="#echo $generate_word_str($show_obj.rls_global_exclude_require, join_chr='<br />')#">Excluded global required words</span>
#end if
#if $show_obj.flatten_folders or $sg_var('NAMING_FORCE_FOLDERS')
<span class="label">Flat folders</span>

57
gui/slick/interfaces/default/editShow.tmpl

@ -1,8 +1,8 @@
#import sickbeard
#import lib.adba as adba
#from sickbeard import (anime, common, helpers, scene_exceptions)
#from sickbeard.helpers import anon_url, generate_word_str
#from lib import exceptions_helper as exceptions
#from sickbeard.helpers import anon_url
#from sickbeard.tv import TVidProdid
<% def sg_var(varname, default=False): return getattr(sickbeard, varname, default) %>#slurp#
<% def sg_str(varname, default=''): return getattr(sickbeard, varname, default) %>#slurp#
@ -131,24 +131,75 @@
<label for="rls_ignore_words">
<span class="component-title">Ignore result with any word</span>
<span class="component-desc">
<input type="text" name="rls_ignore_words" id="rls_ignore_words" value="$show_obj.rls_ignore_words" class="form-control form-control-inline input-sm input350">
<input type="text" name="rls_ignore_words" id="rls_ignore_words" value="$generate_word_str($show_obj.rls_ignore_words - $sickbeard.IGNORE_WORDS, $show_obj.rls_ignore_words_regex)" class="form-control form-control-inline input-sm input350">
<p>e.g. [[regex:]word1, word2, ..., word_n, regex_n]</p>
<p class="note">ignore search result <em class="grey-text">if its title contains any</em> of these comma seperated words or regular expressions</p>
</span>
</label>
</div>
#if $sickbeard.IGNORE_WORDS:
<div class="field-pair">
<label for="rls_global_exclude_ignore">
<span class="component-title">Exclude global ignore word (multi select list)</span>
<span class="component-desc">
<select id="rls_global_exclude_ignore" name="rls_global_exclude_ignore" multiple="multiple" class="form-control form-control-inline input-sm input350">
#set $options = ''
#set $selected = ' selected=\"selected\"'
#set $num_selected = 0
#for $gw in $sickbeard.IGNORE_WORDS:
#set $sel_html = ''
#if $gw in $show_obj.rls_global_exclude_ignore
#set $sel_html = $selected
#set $num_selected += 1
#end if
#set $options += "<option value=\"%s\"%s>%s</option>" % ($gw, $sel_html, $gw)
#end for
<option value=".*"#if $num_selected then '' else $selected#># Use all ignore word(s) (default) #</option>
$options
</select>
</span>
</label>
</div>
#end if
<div class="field-pair">
<label for="rls_require_words">
<span class="component-title">Require at least one word</span>
<span class="component-desc">
<input type="text" name="rls_require_words" id="rls_require_words" value="$show_obj.rls_require_words" class="form-control form-control-inline input-sm input350">
<input type="text" name="rls_require_words" id="rls_require_words" value="$generate_word_str($show_obj.rls_require_words - $sickbeard.REQUIRE_WORDS, $show_obj.rls_require_words_regex)" class="form-control form-control-inline input-sm input350">
<p>e.g. [[regex:]word1, word2, ..., word_n, regex_n]</p>
<p class="note">ignore search result <em class="grey-text">unless its title contains one</em> of these comma seperated words or regular expressions</p>
</span>
</label>
</div>
#if $sickbeard.REQUIRE_WORDS:
<div class="field-pair">
<label for="rls_global_exclude_require">
<span class="component-title">Exclude global require word (multi select list)</span>
<span class="component-desc">
<select id="rls_global_exclude_require" name="rls_global_exclude_require" multiple="multiple" class="form-control form-control-inline input-sm input350">
#set $options = ''
#set $selected = ' selected=\"selected\"'
#set $num_selected = 0
#for $gw in $sickbeard.REQUIRE_WORDS:
#set $sel_html = ''
#if $gw in $show_obj.rls_global_exclude_require
#set $sel_html = $selected
#set $num_selected += 1
#end if
#set $options += "<option value=\"%s\"%s>%s</option>" % ($gw, $sel_html, $gw)
#end for
<option value=".*"#if $num_selected then '' else $selected#># Use all require word(s) (default) #</option>
$options
</select>
</span>
</label>
</div>
#end if
<div class="field-pair">
#set $qualities = $common.Quality.splitQuality(int($show_obj.quality))
#set global $any_qualities = $qualities[0]

21
sickbeard/__init__.py

@ -539,10 +539,14 @@ SG_EXTRA_SCRIPTS = []
GIT_PATH = None
IGNORE_WORDS = 'regex:^(?=.*?\bspanish\b)((?!spanish.?princess).)*$, ' + \
'core2hd, hevc, MrLss, reenc, x265, danish, deutsch, dutch, flemish, french, ' + \
'german, italian, nordic, norwegian, portuguese, swedish, turkish'
REQUIRE_WORDS = ''
IGNORE_WORDS = {
'^(?=.*?\bspanish\b)((?!spanish.?princess).)*$',
'core2hd', 'hevc', 'MrLss', 'reenc', 'x265', 'danish', 'deutsch', 'dutch', 'flemish', 'french',
'german', 'italian', 'nordic', 'norwegian', 'portuguese', 'spanish', 'swedish', 'turkish'
}
IGNORE_WORDS_REGEX = True
REQUIRE_WORDS = set()
REQUIRE_WORDS_REGEX = False
WANTEDLIST_CACHE = None
@ -645,6 +649,7 @@ def init_stage_1(console_logging):
# Search Settings/Episode
global DOWNLOAD_PROPERS, PROPERS_WEBDL_ONEGRP, WEBDL_TYPES, RECENTSEARCH_FREQUENCY, \
BACKLOG_DAYS, BACKLOG_NOFULL, BACKLOG_FREQUENCY, USENET_RETENTION, IGNORE_WORDS, REQUIRE_WORDS, \
IGNORE_WORDS, IGNORE_WORDS_REGEX, REQUIRE_WORDS, REQUIRE_WORDS_REGEX, \
ALLOW_HIGH_PRIORITY, SEARCH_UNAIRED, UNAIRED_RECENT_SEARCH_ONLY
# Search Settings/NZB search
global USE_NZBS, NZB_METHOD, NZB_DIR, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \
@ -1215,8 +1220,8 @@ def init_stage_1(console_logging):
GIT_PATH = check_setting_str(CFG, 'General', 'git_path', '')
IGNORE_WORDS = check_setting_str(CFG, 'General', 'ignore_words', IGNORE_WORDS)
REQUIRE_WORDS = check_setting_str(CFG, 'General', 'require_words', REQUIRE_WORDS)
IGNORE_WORDS, IGNORE_WORDS_REGEX = helpers.split_word_str(check_setting_str(CFG, 'General', 'ignore_words', IGNORE_WORDS))
REQUIRE_WORDS, REQUIRE_WORDS_REGEX = helpers.split_word_str(check_setting_str(CFG, 'General', 'require_words', REQUIRE_WORDS))
CALENDAR_UNPROTECTED = bool(check_setting_int(CFG, 'General', 'calendar_unprotected', 0))
@ -1812,8 +1817,8 @@ def save_config():
new_config['General']['extra_scripts'] = '|'.join(EXTRA_SCRIPTS)
new_config['General']['sg_extra_scripts'] = '|'.join(SG_EXTRA_SCRIPTS)
new_config['General']['git_path'] = GIT_PATH
new_config['General']['ignore_words'] = IGNORE_WORDS
new_config['General']['require_words'] = REQUIRE_WORDS
new_config['General']['ignore_words'] = helpers.generate_word_str(IGNORE_WORDS, IGNORE_WORDS_REGEX)
new_config['General']['require_words'] = helpers.generate_word_str(REQUIRE_WORDS, REQUIRE_WORDS_REGEX)
new_config['General']['calendar_unprotected'] = int(CALENDAR_UNPROTECTED)
default_not_zero = ('enable_recentsearch', 'enable_backlog', 'enable_scheduled_backlog', 'use_after_get_data')

33
sickbeard/config.py

@ -783,25 +783,11 @@ class ConfigMigrator(object):
else:
sickbeard.SHOWLIST_TAGVIEW = 'default'
@staticmethod
def _migrate_v12():
def _migrate_v12(self):
# add words to ignore list and insert spaces to improve the ui config readability
words_to_add = ['hevc', 'reenc', 'x265', 'danish', 'deutsch', 'flemish', 'italian',
'nordic', 'norwegian', 'portuguese', 'spanish', 'turkish']
config_words = sickbeard.IGNORE_WORDS.split(',')
new_list = []
for new_word in words_to_add:
add_word = True
for ignore_word in config_words:
ignored = ignore_word.strip().lower()
if ignored and ignored not in new_list:
new_list += [ignored]
if re.search(r'(?i)%s' % new_word, ignored):
add_word = False
if add_word:
new_list += [new_word]
sickbeard.IGNORE_WORDS = ', '.join(sorted(new_list))
self.add_ignore_words(words_to_add)
@staticmethod
def _migrate_v13():
@ -861,25 +847,24 @@ class ConfigMigrator(object):
if not isinstance(removelist, list):
removelist = ([removelist], [])[None is removelist]
words = sickbeard.IGNORE_WORDS.split(',') + wordlist
new_list = []
new_list = set()
dedupe = []
using_regex = ''
for ignore_word in words:
using_regex = False
for ignore_word in list(sickbeard.IGNORE_WORDS) + wordlist: # words:
word = ignore_word.strip()
if word.startswith('regex:'):
word = word.lstrip('regex:').strip()
using_regex = 'regex:'
using_regex = True # 'regex:'
if word:
check_word = word.lower()
if check_word not in dedupe and check_word not in removelist:
dedupe += [check_word]
if 'spanish' in check_word:
word = re.sub(r'(?i)(portuguese)\|spanish(\|swedish)', r'\1\2', word)
new_list += [word]
new_list.add(word)
sickbeard.IGNORE_WORDS = '%s%s' % (using_regex, ', '.join(new_list))
sickbeard.IGNORE_WORDS = new_list
sickbeard.IGNORE_WORDS_REGEX = using_regex
def _migrate_v17(self):

30
sickbeard/databases/mainDB.py

@ -28,7 +28,7 @@ import encodingKludge as ek
from six import iteritems
MIN_DB_VERSION = 9 # oldest db version we support migrating from
MAX_DB_VERSION = 20011
MAX_DB_VERSION = 20012
TEST_BASE_VERSION = None # the base production db version, only needed for TEST db versions (>=100000)
@ -1642,3 +1642,31 @@ class AddIndexerToTables(db.SchemaUpgrade):
self.setDBVersion(20011)
return self.checkDBVersion()
# 20011 -> 20012
class AddShowExludeGlobals(db.SchemaUpgrade):
def execute(self):
if not self.hasColumn('tv_shows', 'rls_global_exclude_ignore'):
logger.log('Adding rls_global_exclude_ignore, rls_global_exclude_require to tv_shows')
db.backup_database('sickbeard.db', self.checkDBVersion())
self.addColumn('tv_shows', 'rls_global_exclude_ignore', data_type='TEXT', default='')
self.addColumn('tv_shows', 'rls_global_exclude_require', data_type='TEXT', default='')
if self.hasTable('tv_shows_exclude_backup'):
self.connection.mass_action([['UPDATE tv_shows SET rls_global_exclude_ignore = '
'(SELECT te.rls_global_exclude_ignore FROM tv_shows_exclude_backup te WHERE '
'te.show_id = tv_shows.show_id AND te.indexer = tv_shows.indexer), '
'rls_global_exclude_require = (SELECT te.rls_global_exclude_require FROM '
'tv_shows_exclude_backup te WHERE te.show_id = tv_shows.show_id AND '
'te.indexer = tv_shows.indexer) WHERE EXISTS (SELECT 1 FROM '
'tv_shows_exclude_backup WHERE tv_shows.show_id = '
'tv_shows_exclude_backup.show_id AND '
'tv_shows.indexer = tv_shows_exclude_backup.indexer)'],
['DROP TABLE tv_shows_exclude_backup']
])
self.setDBVersion(20012)
return self.checkDBVersion()

1
sickbeard/db.py

@ -618,6 +618,7 @@ def MigrationCode(my_db):
20008: sickbeard.mainDB.AddWatched,
20009: sickbeard.mainDB.AddPrune,
20010: sickbeard.mainDB.AddIndexerToTables,
20011: sickbeard.mainDB.AddShowExludeGlobals,
# 20002: sickbeard.mainDB.AddCoolSickGearFeature3,
}

42
sickbeard/helpers.py

@ -68,7 +68,7 @@ from sg_helpers import chmod_as_parent, clean_data, get_system_temp_dir, \
# noinspection PyUnreachableCode
if False:
# noinspection PyUnresolvedReferences
from typing import Any, AnyStr, Dict, NoReturn, Iterable, Iterator, List, Optional, Tuple, Union
from typing import Any, AnyStr, Dict, NoReturn, Iterable, Iterator, List, Optional, Set, Tuple, Union
from .tv import TVShow
# the following workaround hack resolves a pyc resolution bug
from .name_cache import retrieveNameFromCache
@ -2028,3 +2028,43 @@ def parse_imdb_id(string):
pass
return result
def generate_word_str(words, regex=False, join_chr=','):
# type: (Set[AnyStr], bool, AnyStr) -> AnyStr
"""
combine a list or set to a string with optional prefix 'regex:'
:param words: list or set of words
:type words: set
:param regex: prefix regex: ?
:type regex: bool
:param join_chr: character(s) used for join words
:type join_chr: basestring
:return: combined string
:rtype: basestring
"""
return '%s%s' % (('', 'regex:')[True is regex], join_chr.join(words))
def split_word_str(word_list):
# type: (AnyStr) -> Tuple[Set[AnyStr], bool]
"""
split string into set and boolean regex
:param word_list: string with words
:type word_list: basestring
:return: set of words, is it regex
:rtype: (set, bool)
"""
try:
if word_list.startswith('regex:'):
rx = True
word_list = word_list.replace('regex:', '')
else:
rx = False
s = set(w.strip() for w in word_list.split(',') if w.strip())
except (BaseException, Exception):
rx = False
s = set()
return s, rx

5
sickbeard/properFinder.py

@ -292,13 +292,14 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_dict=
logger.DEBUG)
continue
if not show_name_helpers.pass_wordlist_checks(cur_proper.name, parse=False, indexer_lookup=False):
if not show_name_helpers.pass_wordlist_checks(cur_proper.name, parse=False, indexer_lookup=False,
show_obj=cur_proper.parsed_show_obj):
logger.log('Ignored unwanted Proper [%s]' % cur_proper.name, logger.DEBUG)
continue
re_x = dict(re_prefix='.*', re_suffix='.*')
result = show_name_helpers.contains_any(cur_proper.name, cur_proper.parsed_show_obj.rls_ignore_words,
**re_x)
rx=cur_proper.parsed_show_obj.rls_ignore_words_regex, **re_x)
if None is not result and result:
logger.log('Ignored Proper containing ignore word [%s]' % cur_proper.name, logger.DEBUG)
continue

17
sickbeard/search.py

@ -216,12 +216,14 @@ def pass_show_wordlist_checks(name, show_obj):
:return: passed check
"""
re_extras = dict(re_prefix='.*', re_suffix='.*')
result = show_name_helpers.contains_any(name, show_obj.rls_ignore_words, **re_extras)
result = show_name_helpers.contains_any(name, show_obj.rls_ignore_words, rx=show_obj.rls_ignore_words_regex,
**re_extras)
if None is not result and result:
logger.log(u'Ignored: %s for containing ignore word' % name)
return False
result = show_name_helpers.contains_any(name, show_obj.rls_require_words, **re_extras)
result = show_name_helpers.contains_any(name, show_obj.rls_require_words, rx=show_obj.rls_require_words_regex,
**re_extras)
if None is not result and not result:
logger.log(u'Ignored: %s for not containing any required word match' % name)
return False
@ -797,8 +799,8 @@ def search_providers(
for cur_search_result in search_result_list:
# skip non-tv crap
search_result_list[cur_search_result] = filter_list(
lambda ep_item: show_name_helpers.pass_wordlist_checks(
ep_item.name, parse=False, indexer_lookup=False) and ep_item.show_obj == show_obj,
lambda ep_item: ep_item.show_obj == show_obj and show_name_helpers.pass_wordlist_checks(
ep_item.name, parse=False, indexer_lookup=False, show_obj=ep_item.show_obj),
search_result_list[cur_search_result])
if cur_search_result in found_results:
@ -881,8 +883,8 @@ def search_providers(
individual_results = nzbSplitter.splitResult(best_season_result)
for cur_result in filter_iter(
lambda r: show_name_helpers.pass_wordlist_checks(
r.name, parse=False, indexer_lookup=False) and r.show_obj == show_obj, individual_results):
lambda r: r.show_obj == show_obj and show_name_helpers.pass_wordlist_checks(
r.name, parse=False, indexer_lookup=False, show_obj=r.show_obj), individual_results):
ep_num = None
if 1 == len(cur_result.ep_obj_list):
ep_num = cur_result.ep_obj_list[0].episode
@ -1032,7 +1034,8 @@ def search_providers(
if name:
if not pass_show_wordlist_checks(name, show_obj):
continue
if not show_name_helpers.pass_wordlist_checks(name, indexer_lookup=False):
if not show_name_helpers.pass_wordlist_checks(name, indexer_lookup=False,
show_obj=show_obj):
logger.log('Ignored: %s (debug log has detail)' % name)
continue
best_result.name = name

108
sickbeard/show_name_helpers.py

@ -18,6 +18,7 @@
import datetime
import fnmatch
import os
import copy
import re
# noinspection PyPep8Naming
@ -35,21 +36,32 @@ from six import iterkeys, itervalues
# noinspection PyUnreachableCode
if False:
from typing import AnyStr, List, Union
from typing import AnyStr, List, Optional, Set, Union
from .tv import TVShow
# noinspection PyUnresolvedReferences
from re import Pattern
def pass_wordlist_checks(name, # type: AnyStr
def pass_wordlist_checks(name, # type: AnyStr
parse=True, # type: bool
indexer_lookup=True # type: bool
indexer_lookup=True, # type: bool
show_obj=None # type: TVShow
): # type: (...) -> bool
"""
Filters out non-english and just all-around stupid releases by comparing
the word list contents at boundaries or the end of name.
:param name: the release name to check
:param parse: try to parse release name
:param indexer_lookup: try to look up on tvinfo source
:type name: basestring
:param parse: parse release name
:type parse: bool
:param indexer_lookup: use indexer lookup during paring
:type indexer_lookup: bool
:param show_obj: TVShow object
:type show_obj: TVShow
:return: True if the release name is OK, False if it's bad.
:rtype: bool
"""
if parse:
@ -63,21 +75,43 @@ def pass_wordlist_checks(name, # type: AnyStr
logger.log(err_msg + 'show', logger.DEBUG)
return False
word_list = ['sub(bed|ed|pack|s)', '(dk|fin|heb|kor|nor|nordic|pl|swe)sub(bed|ed|s)?',
word_list = {'sub(bed|ed|pack|s)', '(dk|fin|heb|kor|nor|nordic|pl|swe)sub(bed|ed|s)?',
'(dir|sample|sub|nfo)fix', 'sample', '(dvd)?extras',
'dub(bed)?']
'dub(bed)?'}
# if any of the bad strings are in the name then say no
if sickbeard.IGNORE_WORDS:
word_list = ','.join([sickbeard.IGNORE_WORDS] + word_list)
word_list.update(sickbeard.IGNORE_WORDS)
req_word_list = copy.copy(sickbeard.REQUIRE_WORDS)
result = contains_any(name, word_list)
result = None
if show_obj:
if show_obj.rls_ignore_words and isinstance(show_obj.rls_ignore_words, set):
if sickbeard.IGNORE_WORDS_REGEX == show_obj.rls_ignore_words_regex:
word_list.update(show_obj.rls_ignore_words)
else:
result = contains_any(name, show_obj.rls_ignore_words, rx=show_obj.rls_ignore_words_regex)
if show_obj.rls_global_exclude_ignore and isinstance(show_obj.rls_global_exclude_ignore, set):
word_list = word_list - show_obj.rls_global_exclude_ignore
result = result or contains_any(name, word_list, rx=sickbeard.IGNORE_WORDS_REGEX)
if None is not result and result:
logger.log(u'Ignored: %s for containing ignore word' % name, logger.DEBUG)
return False
result = None
if show_obj:
if show_obj.rls_require_words and isinstance(show_obj.rls_require_words, set):
if sickbeard.REQUIRE_WORDS_REGEX == show_obj.rls_require_words_regex:
req_word_list.update(show_obj.rls_require_words)
else:
result = not_contains_any(name, show_obj.rls_require_words, rx=show_obj.rls_require_words_regex)
if show_obj.rls_global_exclude_require and isinstance(show_obj.rls_global_exclude_require, set):
req_word_list = req_word_list - show_obj.rls_global_exclude_require
# if any of the good strings aren't in the name then say no
result = not_contains_any(name, sickbeard.REQUIRE_WORDS)
result = result or not_contains_any(name, req_word_list, rx=sickbeard.REQUIRE_WORDS_REGEX)
if None is not result and result:
logger.log(u'Ignored: %s for not containing required word match' % name, logger.DEBUG)
return False
@ -86,33 +120,45 @@ def pass_wordlist_checks(name, # type: AnyStr
def not_contains_any(subject, # type: AnyStr
lookup_words, # type: Union[AnyStr, List[AnyStr]]
lookup_words, # type: Union[AnyStr, Set[AnyStr]]
rx=None,
**kwargs
): # type: (...) -> bool
return contains_any(subject, lookup_words, invert=True, **kwargs)
return contains_any(subject, lookup_words, invert=True, rx=rx, **kwargs)
def contains_any(subject, # type: AnyStr
lookup_words, # type: Union[AnyStr, List[AnyStr]]
lookup_words, # type: Union[AnyStr, Set[AnyStr]]
invert=False, # type: bool
rx=None,
**kwargs
): # type: (...) -> Union[bool, None]
): # type: (...) -> Optional[bool]
"""
Check if subject does or does not contain a match from a list or string of regular expression lookup words
word: word to test existence of
re_prefix: insert string to all lookup words
re_suffix: append string to all lookup words
:param subject: word to test existence of
:type subject: basestring
:param lookup_words: List or comma separated string of words to search
:type lookup_words: Union(list, set, basestring)
:param re_prefix: insert string to all lookup words
:type re_prefix: basestring
:param re_suffix: append string to all lookup words
:type re_suffix: basestring
:param invert: invert function logic "contains any" into "does not contain any"
:type invert: bool
:param rx: lookup_words are regex
:type rx: Union(NoneType, bool)
:param subject:
:return: None if no checking was done. True for first match found, or if invert is False,
:param lookup_words: List or comma separated string of words to search
:param invert: invert function logic "contains any" into "does not contain any"
:param kwargs:
:return: None if no checking was done. True for first match found, or if invert is False,
then True for first pattern that does not match, or False
:rtype: Union(NoneType, bool)
"""
compiled_words = compile_word_list(lookup_words, **kwargs)
compiled_words = compile_word_list(lookup_words, rx=rx, **kwargs)
if subject and compiled_words:
for rc_filter in compiled_words:
match = rc_filter.search(subject)
@ -125,19 +171,25 @@ def contains_any(subject, # type: AnyStr
return None
def compile_word_list(lookup_words, # type: AnyStr
def compile_word_list(lookup_words, # type: Union[AnyStr, Set[AnyStr]]
re_prefix=r'(^|[\W_])', # type: AnyStr
re_suffix=r'($|[\W_])' # type: AnyStr
): # type: (...) -> List[AnyStr]
re_suffix=r'($|[\W_])', # type: AnyStr
rx=None
): # type: (...) -> List[Pattern[AnyStr]]
result = []
if lookup_words:
search_raw = isinstance(lookup_words, list)
if not search_raw:
search_raw = not lookup_words.startswith('regex:')
lookup_words = lookup_words[(6, 0)[search_raw]:].split(',')
lookup_words = [x.strip() for x in lookup_words]
for word in [x for x in lookup_words if x]:
if None is rx:
search_raw = isinstance(lookup_words, list)
if not search_raw:
# noinspection PyUnresolvedReferences
search_raw = not lookup_words.startswith('regex:')
# noinspection PyUnresolvedReferences
lookup_words = lookup_words[(6, 0)[search_raw]:].split(',')
lookup_words = [x.strip() for x in lookup_words if x.strip()]
else:
search_raw = not rx
for word in lookup_words:
try:
# !0 == regex and subject = s / 'what\'s the "time"' / what\'s\ the\ \"time\"
subject = search_raw and re.escape(word) or re.sub(r'([\" \'])', r'\\\1', word)

36
sickbeard/show_updater.py

@ -25,6 +25,35 @@ from exceptions_helper import ex
import sickbeard
from . import db, failed_history, logger, network_timezones, properFinder, ui
# noinspection PyUnreachableCode
if False:
from sickbeard.tv import TVShow
def clean_ignore_require_words():
"""
removes duplicate ignore/require words from shows and global lists
"""
try:
for s in sickbeard.showList: # type: TVShow
# test before set to prevent dirty setter from setting unchanged shows to dirty
if s.rls_ignore_words - sickbeard.IGNORE_WORDS != s.rls_ignore_words:
s.rls_ignore_words -= sickbeard.IGNORE_WORDS
if 0 == len(s.rls_ignore_words):
s.rls_ignore_words_regex = False
if s.rls_require_words - sickbeard.REQUIRE_WORDS != s.rls_require_words:
s.rls_require_words -= sickbeard.REQUIRE_WORDS
if 0 == len(s.rls_require_words):
s.rls_require_words_regex = False
if s.rls_global_exclude_ignore & sickbeard.IGNORE_WORDS != s.rls_global_exclude_ignore:
s.rls_global_exclude_ignore &= sickbeard.IGNORE_WORDS
if s.rls_global_exclude_require & sickbeard.REQUIRE_WORDS != s.rls_global_exclude_require:
s.rls_global_exclude_require &= sickbeard.REQUIRE_WORDS
if s.dirty:
s.save_to_db()
except (BaseException, Exception):
pass
class ShowUpdater(object):
def __init__(self):
@ -87,6 +116,13 @@ class ShowUpdater(object):
logger.log('image cache cleanup error', logger.ERROR)
logger.log(traceback.format_exc(), logger.ERROR)
# cleanup ignore and require lists
try:
clean_ignore_require_words()
except Exception:
logger.log('ignore, require words cleanup error', logger.ERROR)
logger.log(traceback.format_exc(), logger.ERROR)
# cleanup manual search history
sickbeard.search_queue.remove_old_fifo(sickbeard.search_queue.MANUAL_SEARCH_HISTORY)

15
sickbeard/tv.py

@ -1206,8 +1206,13 @@ class TVShow(TVShowBase):
self._last_update_indexer = sql_result[0]['last_update_indexer']
self._rls_ignore_words = sql_result[0]['rls_ignore_words']
self._rls_require_words = sql_result[0]['rls_require_words']
self._rls_ignore_words, self._rls_ignore_words_regex = helpers.split_word_str(sql_result[0]['rls_ignore_words'])
self._rls_require_words, self._rls_require_words_regex = helpers.split_word_str(sql_result[0]['rls_require_words'])
self._rls_global_exclude_ignore = helpers.split_word_str(sql_result[0]['rls_global_exclude_ignore'])[0]
self._rls_global_exclude_require = helpers.split_word_str(sql_result[0]['rls_global_exclude_require'])[0]
if not self._imdbid:
imdbid = sql_result[0]['imdb_id'] or ''
@ -1806,8 +1811,10 @@ class TVShow(TVShowBase):
startyear=self.startyear,
lang=self.lang, imdb_id=self.imdbid,
last_update_indexer=self.last_update_indexer,
rls_ignore_words=self.rls_ignore_words,
rls_require_words=self.rls_require_words,
rls_ignore_words=helpers.generate_word_str(self.rls_ignore_words, self.rls_ignore_words_regex),
rls_require_words=helpers.generate_word_str(self.rls_require_words, self.rls_require_words_regex),
rls_global_exclude_ignore=','.join(self.rls_global_exclude_ignore),
rls_global_exclude_require=','.join(self.rls_global_exclude_require),
overview=self.overview,
prune=self.prune,
tag=self.tag)

44
sickbeard/tv_base.py

@ -80,14 +80,50 @@ class TVShowBase(LegacyTVShow, TVBase):
self._sports = 0
self._anime = 0
self._scene = 0
self._rls_ignore_words = ''
self._rls_require_words = ''
self._rls_ignore_words = set()
self._rls_require_words = set()
self._overview = ''
self._prune = 0
self._tag = ''
self._rls_ignore_words_regex = False
self._rls_require_words_regex = False
self._rls_global_exclude_ignore = set()
self._rls_global_exclude_require = set()
# name = property(lambda self: self._name, dirty_setter('_name'))
@property
def rls_ignore_words_regex(self):
return self._rls_ignore_words_regex
@rls_ignore_words_regex.setter
def rls_ignore_words_regex(self, val):
self.dirty_setter('_rls_ignore_words_regex')(self, val)
@property
def rls_require_words_regex(self):
return self._rls_require_words_regex
@rls_require_words_regex.setter
def rls_require_words_regex(self, val):
self.dirty_setter('_rls_require_words_regex')(self, val)
@property
def rls_global_exclude_ignore(self):
return self._rls_global_exclude_ignore
@rls_global_exclude_ignore.setter
def rls_global_exclude_ignore(self, val):
self.dirty_setter('_rls_global_exclude_ignore', set)(self, val)
@property
def rls_global_exclude_require(self):
return self._rls_global_exclude_require
@rls_global_exclude_require.setter
def rls_global_exclude_require(self, val):
self.dirty_setter('_rls_global_exclude_require', set)(self, val)
@property
def name(self):
return self._name
@ -285,7 +321,7 @@ class TVShowBase(LegacyTVShow, TVBase):
@rls_ignore_words.setter
def rls_ignore_words(self, *arg):
self.dirty_setter('_rls_ignore_words')(self, *arg)
self.dirty_setter('_rls_ignore_words', set)(self, *arg)
# rls_require_words = property(lambda self: self._rls_require_words, dirty_setter('_rls_require_words'))
@property
@ -294,7 +330,7 @@ class TVShowBase(LegacyTVShow, TVBase):
@rls_require_words.setter
def rls_require_words(self, *arg):
self.dirty_setter('_rls_require_words')(self, *arg)
self.dirty_setter('_rls_require_words', set)(self, *arg)
# overview = property(lambda self: self._overview, dirty_setter('_overview'))
@property

9
sickbeard/tvcache.py

@ -407,15 +407,16 @@ class TVCache(object):
# for each cache entry
for cur_result in sql_result:
# skip non-tv crap
if not show_name_helpers.pass_wordlist_checks(cur_result['name'], parse=False, indexer_lookup=False):
continue
# get the show object, or if it's not one of our shows then ignore it
show_obj = helpers.find_show_by_id({int(cur_result['indexer']): int(cur_result['indexerid'])})
if not show_obj:
continue
# skip non-tv crap
if not show_name_helpers.pass_wordlist_checks(cur_result['name'], parse=False, indexer_lookup=False,
show_obj=show_obj):
continue
# skip if provider is anime only and show is not anime
if self.provider.anime_only and not show_obj.is_anime:
logger.log(u'' + str(show_obj.name) + ' is not an anime, skipping', logger.DEBUG)

189
sickbeard/webapi.py

@ -25,6 +25,7 @@ from random import randint
import datetime
import glob
import copy
try:
import json
except ImportError:
@ -52,6 +53,7 @@ from .indexers.indexer_config import *
from tvinfo_base.exceptions import *
from .scene_numbering import set_scene_numbering_helper
from .search_backlog import FORCED_BACKLOG
from .show_updater import clean_ignore_require_words
from .sgdatetime import SGDatetime
from .tv import TVEpisode, TVShow, TVidProdid
from .webserve import AddShows
@ -2809,21 +2811,22 @@ class CMD_SickGearListIgnoreWords(ApiCall):
if self.tvid and self.prodid:
my_db = db.DBConnection()
sql_result = my_db.select(
'SELECT show_name, rls_ignore_words'
'SELECT show_name, rls_ignore_words, rls_global_exclude_ignore'
' FROM tv_shows'
' WHERE indexer = ? AND indexer_id = ?',
[self.tvid, self.prodid])
if sql_result:
ignore_words = sql_result[0]['rls_ignore_words']
return_data = {'type': 'show', 'indexer': self.tvid, 'indexerid': self.prodid,
'show name': sql_result[0]['show_name']}
'show name': sql_result[0]['show_name'],
'global_exclude_ignore': sql_result[0]['rls_global_exclude_ignore']}
return_type = '%s:' % sql_result[0]['show_name']
else:
return _responds(RESULT_FAILURE, msg='Show not found.')
elif (None is self.tvid) != (None is self.prodid):
return _responds(RESULT_FAILURE, msg='You must supply indexer + indexerid.')
else:
ignore_words = sickbeard.IGNORE_WORDS
ignore_words = helpers.generate_word_str(sickbeard.IGNORE_WORDS, sickbeard.IGNORE_WORDS_REGEX)
return_data = {'type': 'global'}
return_type = 'Global'
@ -2838,6 +2841,8 @@ class CMD_SickGearSetIgnoreWords(ApiCall):
"indexer": {"desc": "indexer of a show"},
"add": {"desc": "add words to list"},
"remove": {"desc": "remove words from list"},
"add_exclude": {"desc": "add global exclude words"},
"remove_exclude": {"desc": "remove global exclude words"},
"regex": {"desc": "interpret ALL (including existing) ignore words as regex"},
}
}
@ -2850,14 +2855,23 @@ class CMD_SickGearSetIgnoreWords(ApiCall):
[i for i in indexer_api.TVInfoAPI().sources])
self.add, args = self.check_params(args, kwargs, "add", None, False, "list", [])
self.remove, args = self.check_params(args, kwargs, "remove", None, False, "list", [])
self.add_exclude, args = self.check_params(args, kwargs, "add_exclude", None, False, "list", [])
self.remove_exclude, args = self.check_params(args, kwargs, "remove_exclude", None, False, "list", [])
self.regex, args = self.check_params(args, kwargs, "regex", None, False, "bool", [])
# super, missing, help
ApiCall.__init__(self, handler, args, kwargs)
def run(self):
""" set ignore words """
if not self.add and not self.remove:
return _responds(RESULT_FAILURE, msg="No words to add/remove provided")
if (not self.add and not self.remove and not self.add_exclude and not self.remove_exclude) or \
((self.add_exclude or self.remove_exclude) and not (self.tvid and self.prodid)):
return _responds(RESULT_FAILURE, msg=('No indexer, indexerid provided',
'No words to add/remove provided')[None is not self.tvid and
None is not self.prodid])
use_regex = None
return_type = ''
ignore_list = set()
def _create_ignore_words():
_use_regex = ignore_words.startswith('regex:')
@ -2883,35 +2897,63 @@ class CMD_SickGearSetIgnoreWords(ApiCall):
if not show_obj:
return _responds(RESULT_FAILURE, msg="Show not found")
my_db = db.DBConnection()
sql_result = my_db.select('SELECT show_name, rls_ignore_words'
' FROM tv_shows'
' WHERE indexer = ? AND indexer_id = ?',
[self.tvid, self.prodid])
ignore_words = ''
if sql_result:
ignore_words = sql_result[0]['rls_ignore_words']
return_data = {'type': 'show', 'indexer': self.tvid, 'indexerid': self.prodid,
'show name': sql_result[0]['show_name']}
return_type = '%s:' % sql_result[0]['show_name']
'show name': show_obj.name}
use_regex, ignore_list, new_ignore_words = _create_ignore_words()
my_db.action('UPDATE tv_shows SET rls_ignore_words = ? WHERE indexer = ? AND indexer_id = ?',
[new_ignore_words, self.tvid, self.prodid])
my_db = db.DBConnection()
if self.add or self.remove:
sql_results = my_db.select('SELECT show_name, rls_ignore_words FROM tv_shows WHERE indexer = ? AND '
'indexer_id = ?', [self.tvid, self.prodid])
ignore_words = ''
if sql_results:
ignore_words = sql_results[0]['rls_ignore_words']
return_type = '%s:' % sql_results[0]['show_name']
use_regex, ignore_list, new_ignore_words = _create_ignore_words()
my_db.action('UPDATE tv_shows SET rls_ignore_words = ? WHERE indexer = ? AND indexer_id = ?',
[new_ignore_words, self.tvid, self.prodid])
show_obj.rls_ignore_words, show_obj.rls_ignore_words_regex = helpers.split_word_str(new_ignore_words)
if self.add_exclude or self.remove_exclude:
sql_results = my_db.select('SELECT rls_global_exclude_ignore FROM tv_shows WHERE indexer = ? AND '
'indexer_id = ?', [self.tvid, self.prodid])
exclude_ignore = set()
if sql_results:
exclude_ignore = helpers.split_word_str(sql_results[0]['rls_global_exclude_ignore'])[0]
exclude_ignore = {i for i in exclude_ignore if i not in sickbeard.IGNORE_WORDS}
if self.add_exclude:
for a in self.add_exclude:
if a not in sickbeard.IGNORE_WORDS:
exclude_ignore.add(a)
if self.remove_exclude:
for r in self.remove_exclude:
try:
exclude_ignore.remove(r)
except KeyError:
pass
my_db.action('UPDATE tv_shows SET rls_global_exclude_ignore = ? WHERE indexer = ? AND indexer_id = ?',
[helpers.generate_word_str(exclude_ignore), self.tvid, self.prodid])
show_obj.rls_global_exclude_ignore = copy.copy(exclude_ignore)
return_data['global exclude ignore'] = exclude_ignore
elif (None is self.tvid) != (None is self.prodid):
return _responds(RESULT_FAILURE, msg='You must supply indexer + indexerid.')
else:
ignore_words = sickbeard.IGNORE_WORDS
ignore_words = helpers.generate_word_str(sickbeard.IGNORE_WORDS, sickbeard.IGNORE_WORDS_REGEX)
use_regex, ignore_list, new_ignore_words = _create_ignore_words()
sickbeard.IGNORE_WORDS = new_ignore_words
sickbeard.IGNORE_WORDS, sickbeard.IGNORE_WORDS_REGEX = helpers.split_word_str(new_ignore_words)
sickbeard.save_config()
return_data = {'type': 'global'}
return_type = 'Global'
return_data['use regex'] = use_regex
if None is not use_regex:
return_data['use regex'] = use_regex
elif None is not self.regex:
return_data['use regex'] = self.regex
return_data['ignore words'] = ignore_list
clean_ignore_require_words()
return _responds(RESULT_SUCCESS, data=return_data, msg="%s set ignore words" % return_type)
@ -2936,21 +2978,22 @@ class CMD_SickGearListRequireWords(ApiCall):
if self.tvid and self.prodid:
my_db = db.DBConnection()
sql_result = my_db.select(
'SELECT show_name, rls_require_words'
'SELECT show_name, rls_require_words, rls_global_exclude_require'
' FROM tv_shows'
' WHERE indexer = ? AND indexer_id = ?',
[self.tvid, self.prodid])
if sql_result:
required_words = sql_result[0]['rls_require_words']
return_data = {'type': 'show', 'indexer': self.tvid, 'indexerid': self.prodid,
'show name': sql_result[0]['show_name']}
'show name': sql_result[0]['show_name'],
'global_exclude_require': sql_result[0]['rls_global_exclude_require']}
return_type = '%s:' % sql_result[0]['show_name']
else:
return _responds(RESULT_FAILURE, msg='Show not found.')
elif (None is self.tvid) != (None is self.prodid):
return _responds(RESULT_FAILURE, msg='You must supply indexer + indexerid.')
else:
required_words = sickbeard.REQUIRE_WORDS
required_words = helpers.generate_word_str(sickbeard.REQUIRE_WORDS, sickbeard.REQUIRE_WORDS_REGEX)
return_data = {'type': 'global'}
return_type = 'Global'
@ -2966,6 +3009,8 @@ class CMD_SickGearSetRequrieWords(ApiCall):
"indexer": {"desc": "indexer of a show"},
"add": {"desc": "add words to list"},
"remove": {"desc": "remove words from list"},
"add_exclude": {"desc": "add global exclude words"},
"remove_exclude": {"desc": "remove global exclude words"},
"regex": {"desc": "interpret ALL (including existing) ignore words as regex"},
}
}
@ -2978,14 +3023,23 @@ class CMD_SickGearSetRequrieWords(ApiCall):
[i for i in indexer_api.TVInfoAPI().sources])
self.add, args = self.check_params(args, kwargs, "add", None, False, "list", [])
self.remove, args = self.check_params(args, kwargs, "remove", None, False, "list", [])
self.add_exclude, args = self.check_params(args, kwargs, "add_exclude", None, False, "list", [])
self.remove_exclude, args = self.check_params(args, kwargs, "remove_exclude", None, False, "list", [])
self.regex, args = self.check_params(args, kwargs, "regex", None, False, "bool", [])
# super, missing, help
ApiCall.__init__(self, handler, args, kwargs)
def run(self):
""" set require words """
if not self.add and not self.remove:
return _responds(RESULT_FAILURE, msg="No words to add/remove provided")
if (not self.add and not self.remove and not self.add_exclude and not self.remove_exclude) or \
((self.add_exclude or self.remove_exclude) and not (self.tvid and self.prodid)):
return _responds(RESULT_FAILURE, msg=('No indexer, indexerid provided',
'No words to add/remove provided')[None is not self.tvid and
None is not self.prodid])
use_regex = None
return_type = ''
required_list = set()
def _create_required_words():
_use_regex = requried_words.startswith('regex:')
@ -3011,36 +3065,65 @@ class CMD_SickGearSetRequrieWords(ApiCall):
if not show_obj:
return _responds(RESULT_FAILURE, msg="Show not found")
my_db = db.DBConnection()
sql_result = my_db.select(
'SELECT show_name, rls_require_words'
' FROM tv_shows'
' WHERE indexer = ? AND indexer_id = ?',
[self.tvid, self.prodid])
requried_words = ''
if sql_result:
requried_words = sql_result[0]['rls_require_words']
return_data = {'type': 'show', 'indexer': self.tvid, 'indexerid': self.prodid,
'show name': sql_result[0]['show_name']}
return_type = '%s:' % sql_result[0]['show_name']
'show name': show_obj.name}
use_regex, required_list, new_required_words = _create_required_words()
my_db.action('UPDATE tv_shows SET rls_require_words = ? WHERE indexer = ? AND indexer_id = ?',
[new_required_words, self.tvid, self.prodid])
my_db = db.DBConnection()
if self.add or self.remove:
sql_result = my_db.select('SELECT show_name, rls_require_words FROM tv_shows WHERE indexer = ? AND '
'indexer_id = ?', [self.tvid, self.prodid])
requried_words = ''
if sql_result:
requried_words = sql_result[0]['rls_require_words']
return_type = '%s:' % sql_result[0]['show_name']
use_regex, required_list, new_required_words = _create_required_words()
my_db.action('UPDATE tv_shows SET rls_require_words = ? WHERE indexer = ? AND indexer_id = ?',
[new_required_words, self.tvid, self.prodid])
show_obj.rls_require_words, show_obj.rls_require_words_regex = helpers.split_word_str(
new_required_words)
if self.add_exclude or self.remove_exclude:
sql_result = my_db.select('SELECT rls_global_exclude_require FROM tv_shows WHERE indexer = ? AND '
'indexer_id = ?', [self.tvid, self.prodid])
exclude_require = set()
if sql_result:
exclude_require = helpers.split_word_str(sql_result[0]['rls_global_exclude_require'])[0]
exclude_require = {r for r in exclude_require if r not in sickbeard.REQUIRE_WORDS}
if self.add_exclude:
for a in self.add_exclude:
if a not in sickbeard.REQUIRE_WORDS:
exclude_require.add(a)
if self.remove_exclude:
for r in self.remove_exclude:
try:
exclude_require.remove(r)
except KeyError:
pass
my_db.action(
'UPDATE tv_shows SET rls_global_exclude_require = ? WHERE indexer = ? AND indexer_id = ?',
[helpers.generate_word_str(exclude_require), self.tvid, self.prodid])
show_obj.rls_global_exclude_require = copy.copy(exclude_require)
return_data['global exclude require'] = exclude_require
elif (None is self.tvid) != (None is self.prodid):
return _responds(RESULT_FAILURE, msg='You must supply indexer + indexerid.')
else:
requried_words = sickbeard.REQUIRE_WORDS
requried_words = helpers.generate_word_str(sickbeard.REQUIRE_WORDS, sickbeard.REQUIRE_WORDS_REGEX)
use_regex, required_list, new_required_words = _create_required_words()
sickbeard.REQUIRE_WORDS = new_required_words
sickbeard.REQUIRE_WORDS, sickbeard.REQUIRE_WORDS_REGEX = helpers.split_word_str(new_required_words)
sickbeard.save_config()
return_data = {'type': 'global'}
return_type = 'Global'
return_data['use regex'] = use_regex
if None is not use_regex:
return_data['use regex'] = use_regex
elif None is not self.regex:
return_data['use regex'] = self.regex
return_data['required words'] = required_list
clean_ignore_require_words()
return _responds(RESULT_SUCCESS, data=return_data, msg="%s set requried words" % return_type)
@ -3138,8 +3221,10 @@ class CMD_SickGearShow(ApiCall):
showDict["status"] = show_obj.status
showDict["scenenumbering"] = show_obj.is_scene
showDict["upgrade_once"] = show_obj.upgrade_once
showDict["ignorewords"] = show_obj.rls_ignore_words
showDict["requirewords"] = show_obj.rls_require_words
showDict["ignorewords"] = helpers.generate_word_str(show_obj.rls_ignore_words, show_obj.rls_ignore_words_regex)
showDict["global_exclude_ignore"] = helpers.generate_word_str(show_obj.rls_global_exclude_ignore)
showDict["requirewords"] = helpers.generate_word_str(show_obj.rls_require_words, show_obj.rls_require_words_regex)
showDict["global_exclude_require"] = helpers.generate_word_str(show_obj.rls_global_exclude_require)
if self.overview:
showDict["overview"] = show_obj.overview
showDict["prune"] = show_obj.prune
@ -4397,8 +4482,10 @@ class CMD_SickGearShows(ApiCall):
"subtitles": cur_show_obj.subtitles,
"scenenumbering": cur_show_obj.is_scene,
"upgrade_once": cur_show_obj.upgrade_once,
"ignorewords": cur_show_obj.rls_ignore_words,
"requirewords": cur_show_obj.rls_require_words,
"ignorewords": helpers.generate_word_str(cur_show_obj.rls_ignore_words, cur_show_obj.rls_ignore_words_regex),
"global_exclude_ignore": helpers.generate_word_str(cur_show_obj.rls_global_exclude_ignore),
"requirewords": helpers.generate_word_str(cur_show_obj.rls_require_words, cur_show_obj.rls_require_words_regex),
"global_exclude_require": helpers.generate_word_str(cur_show_obj.rls_global_exclude_require),
"prune": cur_show_obj.prune,
"tag": cur_show_obj.tag,
"imdb_id": cur_show_obj.imdbid,

45
sickbeard/webserve.py

@ -59,6 +59,7 @@ from .scene_numbering import get_scene_absolute_numbering_for_show, get_scene_nu
get_xem_absolute_numbering_for_show, get_xem_numbering_for_show, set_scene_numbering_helper
from .search_backlog import FORCED_BACKLOG
from .sgdatetime import SGDatetime
from .show_updater import clean_ignore_require_words
from .trakt_helpers import build_config, trakt_collection_remove_account
from .tv import TVidProdid
@ -2444,7 +2445,8 @@ class Home(MainHandler):
flatten_folders=None, paused=None, direct_call=False, air_by_date=None, sports=None, dvdorder=None,
tvinfo_lang=None, subs=None, upgrade_once=None, rls_ignore_words=None,
rls_require_words=None, anime=None, blacklist=None, whitelist=None,
scene=None, prune=None, tag=None, quality_preset=None, reset_fanart=None, **kwargs):
scene=None, prune=None, tag=None, quality_preset=None, reset_fanart=None,
rls_global_exclude_ignore=None, rls_global_exclude_require=None, **kwargs):
any_qualities = any_qualities if None is not any_qualities else []
best_qualities = best_qualities if None is not best_qualities else []
@ -2612,8 +2614,29 @@ class Home(MainHandler):
if not direct_call:
show_obj.lang = infosrc_lang
show_obj.dvdorder = dvdorder
show_obj.rls_ignore_words = rls_ignore_words.strip()
show_obj.rls_require_words = rls_require_words.strip()
new_ignore_words, new_i_regex = helpers.split_word_str(rls_ignore_words.strip())
new_ignore_words -= sickbeard.IGNORE_WORDS
if 0 == len(new_ignore_words):
new_i_regex = False
show_obj.rls_ignore_words, show_obj.rls_ignore_words_regex = new_ignore_words, new_i_regex
new_require_words, new_r_regex = helpers.split_word_str(rls_require_words.strip())
new_require_words -= sickbeard.REQUIRE_WORDS
if 0 == len(new_require_words):
new_r_regex = False
show_obj.rls_require_words, show_obj.rls_require_words_regex = new_require_words, new_r_regex
if isinstance(rls_global_exclude_ignore, list):
show_obj.rls_global_exclude_ignore = set(r for r in rls_global_exclude_ignore if '.*' != r)
elif isinstance(rls_global_exclude_ignore, string_types) and '.*' != rls_global_exclude_ignore:
show_obj.rls_global_exclude_ignore = {rls_global_exclude_ignore}
else:
show_obj.rls_global_exclude_ignore = set()
if isinstance(rls_global_exclude_require, list):
show_obj.rls_global_exclude_require = set(r for r in rls_global_exclude_require if '.*' != r)
elif isinstance(rls_global_exclude_require, string_types) and '.*' != rls_global_exclude_require:
show_obj.rls_global_exclude_require = {rls_global_exclude_require}
else:
show_obj.rls_global_exclude_require = set()
clean_ignore_require_words()
# if we change location clear the db of episodes, change it, write to db, and rescan
# noinspection PyProtectedMember
@ -6494,11 +6517,17 @@ class ConfigSearch(Config):
t = PageTemplate(web_handler=self, file='config_search.tmpl')
t.submenu = self.config_menu('Search')
t.using_rls_ignore_words = [(cur_so.tvid_prodid, cur_so.name) for cur_so in sickbeard.showList
if cur_so.rls_ignore_words and cur_so.rls_ignore_words.strip()]
if cur_so.rls_ignore_words and cur_so.rls_ignore_words]
t.using_rls_ignore_words.sort(key=lambda x: x[1], reverse=False)
t.using_rls_require_words = [(cur_so.tvid_prodid, cur_so.name) for cur_so in sickbeard.showList
if cur_so.rls_require_words and cur_so.rls_require_words.strip()]
if cur_so.rls_require_words and cur_so.rls_require_words]
t.using_rls_require_words.sort(key=lambda x: x[1], reverse=False)
t.using_exclude_ignore_words = [(cur_so.tvid_prodid, cur_so.name)
for cur_so in sickbeard.showList if cur_so.rls_global_exclude_ignore]
t.using_exclude_ignore_words.sort(key=lambda x: x[1], reverse=False)
t.using_exclude_require_words = [(cur_so.tvid_prodid, cur_so.name)
for cur_so in sickbeard.showList if cur_so.rls_global_exclude_require]
t.using_exclude_require_words.sort(key=lambda x: x[1], reverse=False)
t.using_regex = False
try:
from sickbeard.name_parser.parser import regex
@ -6550,8 +6579,10 @@ class ConfigSearch(Config):
sickbeard.TORRENT_METHOD = torrent_method
sickbeard.USENET_RETENTION = config.to_int(usenet_retention, default=500)
sickbeard.IGNORE_WORDS = ignore_words if ignore_words else ''
sickbeard.REQUIRE_WORDS = require_words if require_words else ''
sickbeard.IGNORE_WORDS, sickbeard.IGNORE_WORDS_REGEX = helpers.split_word_str(ignore_words if ignore_words else '')
sickbeard.REQUIRE_WORDS, sickbeard.REQUIRE_WORDS_REGEX = helpers.split_word_str(require_words if require_words else '')
clean_ignore_require_words()
config.schedule_download_propers(config.checkbox_to_value(download_propers))
sickbeard.PROPERS_WEBDL_ONEGRP = config.checkbox_to_value(propers_webdl_onegrp)

5
sickgear.py

@ -503,6 +503,11 @@ class SickGear(object):
# Build from the DB to start with
sickbeard.classes.loading_msg.message = 'Loading shows from db'
self.load_shows_from_db()
if not db.DBConnection().has_flag('ignore_require_cleaned'):
from sickbeard.show_updater import clean_ignore_require_words
sickbeard.classes.loading_msg.message = 'Cleaning ignore/require words lists'
clean_ignore_require_words()
db.DBConnection().set_flag('ignore_require_cleaned')
# Fire up all our threads
sickbeard.classes.loading_msg.message = 'Starting threads'

2
tests/all_tests.py

@ -26,7 +26,9 @@ if '__main__' == __name__:
import glob
import sys
import unittest
import os
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')))
test_file_strings = [x for x in glob.glob('*_tests.py') if x not in __file__]
module_strings = [file_string[0:len(file_string) - 3] for file_string in test_file_strings]
suites = [unittest.defaultTestLoader.loadTestsFromName(file_string) for file_string in module_strings]

114
tests/ignore_and_require_words_tests.py

@ -3,56 +3,72 @@ import sys
import unittest
import sickbeard
from sickbeard import show_name_helpers
from sickbeard import show_name_helpers, helpers
sys.path.insert(1, os.path.abspath('..'))
class TVShow(object):
def __init__(self, ei=set(), er=set(), i=set(), r=set(), ir=False, rr=False):
self.rls_global_exclude_ignore = ei
self.rls_global_exclude_require = er
self.rls_ignore_words = i
self.rls_ignore_words_regex = ir
self.rls_require_words = r
self.rls_require_words_regex = rr
class TestCase(unittest.TestCase):
cases_pass_wordlist_checks = [
('[GroupName].Show.Name.-.%02d.[null]', '', '', True),
('[GroupName].Show.Name.-.%02d.[ignore]', '', 'required', False),
('[GroupName].Show.Name.-.%02d.[required]', '', 'required', True),
('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', 'GroupName', True),
('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', '[GroupName]', True),
('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', 'Show.Name', True),
('[GroupName].Show.Name.-.%02d.[required]', 'not_ignored', 'required', True),
('[GroupName].Show.Name.-.%02d.[required]', '[not_ignored]', '[required]', True),
('[GroupName].Show.Name.-.%02d.[ignore]', '[ignore]', '', False),
('[GroupName].Show.Name.-.%02d.[required]', '[GroupName]', 'required', False),
('[GroupName].Show.Name.-.%02d.[required]', 'GroupName', 'required', False),
('[GroupName].Show.Name.-.%02d.[ignore]', 'ignore', 'GroupName', False),
('[GroupName].Show.Name.-.%02d.[required]', 'Show.Name', 'required', False),
('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: no_ignore', '', True),
('[GroupName].Show.Name.-.%02d.[480p]', 'ignore', r'regex: \d?\d80p', True),
('[GroupName].Show.Name.-.%02d.[480p]', 'ignore', r'regex: \[\d?\d80p\]', True),
('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: ignore', '', False),
('[GroupName].Show.Name.-.%02d.[ignore]', r'regex: \[ignore\]', '', False),
('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: ignore', 'required', False),
('[GroupName].Show.Name.-.%02d.[null]', '', '', True, TVShow()),
('[GroupName].Show.Name.-.%02d.[ignore]', '', 'required', False, TVShow()),
('[GroupName].Show.Name.-.%02d.[required]', '', 'required', True, TVShow()),
('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', 'GroupName', True, TVShow()),
('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', '[GroupName]', True, TVShow()),
('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', 'Show.Name', True, TVShow()),
('[GroupName].Show.Name.-.%02d.[required]', 'not_ignored', 'required', True, TVShow()),
('[GroupName].Show.Name.-.%02d.[required]', '[not_ignored]', '[required]', True, TVShow()),
('[GroupName].Show.Name.-.%02d.[ignore]', '[ignore]', '', False, TVShow()),
('[GroupName].Show.Name.-.%02d.[required]', '[GroupName]', 'required', False, TVShow()),
('[GroupName].Show.Name.-.%02d.[required]', 'GroupName', 'required', False, TVShow()),
('[GroupName].Show.Name.-.%02d.[ignore]', 'ignore', 'GroupName', False, TVShow()),
('[GroupName].Show.Name.-.%02d.[required]', 'Show.Name', 'required', False, TVShow()),
('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: no_ignore', '', True, TVShow()),
('[GroupName].Show.Name.-.%02d.[480p]', 'ignore', r'regex: \d?\d80p', True, TVShow()),
('[GroupName].Show.Name.-.%02d.[480p]', 'ignore', r'regex: \[\d?\d80p\]', True, TVShow()),
('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: ignore', '', False, TVShow()),
('[GroupName].Show.Name.-.%02d.[ignore]', r'regex: \[ignore\]', '', False, TVShow()),
('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: ignore', 'required', False, TVShow()),
# The following test is True because a boundary is added to each regex not overridden with the prefix param
('[GroupName].Show.ONEONE.-.%02d.[required]', 'regex: (one(two)?)', '', True),
('[GroupName].Show.ONETWO.-.%02d.[required]', 'regex: ((one)?two)', 'required', False),
('[GroupName].Show.TWO.-.%02d.[required]', 'regex: ((one)?two)', 'required', False),
('[GroupName].Show.ONEONE.-.%02d.[required]', 'regex: (one(two)?)', '', True, TVShow()),
('[GroupName].Show.ONETWO.-.%02d.[required]', 'regex: ((one)?two)', 'required', False, TVShow()),
('[GroupName].Show.TWO.-.%02d.[required]', 'regex: ((one)?two)', 'required', False, TVShow()),
('[GroupName].Show.TWO.-.%02d.[required]', '[GroupName]', '', True, TVShow(ei={'[GroupName]'})),
('[GroupName].Show.TWO.-.%02d.[something]', '[GroupName]', 'required', False, TVShow(er={'required'})),
('[GroupName].Show.TWO.-.%02d.[required]-[GroupName]', '', '', False, TVShow(i={'[GroupName]'})),
('[GroupName].Show.TWO.-.%02d.[something]-required', '', '', True, TVShow(r={'required'})),
('The.Spanish.Princess.-.%02d',
r'regex:^(?:(?=.*?\bspanish\b)((?!spanish.?princess).)*|.*princess.*?spanish.*)$, ignore', '', True),
r'regex:^(?:(?=.*?\bspanish\b)((?!spanish.?princess).)*|.*princess.*?spanish.*)$, ignore', '', True, TVShow()),
('Spanish.Princess.Spanish.-.%02d',
r'regex:^(?:(?=.*?\bspanish\b)((?!spanish.?princess).)*|.*princess.*?spanish.*)$, ignore', '', False)
r'regex:^(?:(?=.*?\bspanish\b)((?!spanish.?princess).)*|.*princess.*?spanish.*)$, ignore', '', False, TVShow())
]
cases_contains = [
('[GroupName].Show.Name.-.%02d.[illegal_regex]', 'regex:??illegal_regex', False),
('[GroupName].Show.Name.-.%02d.[illegal_regex]', 'regex:??illegal_regex', None),
('[GroupName].Show.Name.-.%02d.[480p]', 'regex:(480|1080)p', True),
('[GroupName].Show.Name.-.%02d.[contains]', r'regex:\[contains\]', True),
('[GroupName].Show.Name.-.%02d.[contains]', '[contains]', True),
('[GroupName].Show.Name.-.%02d.[contains]', 'contains', True),
('[GroupName].Show.Name.-.%02d.[contains]', '[not_contains]', False),
('[GroupName].Show.Name.-.%02d.[null]', '', False)
('[GroupName].Show.Name.-.%02d.[null]', '', None)
]
cases_not_contains = [
@ -61,7 +77,7 @@ class TestCase(unittest.TestCase):
('[GroupName].Show.Name.-.%02d.[contains]', '[contains]', False),
('[GroupName].Show.Name.-.%02d.[contains]', 'contains', False),
('[GroupName].Show.Name.-.%02d.[not_contains]', '[blah_blah]', True),
('[GroupName].Show.Name.-.%02d.[null]', '', False)
('[GroupName].Show.Name.-.%02d.[null]', '', None)
]
def test_pass_wordlist_checks(self):
@ -69,11 +85,21 @@ class TestCase(unittest.TestCase):
isolated = []
test_cases = (self.cases_pass_wordlist_checks, isolated)[len(isolated)]
for case_num, (name, ignore_list, require_list, expected_result) in enumerate(test_cases):
for case_num, (name, ignore_list, require_list, expected_result, show_obj) in enumerate(test_cases):
name = name if '%02d' not in name else name % case_num
sickbeard.IGNORE_WORDS = ignore_list
sickbeard.REQUIRE_WORDS = require_list
self.assertEqual(expected_result, show_name_helpers.pass_wordlist_checks(name, False),
if ignore_list.startswith('regex:'):
sickbeard.IGNORE_WORDS_REGEX = True
ignore_list = ignore_list.replace('regex:', '')
else:
sickbeard.IGNORE_WORDS_REGEX = False
sickbeard.IGNORE_WORDS = set(i.strip() for i in ignore_list.split(',') if i.strip())
if require_list.startswith('regex:'):
sickbeard.REQUIRE_WORDS_REGEX = True
require_list = require_list.replace('regex:', '')
else:
sickbeard.REQUIRE_WORDS_REGEX = False
sickbeard.REQUIRE_WORDS = set(r.strip() for r in require_list.split(',') if r.strip())
self.assertEqual(expected_result, show_name_helpers.pass_wordlist_checks(name, False, show_obj=show_obj),
'Expected %s with test: "%s" with ignore: "%s", require: "%s"' %
(expected_result, name, ignore_list, require_list))
@ -83,16 +109,17 @@ class TestCase(unittest.TestCase):
test_cases = (self.cases_contains, isolated)[len(isolated)]
for case_num, (name, csv_words, expected_result) in enumerate(test_cases):
s_words, s_regex = helpers.split_word_str(csv_words)
name = name if '%02d' not in name else name % case_num
self.assertEqual(expected_result, self.call_contains_any(name, csv_words),
self.assertEqual(expected_result, self.call_contains_any(name, s_words, rx=s_regex),
'Expected %s test: "%s" with csv_words: "%s"' %
(expected_result, name, csv_words))
@staticmethod
def call_contains_any(name, csv_words):
def call_contains_any(name, csv_words, *args, **kwargs):
re_extras = dict(re_prefix='.*', re_suffix='.*')
match = show_name_helpers.contains_any(name, csv_words, **re_extras)
return None is not match and match
re_extras.update(kwargs)
return show_name_helpers.contains_any(name, csv_words, *args, **re_extras)
def test_not_contains_any(self):
# default:[] or copy in a test case tuple to debug in isolation
@ -100,16 +127,17 @@ class TestCase(unittest.TestCase):
test_cases = (self.cases_not_contains, isolated)[len(isolated)]
for case_num, (name, csv_words, expected_result) in enumerate(test_cases):
s_words, s_regex = helpers.split_word_str(csv_words)
name = name if '%02d' not in name else name % case_num
self.assertEqual(expected_result, self.call_not_contains_any(name, csv_words),
self.assertEqual(expected_result, self.call_not_contains_any(name, s_words, rx=s_regex),
'Expected %s test: "%s" with csv_words:"%s"' %
(expected_result, name, csv_words))
@staticmethod
def call_not_contains_any(name, csv_words):
def call_not_contains_any(name, csv_words, *args, **kwargs):
re_extras = dict(re_prefix='.*', re_suffix='.*')
match = show_name_helpers.not_contains_any(name, csv_words, **re_extras)
return None is not match and match
re_extras.update(kwargs)
return show_name_helpers.not_contains_any(name, csv_words, *args, **re_extras)
if '__main__' == __name__:

Loading…
Cancel
Save