dockeruser
/
SickGear
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Merge branch 'feature/ChangeImproveSceneCache' into develop

pull/1289/head
JackDandy 5 years ago
parent
1729535f33 b0a3e01660
commit
2d6dcfbf9c
11 changed files with 67 additions and 52 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      CHANGES.md
  2. 27
      sickbeard/helpers.py
  3. 65
      sickbeard/name_cache.py
  4. 4
      sickbeard/name_parser/parser.py
  5. 2
      sickbeard/postProcessor.py
  6. 4
      sickbeard/processTV.py
  7. 5
      sickbeard/properFinder.py
  8. 5
      sickbeard/scene_exceptions.py
  9. 2
      sickbeard/show_queue.py
  10. 2
      sickbeard/webserve.py
  11. 2
      tests/common_tests.py

1
CHANGES.md
View File

@ -17,6 +17,7 @@
* Update Tornado_py3 Web Server 6.0.3 (ff985fe) to 6.0.xx (18b653c) * Update Tornado_py3 Web Server 6.0.3 (ff985fe) to 6.0.xx (18b653c)
* Update urllib3 release 1.25.6 (4a6c288) to 1.25.7 (37ba61a) * Update urllib3 release 1.25.6 (4a6c288) to 1.25.7 (37ba61a)
* Add Telegram notifier * Add Telegram notifier
* Change update sceneNameCache after scene names are updated
[develop changelog] [develop changelog]

27
sickbeard/helpers.py
View File

@ -68,6 +68,9 @@ from sg_helpers import chmod_as_parent, clean_data, get_system_temp_dir, \
if False: if False:
# noinspection PyUnresolvedReferences # noinspection PyUnresolvedReferences
from typing import Any, AnyStr, Dict, NoReturn, Iterable, Iterator, List, Optional, Tuple, Union from typing import Any, AnyStr, Dict, NoReturn, Iterable, Iterator, List, Optional, Tuple, Union
from .tv import TVShow
# the following workaround hack resolves a pyc resolution bug
from .name_cache import retrieveNameFromCache
RE_XML_ENCODING = re.compile(r'^(<\?xml[^>]+)\s+(encoding\s*=\s*[\"\'][^\"\']*[\"\'])(\s*\?>|)', re.U) RE_XML_ENCODING = re.compile(r'^(<\?xml[^>]+)\s+(encoding\s*=\s*[\"\'][^\"\']*[\"\'])(\s*\?>|)', re.U)
@ -272,10 +275,10 @@ def remove_file(filepath, tree=False, prefix_failure='', log_level=logger.MESSAG
def find_show_by_id( def find_show_by_id(
show_id, # type: Union[AnyStr, Dict[int, int], int] show_id, # type: Union[AnyStr, Dict[int, int], int]
show_list=None, # type: Optional[List[sickbeard.tv.TVShow]] show_list=None, # type: Optional[List[TVShow]]
no_mapped_ids=True # type: bool no_mapped_ids=True # type: bool
): ):
# type: (...) -> sickbeard.tv.TVShow or MultipleShowObjectsException # type: (...) -> TVShow or MultipleShowObjectsException
""" """
:param show_id: {indexer: id} or 'tvid_prodid'. :param show_id: {indexer: id} or 'tvid_prodid'.
:param show_list: (optional) TVShow objects list :param show_list: (optional) TVShow objects list
@ -681,7 +684,7 @@ def get_absolute_number_from_season_and_episode(show_obj, season, episode):
""" """
:param show_obj: show object :param show_obj: show object
:type show_obj: sickbeard.tv.TVShow :type show_obj: TVShow
:param season: season number :param season: season number
:type season: int :type season: int
:param episode: episode number :param episode: episode number
@ -713,7 +716,7 @@ def get_absolute_number_from_season_and_episode(show_obj, season, episode):
def get_all_episodes_from_absolute_number(show_obj, absolute_numbers): def get_all_episodes_from_absolute_number(show_obj, absolute_numbers):
# type: (sickbeard.tv.TVShow, List[int]) -> Tuple[int, List[int]] # type: (TVShow, List[int]) -> Tuple[int, List[int]]
""" """
:param show_obj: show object :param show_obj: show object
@ -1097,7 +1100,8 @@ def full_sanitize_scene_name(name):
return re.sub('[. -]', ' ', sanitize_scene_name(name)).lower().lstrip() return re.sub('[. -]', ' ', sanitize_scene_name(name)).lower().lstrip()
def get_show(name, try_scene_exceptions=False, use_cache=True): def get_show(name, try_scene_exceptions=False):
# type: (AnyStr, bool) -> Optional[TVShow]
""" """
get show object for show with given name get show object for show with given name
@ -1105,32 +1109,23 @@ def get_show(name, try_scene_exceptions=False, use_cache=True):
:type name: AnyStr :type name: AnyStr
:param try_scene_exceptions: check scene exceptions :param try_scene_exceptions: check scene exceptions
:type try_scene_exceptions: bool :type try_scene_exceptions: bool
:param use_cache: use cache
:type use_cache: bool
:return: None or show object :return: None or show object
:type: sickbeard.tv.TVShow or None :type: TVShow or None
""" """
if not sickbeard.showList or None is name: if not sickbeard.showList or None is name:
return return
show_obj = None show_obj = None
from_cache = False
try: try:
tvid, prodid = sickbeard.name_cache.retrieveNameFromCache(name) tvid, prodid = sickbeard.name_cache.retrieveNameFromCache(name)
if tvid and prodid: if tvid and prodid:
from_cache = True
show_obj = find_show_by_id({tvid: prodid}) show_obj = find_show_by_id({tvid: prodid})
if not show_obj and try_scene_exceptions: if not show_obj and try_scene_exceptions:
tvid, prodid, season = sickbeard.scene_exceptions.get_scene_exception_by_name(name) tvid, prodid, season = sickbeard.scene_exceptions.get_scene_exception_by_name(name)
if tvid and prodid: if tvid and prodid:
show_obj = find_show_by_id({tvid: prodid}) show_obj = find_show_by_id({tvid: prodid})
# add show to cache
if use_cache and show_obj and not from_cache:
from sickbeard.name_cache import addNameToCache
sickbeard.name_cache.addNameToCache(name, tvid=show_obj.tvid, prodid=show_obj.prodid)
except (BaseException, Exception) as e: except (BaseException, Exception) as e:
logger.log(u'Error when attempting to find show: ' + name + ' in SickGear: ' + ex(e), logger.DEBUG) logger.log(u'Error when attempting to find show: ' + name + ' in SickGear: ' + ex(e), logger.DEBUG)
@ -1169,7 +1164,7 @@ def validate_show(show_obj, season=None, episode=None):
""" """
:param show_obj: show object :param show_obj: show object
:type show_obj: sickbeard.tv.TVShow :type show_obj: TVShow
:param season: optional season :param season: optional season
:type season: int or None :type season: int or None
:param episode: opitonal episode :param episode: opitonal episode

65
sickbeard/name_cache.py
View File

@ -20,7 +20,7 @@ import threading
import sickbeard import sickbeard
from . import db from . import db
from .helpers import try_int from .helpers import full_sanitize_scene_name, try_int
from six import iteritems from six import iteritems
@ -30,6 +30,7 @@ if False:
from .tv import TVShow, TVShowBase from .tv import TVShow, TVShowBase
nameCache = {} nameCache = {}
sceneNameCache = {}
nameCacheLock = threading.Lock() nameCacheLock = threading.Lock()
@ -49,7 +50,7 @@ def addNameToCache(name, tvid=0, prodid=0, season=-1):
with nameCacheLock: with nameCacheLock:
# standardize the name we're using to account for small differences in providers # standardize the name we're using to account for small differences in providers
name = sickbeard.helpers.full_sanitize_scene_name(name) name = full_sanitize_scene_name(name)
if name not in nameCache: if name not in nameCache:
nameCache[name] = [int(tvid), int(prodid), season] nameCache[name] = [int(tvid), int(prodid), season]
@ -63,7 +64,7 @@ def retrieveNameFromCache(name):
""" """
global nameCache global nameCache
name = sickbeard.helpers.full_sanitize_scene_name(name) name = full_sanitize_scene_name(name)
try: try:
if name in nameCache: if name in nameCache:
return int(nameCache[name][0]), int(nameCache[name][1]) return int(nameCache[name][0]), int(nameCache[name][1])
@ -72,37 +73,51 @@ def retrieveNameFromCache(name):
return None, None return None, None
def buildNameCache(show_obj=None): def buildNameCache(show_obj=None, update_only_scene=False):
# type: (Optional[Union[TVShow, TVShowBase]]) -> None # type: (Optional[Union[TVShow, TVShowBase]], bool) -> None
"""Adds all new name exceptions to the namecache memory and flushes any removed name exceptions """Adds all new name exceptions to the namecache memory and flushes any removed name exceptions
:param show_obj : Only update name cache for this show object, otherwise update all :param show_obj : Only update name cache for this show object, otherwise update all
:param update_only_scene: (optional) only update scene name cache
""" """
global nameCache global nameCache, sceneNameCache
with nameCacheLock: with nameCacheLock:
if show_obj: if not update_only_scene:
# search for only the requested show id and flush old show entries from namecache if show_obj:
show_ids = {show_obj.tvid: [show_obj.prodid]} # search for only the requested show id and flush old show entries from namecache
nameCache = dict([(k, v) for k, v in iteritems(nameCache) show_ids = {show_obj.tvid: [show_obj.prodid]}
if not (v[0] == show_obj.tvid and v[1] == show_obj.prodid)]) nameCache = dict([(k, v) for k, v in iteritems(nameCache)
if not (v[0] == show_obj.tvid and v[1] == show_obj.prodid)])
sceneNameCache = dict([(k, v) for k, v in iteritems(sceneNameCache)
if not (v[0] == show_obj.tvid and v[1] == show_obj.prodid)])
# add standard indexer name to namecache
nameCache[full_sanitize_scene_name(show_obj.name)] = [show_obj.tvid, show_obj.prodid, -1]
else:
# generate list of production ids to look up in cache.db
show_ids = {}
for cur_show_obj in sickbeard.showList:
show_ids.setdefault(cur_show_obj.tvid, []).append(cur_show_obj.prodid)
# add all standard show indexer names to namecache
nameCache = dict(
[(full_sanitize_scene_name(cur_so.name), [cur_so.tvid, cur_so.prodid, -1])
for cur_so in sickbeard.showList if cur_so])
sceneNameCache = {}
# add standard indexer name to namecache cacheDB = db.DBConnection()
nameCache[sickbeard.helpers.full_sanitize_scene_name(show_obj.name)] = [show_obj.tvid, show_obj.prodid, -1]
else: cache_results = []
if update_only_scene:
# generate list of production ids to look up in cache.db # generate list of production ids to look up in cache.db
show_ids = {} show_ids = {}
for cur_show_obj in sickbeard.showList: for cur_show_obj in sickbeard.showList:
show_ids.setdefault(cur_show_obj.tvid, []).append(cur_show_obj.prodid) show_ids.setdefault(cur_show_obj.tvid, []).append(cur_show_obj.prodid)
tmp_scene_name_cache = {}
else:
tmp_scene_name_cache = sceneNameCache.copy()
# add all standard show indexer names to namecache
nameCache = dict(
[(sickbeard.helpers.full_sanitize_scene_name(cur_so.name), [cur_so.tvid, cur_so.prodid, -1])
for cur_so in sickbeard.showList if cur_so])
cacheDB = db.DBConnection()
cache_results = []
for t, s in iteritems(show_ids): for t, s in iteritems(show_ids):
cache_results += cacheDB.select( cache_results += cacheDB.select(
'SELECT show_name, indexer AS tv_id, indexer_id AS prod_id, season' 'SELECT show_name, indexer AS tv_id, indexer_id AS prod_id, season'
@ -114,8 +129,10 @@ def buildNameCache(show_obj=None):
tvid = int(cache_result['tv_id']) tvid = int(cache_result['tv_id'])
prodid = int(cache_result['prod_id']) prodid = int(cache_result['prod_id'])
season = try_int(cache_result['season'], -1) season = try_int(cache_result['season'], -1)
name = sickbeard.helpers.full_sanitize_scene_name(cache_result['show_name']) name = full_sanitize_scene_name(cache_result['show_name'])
nameCache[name] = [tvid, prodid, season] tmp_scene_name_cache[name] = [tvid, prodid, season]
sceneNameCache = tmp_scene_name_cache
def remove_from_namecache(tvid, prodid): def remove_from_namecache(tvid, prodid):

4
sickbeard/name_parser/parser.py
View File

@ -56,7 +56,7 @@ class NameParser(object):
NORMAL_REGEX = 1 NORMAL_REGEX = 1
ANIME_REGEX = 2 ANIME_REGEX = 2
def __init__(self, file_name=True, show_obj=None, try_scene_exceptions=False, convert=False, def __init__(self, file_name=True, show_obj=None, try_scene_exceptions=True, convert=False,
naming_pattern=False, testing=False, indexer_lookup=True): naming_pattern=False, testing=False, indexer_lookup=True):
self.file_name = file_name # type: bool self.file_name = file_name # type: bool
@ -212,7 +212,7 @@ class NameParser(object):
if hasattr(self.show_obj, 'get_episode'): if hasattr(self.show_obj, 'get_episode'):
_ep_obj = self.show_obj.get_episode(parse_result.season_number, ep_num) _ep_obj = self.show_obj.get_episode(parse_result.season_number, ep_num)
else: else:
tmp_show_obj = helpers.get_show(parse_result.series_name, True, False) tmp_show_obj = helpers.get_show(parse_result.series_name, True)
if tmp_show_obj and hasattr(tmp_show_obj, 'get_episode'): if tmp_show_obj and hasattr(tmp_show_obj, 'get_episode'):
_ep_obj = tmp_show_obj.get_episode(parse_result.season_number, ep_num) _ep_obj = tmp_show_obj.get_episode(parse_result.season_number, ep_num)
else: else:

2
sickbeard/postProcessor.py
View File

@ -559,7 +559,7 @@ class PostProcessor(object):
return to_return return to_return
# parse the name to break it into show name, season, and episode # parse the name to break it into show name, season, and episode
np = NameParser(resource, try_scene_exceptions=True, convert=True, show_obj=self.show_obj or show_obj) np = NameParser(resource, convert=True, show_obj=self.show_obj or show_obj)
parse_result = np.parse(name) parse_result = np.parse(name)
self._log(u'Parsed %s<br />.. from %s' self._log(u'Parsed %s<br />.. from %s'
% (decode_str(str(parse_result), errors='xmlcharrefreplace'), name), logger.DEBUG) % (decode_str(str(parse_result), errors='xmlcharrefreplace'), name), logger.DEBUG)

4
sickbeard/processTV.py
View File

@ -934,13 +934,13 @@ class ProcessTVShow(object):
parse_result = None parse_result = None
try: try:
parse_result = NameParser(try_scene_exceptions=True, convert=True).parse(videofile, cache_result=False) parse_result = NameParser(convert=True).parse(videofile, cache_result=False)
except (InvalidNameException, InvalidShowException): except (InvalidNameException, InvalidShowException):
# Does not parse, move on to directory check # Does not parse, move on to directory check
pass pass
if None is parse_result: if None is parse_result:
try: try:
parse_result = NameParser(try_scene_exceptions=True, convert=True).parse(dir_name, cache_result=False) parse_result = NameParser(convert=True).parse(dir_name, cache_result=False)
except (InvalidNameException, InvalidShowException): except (InvalidNameException, InvalidShowException):
# If the filename doesn't parse, then return false as last # If the filename doesn't parse, then return false as last
# resort. We can assume that unparseable filenames are not # resort. We can assume that unparseable filenames are not

5
sickbeard/properFinder.py
View File

@ -266,8 +266,7 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_dict=
continue continue
try: try:
np = NameParser(False, try_scene_exceptions=True, show_obj=cur_proper.parsed_show_obj, np = NameParser(False, show_obj=cur_proper.parsed_show_obj, indexer_lookup=False)
indexer_lookup=False)
parse_result = np.parse(cur_proper.name) parse_result = np.parse(cur_proper.name)
except (InvalidNameException, InvalidShowException, Exception): except (InvalidNameException, InvalidShowException, Exception):
continue continue
@ -346,7 +345,7 @@ def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_dict=
or (cur_proper.is_repack and not same_release_group): or (cur_proper.is_repack and not same_release_group):
continue continue
np = NameParser(False, try_scene_exceptions=True, show_obj=cur_proper.parsed_show_obj, indexer_lookup=False) np = NameParser(False, show_obj=cur_proper.parsed_show_obj, indexer_lookup=False)
try: try:
extra_info = np.parse(sql_result[0]['release_name']).extra_info_no_name() extra_info = np.parse(sql_result[0]['release_name']).extra_info_no_name()
except (BaseException, Exception): except (BaseException, Exception):

5
sickbeard/scene_exceptions.py
View File

@ -194,7 +194,7 @@ def get_scene_exception_by_name_multiple(show_name):
:rtype: Tuple[None, None, None] or Tuple[int, int or long, int] :rtype: Tuple[None, None, None] or Tuple[int, int or long, int]
""" """
try: try:
exception_result = name_cache.nameCache[helpers.full_sanitize_scene_name(show_name)] exception_result = name_cache.sceneNameCache[helpers.full_sanitize_scene_name(show_name)]
return [exception_result] return [exception_result]
except (BaseException, Exception): except (BaseException, Exception):
return [[None, None, None]] return [[None, None, None]]
@ -294,6 +294,7 @@ def retrieve_exceptions():
if cl: if cl:
my_db.mass_action(cl) my_db.mass_action(cl)
name_cache.buildNameCache(update_only_scene=True)
# since this could invalidate the results of the cache we clear it out after updating # since this could invalidate the results of the cache we clear it out after updating
if changed_exceptions: if changed_exceptions:
@ -339,6 +340,8 @@ def update_scene_exceptions(tvid, prodid, scene_exceptions):
' (indexer, indexer_id, show_name, season) VALUES (?,?,?,?)', ' (indexer, indexer_id, show_name, season) VALUES (?,?,?,?)',
[tvid, prodid, cur_exception, cur_season]) [tvid, prodid, cur_exception, cur_season])
sickbeard.name_cache.buildNameCache(update_only_scene=True)
def _anidb_exceptions_fetcher(): def _anidb_exceptions_fetcher():
global anidb_exception_dict global anidb_exception_dict

2
sickbeard/show_queue.py
View File

@ -838,7 +838,7 @@ class QueueItemAdd(ShowQueueItem):
if oh: if oh:
found = False found = False
for o in oh: for o in oh:
np = NameParser(file_name=True, indexer_lookup=False, try_scene_exceptions=True) np = NameParser(file_name=True, indexer_lookup=False)
try: try:
pr = np.parse(o['resource']) pr = np.parse(o['resource'])
except (BaseException, Exception): except (BaseException, Exception):

2
sickbeard/webserve.py
View File

@ -2655,7 +2655,7 @@ class Home(MainHandler):
if do_update_exceptions: if do_update_exceptions:
try: try:
scene_exceptions.update_scene_exceptions(show_obj.tvid, show_obj.prodid, exceptions_list) scene_exceptions.update_scene_exceptions(show_obj.tvid, show_obj.prodid, exceptions_list)
buildNameCache(show_obj) # buildNameCache(show_obj)
helpers.cpu_sleep() helpers.cpu_sleep()
except exceptions_helper.CantUpdateException: except exceptions_helper.CantUpdateException:
errors.append('Unable to force an update on scene exceptions of the show.') errors.append('Unable to force an update on scene exceptions of the show.')

2
tests/common_tests.py
View File

@ -140,7 +140,7 @@ class QualityTests(unittest.TestCase):
(Quality.qualityStrings[quality], Quality.qualityStrings[second], fn)) (Quality.qualityStrings[quality], Quality.qualityStrings[second], fn))
def check_proper_level(self, cases, is_anime=False): def check_proper_level(self, cases, is_anime=False):
np = NameParser(False, indexer_lookup=False, try_scene_exceptions=False, testing=True) np = NameParser(False, indexer_lookup=False, testing=True)
for case, level in cases: for case, level in cases:
p = np.parse(case) p = np.parse(case)
second = common.Quality.get_proper_level(p.extra_info_no_name(), p.version, is_anime) second = common.Quality.get_proper_level(p.extra_info_no_name(), p.version, is_anime)

Write Preview
Loading…
Cancel
Save