From c1d3487a214c7956949e00c8404c194c8a28c47b Mon Sep 17 00:00:00 2001 From: Prinz23 Date: Wed, 11 Apr 2018 15:13:20 +0100 Subject: [PATCH] Change increase namecache size and fix deleting items from it when at capacity. Add thread lock to NameParserCache. Add exception handling for add to cache. Change increase NameParserCache to 1000 entries. Change NameParserCache to OrderedDefaultdict. Add move_to_end to OrderedDefaultdict class (backported from python 3). Add first_key, last_key to OrderedDefaultdict. Change use much faster first_key in NameParserCache. Change remove import of python pre 2.7 OrderedDict. --- CHANGES.md | 1 + sickbeard/classes.py | 43 +++++++++++++++++++++++++++++++-------- sickbeard/databases/cache_db.py | 3 ++- sickbeard/name_parser/parser.py | 33 +++++++++++++++++++++--------- sickbeard/providers/blutopia.py | 6 ++---- sickbeard/providers/fano.py | 6 ++---- sickbeard/providers/hdspace.py | 6 ++---- sickbeard/providers/hdtorrents.py | 6 ++---- sickbeard/providers/newznab.py | 37 ++++++++++++++++----------------- sickbeard/providers/privatehd.py | 6 ++---- sickbeard/providers/ptf.py | 6 ++---- 11 files changed, 90 insertions(+), 63 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 73ae759..1146f16 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ ### 0.16.0 (2018-xx-xx xx:xx:xx UTC) +* Change increase namecache size and fix deleting items from it when at capacity * Change improve security with cross-site request forgery (xsrf) protection on web forms * Change improve security by sending header flags httponly and secure with cookies * Change improve security with DNS rebinding prevention, set "Allowed browser hostnames" at config/General/Web Interface diff --git a/sickbeard/classes.py b/sickbeard/classes.py index aef64b9..9ea85d8 100644 --- a/sickbeard/classes.py +++ b/sickbeard/classes.py @@ -15,18 +15,16 @@ # # You should have received a copy of the GNU General Public License # along with SickGear. If not, see . -import re -import datetime -import os +from collections import OrderedDict -import sickbeard from sickbeard.common import Quality from unidecode import unidecode -try: - from collections import OrderedDict -except ImportError: - from requests.compat import OrderedDict +import datetime +import os +import re + +import sickbeard class SearchResult(object): @@ -343,6 +341,35 @@ class OrderedDefaultdict(OrderedDict): args = (self.default_factory,) if self.default_factory else () return self.__class__, args, None, None, self.iteritems() + # backport from python 3 + def move_to_end(self, key, last=True): + """Move an existing element to the end (or beginning if last==False). + + Raises KeyError if the element does not exist. + When last=True, acts like a fast version of self[key]=self.pop(key). + + """ + link_prev, link_next, key = link = self._OrderedDict__map[key] + link_prev[1] = link_next + link_next[0] = link_prev + root = self._OrderedDict__root + if last: + last = root[0] + link[0] = last + link[1] = root + last[1] = root[0] = link + else: + first = root[1] + link[0] = root + link[1] = first + root[1] = first[0] = link + + def first_key(self): + return self._OrderedDict__root[1][2] + + def last_key(self): + return self._OrderedDict__root[0][2] + class ImageUrlList(list): def __init__(self, max_age=30): diff --git a/sickbeard/databases/cache_db.py b/sickbeard/databases/cache_db.py index de3dc25..a890f43 100644 --- a/sickbeard/databases/cache_db.py +++ b/sickbeard/databases/cache_db.py @@ -16,8 +16,9 @@ # You should have received a copy of the GNU General Public License # along with SickGear. If not, see . -from sickbeard import db from collections import OrderedDict +from sickbeard import db + import re MIN_DB_VERSION = 1 diff --git a/sickbeard/name_parser/parser.py b/sickbeard/name_parser/parser.py index 97209c5..7fabdfa 100644 --- a/sickbeard/name_parser/parser.py +++ b/sickbeard/name_parser/parser.py @@ -23,6 +23,7 @@ import os import os.path import re import time +import threading import regexes import sickbeard @@ -35,6 +36,7 @@ except ImportError: from sickbeard import logger, helpers, scene_numbering, common, scene_exceptions, encodingKludge as ek, db from sickbeard.exceptions import ex +from sickbeard.classes import OrderedDefaultdict class NameParser(object): @@ -706,20 +708,31 @@ class ParseResult(object): class NameParserCache(object): - _previous_parsed = {} - _cache_size = 100 + def __init__(self): + super(NameParserCache, self).__init__() + self._previous_parsed = OrderedDefaultdict() + self._cache_size = 1000 + self.lock = threading.Lock() def add(self, name, parse_result): - self._previous_parsed[name] = parse_result - _current_cache_size = len(self._previous_parsed) - if _current_cache_size > self._cache_size: - for i in range(_current_cache_size - self._cache_size): - del self._previous_parsed[self._previous_parsed.keys()[0]] + with self.lock: + self._previous_parsed[name] = parse_result + _current_cache_size = len(self._previous_parsed) + if _current_cache_size > self._cache_size: + key = None + for i in range(_current_cache_size - self._cache_size): + try: + key = self._previous_parsed.first_key() + del self._previous_parsed[key] + except KeyError: + logger.log('Could not remove old NameParserCache entry: %s' % key, logger.DEBUG) def get(self, name): - if name in self._previous_parsed: - logger.log('Using cached parse result for: ' + name, logger.DEBUG) - return self._previous_parsed[name] + with self.lock: + if name in self._previous_parsed: + logger.log('Using cached parse result for: ' + name, logger.DEBUG) + self._previous_parsed.move_to_end(name) + return self._previous_parsed[name] name_parser_cache = NameParserCache() diff --git a/sickbeard/providers/blutopia.py b/sickbeard/providers/blutopia.py index 5639c93..69bfe25 100644 --- a/sickbeard/providers/blutopia.py +++ b/sickbeard/providers/blutopia.py @@ -15,10 +15,8 @@ # You should have received a copy of the GNU General Public License # along with SickGear. If not, see . -try: - from collections import OrderedDict -except ImportError: - from requests.compat import OrderedDict +from collections import OrderedDict + import re import traceback diff --git a/sickbeard/providers/fano.py b/sickbeard/providers/fano.py index a1e9489..d3ef039 100644 --- a/sickbeard/providers/fano.py +++ b/sickbeard/providers/fano.py @@ -15,10 +15,8 @@ # You should have received a copy of the GNU General Public License # along with SickGear. If not, see . -try: - from collections import OrderedDict -except ImportError: - from requests.compat import OrderedDict +from collections import OrderedDict + import re import traceback diff --git a/sickbeard/providers/hdspace.py b/sickbeard/providers/hdspace.py index f6e63ff..9df1261 100644 --- a/sickbeard/providers/hdspace.py +++ b/sickbeard/providers/hdspace.py @@ -15,10 +15,8 @@ # You should have received a copy of the GNU General Public License # along with SickGear. If not, see . -try: - from collections import OrderedDict -except ImportError: - from requests.compat import OrderedDict +from collections import OrderedDict + import re import traceback diff --git a/sickbeard/providers/hdtorrents.py b/sickbeard/providers/hdtorrents.py index ec8049d..0c8e738 100644 --- a/sickbeard/providers/hdtorrents.py +++ b/sickbeard/providers/hdtorrents.py @@ -15,10 +15,8 @@ # You should have received a copy of the GNU General Public License # along with SickGear. If not, see . -try: - from collections import OrderedDict -except ImportError: - from requests.compat import OrderedDict +from collections import OrderedDict + import re import traceback diff --git a/sickbeard/providers/newznab.py b/sickbeard/providers/newznab.py index 664ea21..2024572 100755 --- a/sickbeard/providers/newznab.py +++ b/sickbeard/providers/newznab.py @@ -17,26 +17,28 @@ # along with SickGear. If not, see . from __future__ import division +from collections import OrderedDict +from math import ceil -import time -import sickbeard import datetime import re +import time import urllib -from math import ceil -from sickbeard.sbdatetime import sbdatetime +import sickbeard + +from io import BytesIO +from lib.dateutil import parser from . import generic -from sickbeard import helpers, logger, tvcache, classes, db -from sickbeard.common import neededQualities, Quality, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, DOWNLOADED +from sickbeard import classes, db, helpers, logger, tvcache +from sickbeard.common import neededQualities, Quality, DOWNLOADED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST from sickbeard.exceptions import AuthException, MultipleShowObjectsException +from sickbeard.helpers import tryInt from sickbeard.indexers.indexer_config import * -from io import BytesIO -from lib.dateutil import parser from sickbeard.network_timezones import sb_timezone -from sickbeard.helpers import tryInt +from sickbeard.sbdatetime import sbdatetime +from sickbeard.search import get_aired_in_season, get_wanted_qualities from sickbeard.show_name_helpers import get_show_names -from sickbeard.search import get_wanted_qualities, get_aired_in_season try: from lxml import etree @@ -46,11 +48,6 @@ except ImportError: except ImportError: import xml.etree.ElementTree as etree -try: - from collections import OrderedDict -except ImportError: - from requests.compat import OrderedDict - class NewznabConstants: SEARCH_TEXT = -100 @@ -73,7 +70,7 @@ class NewznabConstants: r'^BoxHD$': CAT_HD, r'^UHD$': CAT_UHD, r'^4K$': CAT_UHD, - #r'^HEVC$': CAT_HEVC, + # r'^HEVC$': CAT_HEVC, r'^WEB.?DL$': CAT_WEBDL} providerToIndexerMapping = {'tvdbid': INDEXER_TVDB, @@ -101,8 +98,8 @@ class NewznabConstants: class NewznabProvider(generic.NZBProvider): - def __init__(self, name, url, key='', cat_ids=None, search_mode=None, - search_fallback=False, enable_recentsearch=False, enable_backlog=False, enable_scheduled_backlog=False): + def __init__(self, name, url, key='', cat_ids=None, search_mode=None, search_fallback=False, + enable_recentsearch=False, enable_backlog=False, enable_scheduled_backlog=False): generic.NZBProvider.__init__(self, name, True, False) self.url = url @@ -235,8 +232,8 @@ class NewznabProvider(generic.NZBProvider): limit = xml_caps.find('.//limits') if None is not limit: - l = helpers.tryInt(limit.get('max'), 100) - self._limits = (100, l)[l >= 100] + lim = helpers.tryInt(limit.get('max'), 100) + self._limits = (100, lim)[lim >= 100] try: for category in xml_caps.iter('category'): diff --git a/sickbeard/providers/privatehd.py b/sickbeard/providers/privatehd.py index 5360b0c..86f8080 100644 --- a/sickbeard/providers/privatehd.py +++ b/sickbeard/providers/privatehd.py @@ -15,10 +15,8 @@ # You should have received a copy of the GNU General Public License # along with SickGear. If not, see . -try: - from collections import OrderedDict -except ImportError: - from requests.compat import OrderedDict +from collections import OrderedDict + import re import traceback diff --git a/sickbeard/providers/ptf.py b/sickbeard/providers/ptf.py index 9a76c18..d59c644 100644 --- a/sickbeard/providers/ptf.py +++ b/sickbeard/providers/ptf.py @@ -15,10 +15,8 @@ # You should have received a copy of the GNU General Public License # along with SickGear. If not, see . -try: - from collections import OrderedDict -except ImportError: - from requests.compat import OrderedDict +from collections import OrderedDict + import re import time import traceback