diff --git a/CHANGES.md b/CHANGES.md index 83bca39..82f2028 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -57,7 +57,8 @@ * Update urllib3 release 1.25.6 (4a6c288) to 1.25.7 (37ba61a) * Fix import in Tornado py2 * Fix removed unneeded import - +* Change add exclude_no_data to get_url, in most cases `no_data` is not a failure +* Change prevent log sources containing `sg.` from clipping in log format ### 0.21.30 (2020-04-30 10:20:00 UTC) diff --git a/lib/plex/plex.py b/lib/plex/plex.py index 1b6472c..35170cc 100644 --- a/lib/plex/plex.py +++ b/lib/plex/plex.py @@ -118,7 +118,7 @@ class Plex(object): 'X-Plex-Username': user }, parse_json=True, - failure_handling=False, + failure_monitor=False, post_data=urlencode({b'user[login]': user, b'user[password]': passw}).encode('utf-8') )['user']['authentication_token'] except TypeError: @@ -199,7 +199,7 @@ class Plex(object): } if self.username: headers.update({'X-Plex-Username': self.username}) - page = get_url(url, headers=headers, failure_handling=False, **kwargs) + page = get_url(url, headers=headers, failure_monitor=False, **kwargs) if page: parsed = parse_xml(page) if None is not parsed and len(parsed): diff --git a/lib/sg_helpers.py b/lib/sg_helpers.py index 91656b4..2beace7 100644 --- a/lib/sg_helpers.py +++ b/lib/sg_helpers.py @@ -687,11 +687,12 @@ def get_url(url, # type: AnyStr raise_exceptions=False, # type: bool as_binary=False, # type: bool encoding=None, # type: Optional[AnyStr] - failure_handling=True, # type: bool + failure_monitor=True, # type: bool use_tmr_limit=True, # type: bool raise_skip_exception=False, # type: bool exclude_client_http_codes=True, # type: bool exclude_http_codes=(404, 429), # type: Tuple[integer_types] + exclude_no_data=True, # type: bool **kwargs): # type: (...) -> Optional[Union[AnyStr, bool, bytes, Dict, Tuple[Union[Dict, List], requests.Session]]] """ @@ -715,17 +716,18 @@ def get_url(url, # type: AnyStr :param raise_exceptions: raise exceptions :param as_binary: return bytes instead of text :param encoding: overwrite encoding return header if as_binary is False - :param failure_handling: if True, will enable failure handling for this request + :param failure_monitor: if True, will enable failure monitor for this request :param use_tmr_limit: an API limit can be +ve before a fetch, but unwanted, set False to short should_skip :param raise_skip_exception: if True, will raise ConnectionSkipException if this request should be skipped - :param exclude_client_http_codes: if True, exclude client http codes 4XX from failure handling - :param exclude_http_codes: http codes to exclude from failure handling, default: (404, 429) + :param exclude_client_http_codes: if True, exclude client http codes 4XX from failure monitor + :param exclude_http_codes: http codes to exclude from failure monitor, default: (404, 429) + :param exclude_no_data: exclude no data as failure :param kwargs: keyword params to passthru to Requests :return: None or data fetched from address """ domain = None - if failure_handling: + if failure_monitor: domain = DOMAIN_FAILURES.get_domain(url) if domain not in DOMAIN_FAILURES.domain_list: DOMAIN_FAILURES.domain_list[domain] = ConnectionFailList(domain) @@ -852,7 +854,7 @@ def get_url(url, # type: AnyStr except requests.exceptions.HTTPError as e: raised = e is_client_error = 400 <= e.response.status_code < 500 - if failure_handling and e.response.status_code not in exclude_http_codes and \ + if failure_monitor and e.response.status_code not in exclude_http_codes and \ not (exclude_client_http_codes and is_client_error): connection_fail_params = dict(fail_type=ConnectionFailTypes.http, code=e.response.status_code) if not raise_status_code: @@ -861,26 +863,26 @@ def get_url(url, # type: AnyStr raised = e if 'mute_connect_err' not in mute: logger.warning(u'Connection error msg:%s while loading URL%s' % (ex(e), _maybe_request_url(e))) - if failure_handling: + if failure_monitor: connection_fail_params = dict(fail_type=ConnectionFailTypes.connection) except requests.exceptions.ReadTimeout as e: raised = e if 'mute_read_timeout' not in mute: logger.warning(u'Read timed out msg:%s while loading URL%s' % (ex(e), _maybe_request_url(e))) - if failure_handling: + if failure_monitor: connection_fail_params = dict(fail_type=ConnectionFailTypes.timeout) except (requests.exceptions.Timeout, socket.timeout) as e: raised = e if 'mute_connect_timeout' not in mute: logger.warning(u'Connection timed out msg:%s while loading URL %s' % (ex(e), _maybe_request_url(e, url))) - if failure_handling: + if failure_monitor: connection_fail_params = dict(fail_type=ConnectionFailTypes.connection_timeout) except (BaseException, Exception) as e: raised = e logger.warning((u'Exception caught while loading URL {0}\r\nDetail... %s\r\n{1}' % ex(e), u'Unknown exception while loading URL {0}\r\nDetail... {1}')[not ex(e)] .format(url, traceback.format_exc())) - if failure_handling: + if failure_monitor: connection_fail_params = dict(fail_type=ConnectionFailTypes.other) log_failure_url = True finally: @@ -919,7 +921,7 @@ def get_url(url, # type: AnyStr if raise_exceptions and isinstance(raised, Exception): raise raised - if failure_handling: + if failure_monitor: if result and not isinstance(result, tuple) \ or isinstance(result, tuple) and result[0]: domain = DOMAIN_FAILURES.get_domain(url) @@ -928,7 +930,7 @@ def get_url(url, # type: AnyStr DOMAIN_FAILURES.domain_list[domain].failure_count = 0 DOMAIN_FAILURES.domain_list[domain].failure_time = None save_failure(url, domain, False, post_data, post_json) - else: + elif not exclude_no_data: DOMAIN_FAILURES.inc_failure_count(url, ConnectionFail(fail_type=ConnectionFailTypes.nodata)) save_failure(url, domain, True, post_data, post_json) diff --git a/sickbeard/logger.py b/sickbeard/logger.py index d311e5e..bd37945 100644 --- a/sickbeard/logger.py +++ b/sickbeard/logger.py @@ -171,7 +171,8 @@ class SBRotatingLogHandler(object): """ fmt = {} for logger_name in self.log_types + self.external_loggers: - source = (re.sub(r'(.*\.\w\w\w).*$', r'\1', logger_name).upper() + ' :: ', '')['sickbeard' == logger_name] + source = ((re.sub(r'(.*\.\w\w\w).*$', r'\1', logger_name), logger_name)['sg.' in logger_name] + .upper() + ' :: ', '')['sickbeard' == logger_name] fmt.setdefault(logger_name, logging.Formatter( '%(asctime)s %(levelname)' + ('-8', '')[log_simple] + 's ' + source + '%(message)s', ('%Y-%m-%d ', '')[log_simple] + '%H:%M:%S')) diff --git a/sickbeard/nzbSplitter.py b/sickbeard/nzbSplitter.py index 3acbf7a..16589e4 100644 --- a/sickbeard/nzbSplitter.py +++ b/sickbeard/nzbSplitter.py @@ -180,7 +180,7 @@ def splitResult(result): :return: list of search results :rtype: List[sickbeard.classes.SearchResult] """ - resp = helpers.get_url(result.url, failure_handling=False) + resp = helpers.get_url(result.url, failure_monitor=False) if None is resp: logger.log(u'Unable to load url %s, can\'t download season NZB' % result.url, logger.ERROR) return False diff --git a/sickbeard/providers/generic.py b/sickbeard/providers/generic.py index 59c1951..cbe771c 100644 --- a/sickbeard/providers/generic.py +++ b/sickbeard/providers/generic.py @@ -558,7 +558,8 @@ class GenericProvider(object): kwargs['raise_exceptions'] = True kwargs['raise_status_code'] = True - kwargs['failure_handling'] = False + kwargs['failure_monitor'] = False + kwargs['exclude_no_data'] = False for k, v in iteritems(dict(headers=self.headers, hooks=dict(response=self.cb_response))): kwargs.setdefault(k, v) if 'nzbs.in' not in url: # this provider returns 503's 3 out of 4 requests with the persistent session system @@ -753,7 +754,7 @@ class GenericProvider(object): self.session.headers['Referer'] = url if cached or helpers.download_file(url, cache_file, session=self.session, allow_redirects='/it' not in url, - failure_handling=False): + failure_monitor=False): if self._verify_download(cache_file): logger.log(u'Downloaded %s result from %s' % (self.name, url)) diff --git a/sickbeard/search.py b/sickbeard/search.py index d43325f..61176d6 100644 --- a/sickbeard/search.py +++ b/sickbeard/search.py @@ -1007,7 +1007,7 @@ def search_providers( cache_file = ek.ek(os.path.join, sickbeard.CACHE_DIR or helpers.get_system_temp_dir(), '%s.torrent' % (helpers.sanitize_filename(best_result.name))) if not helpers.download_file(best_result.url, cache_file, session=best_result.provider.session, - failure_handling=False): + failure_monitor=False): continue try: diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index 82bbd13..d8a928c 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -6142,7 +6142,7 @@ class History(MainHandler): mapped = 0 mapping = None maps = [x.split('=') for x in sickbeard.EMBY_PARENT_MAPS.split(',') if any(x)] - args = dict(params=dict(format='json'), timeout=10, parse_json=True, failure_handling=False) + args = dict(params=dict(format='json'), timeout=10, parse_json=True, failure_monitor=False) for i, cur_host in enumerate(hosts): base_url = 'http://%s/emby' % cur_host headers.update({'X-MediaBrowser-Token': keys[i]}) @@ -6166,7 +6166,7 @@ class History(MainHandler): if not folder or 'tvshows' != folder.get('CollectionType', ''): continue - items = helpers.get_url('%s/Items' % user_url, failure_handling=False, headers=headers, + items = helpers.get_url('%s/Items' % user_url, failure_monitor=False, headers=headers, params=dict(SortBy='DatePlayed,SeriesSortName,SortName', SortOrder='Descending', IncludeItemTypes='Episode',