From f6030a333a17e027607b69ddb42d10d1ae2d04cf Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 29 Jun 2014 23:15:36 +0200 Subject: [PATCH 001/202] Don't remove pyc files when using desktop updater --- couchpotato/core/_base/updater/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index 9f6613a..7730d3b 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -143,7 +143,7 @@ class Updater(Plugin): } def doShutdown(self, *args, **kwargs): - if not Env.get('dev'): + if not Env.get('dev') and not Env.get('desktop'): removePyc(Env.get('app_dir'), show_logs = False) return super(Updater, self).doShutdown(*args, **kwargs) From d20c0ee37ea306a2a27c09d1bf3d5e8daa5f1cc8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 29 Jun 2014 23:34:48 +0200 Subject: [PATCH 002/202] Remove Smackdown from defaults --- couchpotato/core/media/_base/providers/nzb/newznab.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/media/_base/providers/nzb/newznab.py b/couchpotato/core/media/_base/providers/nzb/newznab.py index 7db7e86..495463b 100644 --- a/couchpotato/core/media/_base/providers/nzb/newznab.py +++ b/couchpotato/core/media/_base/providers/nzb/newznab.py @@ -220,7 +220,7 @@ config = [{ 'description': 'Enable NewzNab such as NZB.su, \ NZBs.org, DOGnzb.cr, \ Spotweb, NZBGeek, \ - SmackDown, NZBFinder', + NZBFinder', 'wizard': True, 'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQAgMAAABinRfyAAAACVBMVEVjhwD///86aRovd/sBAAAAMklEQVQI12NgAIPQUCCRmQkjssDEShiRuRIqwZqZGcDAGBrqANUhGgIkWAOABKMDxCAA24UK50b26SAAAAAASUVORK5CYII=', 'options': [ @@ -231,30 +231,30 @@ config = [{ }, { 'name': 'use', - 'default': '0,0,0,0,0,0' + 'default': '0,0,0,0,0' }, { 'name': 'host', - 'default': 'api.nzb.su,api.dognzb.cr,nzbs.org,https://index.nzbgeek.info, https://smackdownonyou.com, https://www.nzbfinder.ws', + 'default': 'api.nzb.su,api.dognzb.cr,nzbs.org,https://index.nzbgeek.info,https://www.nzbfinder.ws', 'description': 'The hostname of your newznab provider', }, { 'name': 'extra_score', 'advanced': True, 'label': 'Extra Score', - 'default': '0,0,0,0,0,0', + 'default': '0,0,0,0,0', 'description': 'Starting score for each release found via this provider.', }, { 'name': 'custom_tag', 'advanced': True, 'label': 'Custom tag', - 'default': ',,,,,', + 'default': ',,,,', 'description': 'Add custom tags, for example add rls=1 to get only scene releases from nzbs.org', }, { 'name': 'api_key', - 'default': ',,,,,', + 'default': ',,,,', 'label': 'Api Key', 'description': 'Can be found on your profile page', 'type': 'combined', From 099b72ed2759f664937e6c15584752f3375ead33 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 30 Jun 2014 16:28:45 +0200 Subject: [PATCH 003/202] Allow 720p in lower qualities. fix #3539 --- couchpotato/core/plugins/quality/main.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index bc61afa..2687cb1 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -29,9 +29,9 @@ class QualityPlugin(Plugin): {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']}, {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]}, - {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':[]}, - {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':[]}, - {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':[]}, + {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p'], 'ext':[]}, + {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p'], 'ext':[]}, + {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p'], 'ext':[]}, {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': ['720p'], 'ext':[]} ] pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr'] @@ -454,6 +454,9 @@ class QualityPlugin(Plugin): 'Moviename 2014 720p HDCAM XviD DualAudio': {'size': 4000, 'quality': 'cam'}, 'Moviename (2014) - 720p CAM x264': {'size': 2250, 'quality': 'cam'}, 'Movie Name (2014).mp4': {'size': 750, 'quality': 'brrip'}, + 'Moviename.2014.720p.R6.WEB-DL.x264.AC3-xyz': {'size': 750, 'quality': 'r5'}, + 'Movie name 2014 New Source 720p HDCAM x264 AC3 xyz': {'size': 750, 'quality': 'cam'}, + 'Movie.Name.2014.720p.HD.TS.AC3.x264': {'size': 750, 'quality': 'ts'} } correct = 0 From 265f90fe6941bf0b7aa31b3a4e2e03adc4811e6c Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 30 Jun 2014 21:36:32 +0200 Subject: [PATCH 004/202] Unrar cleanup --- libs/unrar2/unix.py | 8 +++++--- libs/unrar2/windows.py | 22 +++++++++++----------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/libs/unrar2/unix.py b/libs/unrar2/unix.py index 9ebab40..a37b6df 100644 --- a/libs/unrar2/unix.py +++ b/libs/unrar2/unix.py @@ -24,12 +24,14 @@ import subprocess import gc - -import os, os.path -import time, re +import os +import os.path +import time +import re from rar_exceptions import * + class UnpackerNotInstalled(Exception): pass rar_executable_cached = None diff --git a/libs/unrar2/windows.py b/libs/unrar2/windows.py index e249f8f..bf8ffa5 100644 --- a/libs/unrar2/windows.py +++ b/libs/unrar2/windows.py @@ -174,7 +174,7 @@ class PassiveReader: def __init__(self, usercallback = None): self.buf = [] self.ucb = usercallback - + def _callback(self, msg, UserData, P1, P2): if msg == UCM_PROCESSDATA: data = (ctypes.c_char*P2).from_address(P1).raw @@ -183,7 +183,7 @@ class PassiveReader: else: self.buf.append(data) return 1 - + def get_result(self): return ''.join(self.buf) @@ -197,10 +197,10 @@ class RarInfoIterator(object): raise IncorrectRARPassword self.arc.lockStatus = "locked" self.arc.needskip = False - + def __iter__(self): return self - + def next(self): if self.index>0: if self.arc.needskip: @@ -208,9 +208,9 @@ class RarInfoIterator(object): self.res = RARReadHeaderEx(self.arc._handle, ctypes.byref(self.headerData)) if self.res: - raise StopIteration + raise StopIteration self.arc.needskip = True - + data = {} data['index'] = self.index data['filename'] = self.headerData.FileName @@ -224,7 +224,7 @@ class RarInfoIterator(object): self.index += 1 return data - + def __del__(self): self.arc.lockStatus = "finished" @@ -254,9 +254,9 @@ class RarFileImplementation(object): if password: RARSetPassword(self._handle, password) - + self.lockStatus = "ready" - + def destruct(self): @@ -287,7 +287,7 @@ class RarFileImplementation(object): self.needskip = False res.append((info, reader.get_result())) return res - + def extract(self, checker, path, withSubpath, overwrite): res = [] @@ -300,7 +300,7 @@ class RarFileImplementation(object): fn = os.path.split(fn)[-1] target = os.path.join(path, fn) else: - raise DeprecationWarning, "Condition callbacks returning strings are deprecated and only supported in Windows" + raise DeprecationWarning, "Condition callbacks returning strings are deprecated and only supported in Windows" target = checkres if overwrite or (not os.path.exists(target)): tmpres = RARProcessFile(self._handle, RAR_EXTRACT, None, target) From 33ad4c22c7f8edc8062c13731da2bfc6a0644e6d Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 30 Jun 2014 21:58:19 +0200 Subject: [PATCH 005/202] Try make unrar executable for user --- libs/unrar2/unix.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/libs/unrar2/unix.py b/libs/unrar2/unix.py index a37b6df..ce2d6a7 100644 --- a/libs/unrar2/unix.py +++ b/libs/unrar2/unix.py @@ -21,7 +21,8 @@ # SOFTWARE. # Unix version uses unrar command line executable - +import platform +import stat import subprocess import gc import os @@ -37,11 +38,19 @@ class UnpackerNotInstalled(Exception): pass rar_executable_cached = None rar_executable_version = None +osx_unrar = os.path.join(os.path.dirname(__file__), 'unrar') +if os.path.isfile(osx_unrar) and 'darwin' in platform.platform().lower(): + try: + st = os.stat(osx_unrar) + os.chmod(osx_unrar, st.st_mode | stat.S_IEXEC) + except: + pass + def call_unrar(params): "Calls rar/unrar command line executable, returns stdout pipe" global rar_executable_cached if rar_executable_cached is None: - for command in ('unrar', 'rar', os.path.join(os.path.dirname(__file__), 'unrar')): + for command in ('unrar', 'rar', osx_unrar): try: subprocess.Popen([command], stdout = subprocess.PIPE) rar_executable_cached = command From 169ddeef5df2eb898ea2496fb9240f0cbb70d6fb Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 30 Jun 2014 22:38:18 +0200 Subject: [PATCH 006/202] Allow custom unrar path. fix #3460 --- couchpotato/core/plugins/renamer.py | 7 ++++++- libs/unrar2/__init__.py | 28 ++++++++++++++-------------- libs/unrar2/unix.py | 11 ++++++----- libs/unrar2/windows.py | 2 +- 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 481c9dd..46cd677 100644 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -1140,7 +1140,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) log.info('Archive %s found. Extracting...', os.path.basename(archive['file'])) try: - rar_handle = RarFile(archive['file']) + rar_handle = RarFile(archive['file'], custom_path = self.conf('unrar_path')) extr_path = os.path.join(from_folder, os.path.relpath(os.path.dirname(archive['file']), folder)) self.makeDir(extr_path) for packedinfo in rar_handle.infolist(): @@ -1283,6 +1283,11 @@ config = [{ 'default': False, }, { + 'advanced': True, + 'name': 'unrar_path', + 'description': 'Custom path to unrar bin', + }, + { 'name': 'cleanup', 'type': 'bool', 'description': 'Cleanup leftover files after successful rename.', diff --git a/libs/unrar2/__init__.py b/libs/unrar2/__init__.py index fe27cfe..41b0d71 100644 --- a/libs/unrar2/__init__.py +++ b/libs/unrar2/__init__.py @@ -21,7 +21,7 @@ # SOFTWARE. """ -pyUnRAR2 is a ctypes based wrapper around the free UnRAR.dll. +pyUnRAR2 is a ctypes based wrapper around the free UnRAR.dll. It is an modified version of Jimmy Retzlaff's pyUnRAR - more simple, stable and foolproof. @@ -45,8 +45,8 @@ if in_windows: from windows import RarFileImplementation else: from unix import RarFileImplementation - - + + import fnmatch, time, weakref class RarInfo(object): @@ -62,7 +62,7 @@ class RarInfo(object): isdir - True if the file is a directory size - size in bytes of the uncompressed file comment - comment associated with the file - + Note - this is not currently intended to be a Python file-like object. """ @@ -74,7 +74,7 @@ class RarInfo(object): self.size = data['size'] self.datetime = data['datetime'] self.comment = data['comment'] - + def __str__(self): @@ -86,7 +86,7 @@ class RarInfo(object): class RarFile(RarFileImplementation): - def __init__(self, archiveName, password=None): + def __init__(self, archiveName, password=None, custom_path = None): """Instantiate the archive. archiveName is the name of the RAR file. @@ -99,7 +99,7 @@ class RarFile(RarFileImplementation): This is a test. """ self.archiveName = archiveName - RarFileImplementation.init(self, password) + RarFileImplementation.init(self, password, custom_path) def __del__(self): self.destruct() @@ -130,31 +130,31 @@ class RarFile(RarFileImplementation): """Read specific files from archive into memory. If "condition" is a list of numbers, then return files which have those positions in infolist. If "condition" is a string, then it is treated as a wildcard for names of files to extract. - If "condition" is a function, it is treated as a callback function, which accepts a RarInfo object + If "condition" is a function, it is treated as a callback function, which accepts a RarInfo object and returns boolean True (extract) or False (skip). If "condition" is omitted, all files are returned. - + Returns list of tuples (RarInfo info, str contents) """ checker = condition2checker(condition) return RarFileImplementation.read_files(self, checker) - + def extract(self, condition='*', path='.', withSubpath=True, overwrite=True): """Extract specific files from archive to disk. - + If "condition" is a list of numbers, then extract files which have those positions in infolist. If "condition" is a string, then it is treated as a wildcard for names of files to extract. If "condition" is a function, it is treated as a callback function, which accepts a RarInfo object and returns either boolean True (extract) or boolean False (skip). - DEPRECATED: If "condition" callback returns string (only supported for Windows) - + DEPRECATED: If "condition" callback returns string (only supported for Windows) - that string will be used as a new name to save the file under. If "condition" is omitted, all files are extracted. - + "path" is a directory to extract to "withSubpath" flag denotes whether files are extracted with their full path in the archive. "overwrite" flag denotes whether extracted files will overwrite old ones. Defaults to true. - + Returns list of RarInfos for extracted files.""" checker = condition2checker(condition) return RarFileImplementation.extract(self, checker, path, withSubpath, overwrite) diff --git a/libs/unrar2/unix.py b/libs/unrar2/unix.py index ce2d6a7..91ed4b6 100644 --- a/libs/unrar2/unix.py +++ b/libs/unrar2/unix.py @@ -46,11 +46,12 @@ if os.path.isfile(osx_unrar) and 'darwin' in platform.platform().lower(): except: pass -def call_unrar(params): +def call_unrar(params, custom_path = None): "Calls rar/unrar command line executable, returns stdout pipe" global rar_executable_cached if rar_executable_cached is None: - for command in ('unrar', 'rar', osx_unrar): + for command in (custom_path, 'unrar', 'rar', osx_unrar): + if not command: continue try: subprocess.Popen([command], stdout = subprocess.PIPE) rar_executable_cached = command @@ -70,10 +71,10 @@ def call_unrar(params): class RarFileImplementation(object): - def init(self, password = None): + def init(self, password = None, custom_path = None): global rar_executable_version self.password = password - + self.custom_path = custom_path stdoutdata, stderrdata = self.call('v', []).communicate() @@ -129,7 +130,7 @@ class RarFileImplementation(object): def call(self, cmd, options = [], files = []): options2 = options + ['p' + self.escaped_password()] soptions = ['-' + x for x in options2] - return call_unrar([cmd] + soptions + ['--', self.archiveName] + files) + return call_unrar([cmd] + soptions + ['--', self.archiveName] + files, self.custom_path) def infoiter(self): diff --git a/libs/unrar2/windows.py b/libs/unrar2/windows.py index bf8ffa5..e3d920f 100644 --- a/libs/unrar2/windows.py +++ b/libs/unrar2/windows.py @@ -237,7 +237,7 @@ def generate_password_provider(password): class RarFileImplementation(object): - def init(self, password=None): + def init(self, password=None, custom_path = None): self.password = password archiveData = RAROpenArchiveDataEx(ArcNameW=self.archiveName, OpenMode=RAR_OM_EXTRACT) self._handle = RAROpenArchiveEx(ctypes.byref(archiveData)) From 34320e617d7c908c3c9f1bad0387ca40f7596865 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 30 Jun 2014 22:47:41 +0200 Subject: [PATCH 007/202] Unrar extract time options. closes #2733 --- couchpotato/core/plugins/renamer.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 46cd677..ef56309 100644 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -1144,10 +1144,16 @@ Remove it if you want it to be renamed (again, or at least let it try again) extr_path = os.path.join(from_folder, os.path.relpath(os.path.dirname(archive['file']), folder)) self.makeDir(extr_path) for packedinfo in rar_handle.infolist(): - if not packedinfo.isdir and not os.path.isfile(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename)))): + extr_file_path = sp(os.path.join(extr_path, os.path.basename(packedinfo.filename))) + if not packedinfo.isdir and not os.path.isfile(extr_file_path): log.debug('Extracting %s...', packedinfo.filename) rar_handle.extract(condition = [packedinfo.index], path = extr_path, withSubpath = False, overwrite = False) - extr_files.append(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename)))) + if self.conf('unrar_modify_date'): + try: + os.utime(extr_file_path, (os.path.getatime(archive['file']), os.path.getmtime(archive['file']))) + except: + log.error('Rar modify date enabled, but failed: %s', traceback.format_exc()) + extr_files.append(extr_file_path) del rar_handle except Exception as e: log.error('Failed to extract %s: %s %s', (archive['file'], e, traceback.format_exc())) @@ -1288,6 +1294,13 @@ config = [{ 'description': 'Custom path to unrar bin', }, { + 'advanced': True, + 'name': 'unrar_modify_date', + 'type': 'bool', + 'description': ('Set modify date of unrar-ed files to the rar-file\'s date.', 'This will allow XBMC to recognize extracted files as recently added even if the movie was released some time ago.'), + 'default': False, + }, + { 'name': 'cleanup', 'type': 'bool', 'description': 'Cleanup leftover files after successful rename.', From f765794c99feaf548db0a77c50883b93e0fd85df Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 1 Jul 2014 21:00:05 +0200 Subject: [PATCH 008/202] Don't add managed to no-process renamer list. fix #3538 --- couchpotato/core/plugins/renamer.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index ef56309..024c438 100644 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -123,11 +123,6 @@ class Renamer(Plugin): no_process = [to_folder] cat_list = fireEvent('category.all', single = True) or [] no_process.extend([item['destination'] for item in cat_list]) - try: - if Env.setting('library', section = 'manage').strip(): - no_process.extend([sp(manage_folder) for manage_folder in splitString(Env.setting('library', section = 'manage'), '::')]) - except: - pass # Check to see if the no_process folders are inside the "from" folder. if not os.path.isdir(base_folder) or not os.path.isdir(to_folder): From d201d9fff9f99e264ac914c2f3b24f7797abb556 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 2 Jul 2014 19:02:55 +0200 Subject: [PATCH 009/202] Allow change of file move action --- couchpotato/core/plugins/renamer.py | 47 +++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 024c438..2bc4787 100644 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -197,14 +197,18 @@ class Renamer(Plugin): db = get_db() # Extend the download info with info stored in the downloaded release + keep_original = self.moveTypeIsLinked() + is_torrent = False if release_download: release_download = self.extendReleaseDownload(release_download) + is_torrent = self.downloadIsTorrent(release_download) + keep_original = True if is_torrent and self.conf('file_action') not in ['move'] else keep_original # Unpack any archives extr_files = None if self.conf('unrar'): folder, media_folder, files, extr_files = self.extractFiles(folder = folder, media_folder = media_folder, files = files, - cleanup = self.conf('cleanup') and not self.downloadIsTorrent(release_download)) + cleanup = self.conf('cleanup') and not keep_original) groups = fireEvent('scanner.scan', folder = folder if folder else base_folder, files = files, release_download = release_download, return_ignored = False, single = True) or [] @@ -321,7 +325,7 @@ class Renamer(Plugin): if file_type is 'nfo' and not self.conf('rename_nfo'): log.debug('Skipping, renaming of %s disabled', file_type) for current_file in group['files'][file_type]: - if self.conf('cleanup') and (not self.downloadIsTorrent(release_download) or self.fileIsAdded(current_file, group)): + if self.conf('cleanup') and (not keep_original or self.fileIsAdded(current_file, group)): remove_files.append(current_file) continue @@ -527,7 +531,7 @@ class Renamer(Plugin): log.debug('Removing leftover files') for current_file in group['files']['leftover']: if self.conf('cleanup') and not self.conf('move_leftover') and \ - (not self.downloadIsTorrent(release_download) or self.fileIsAdded(current_file, group)): + (not keep_original or self.fileIsAdded(current_file, group)): remove_files.append(current_file) # Remove files @@ -574,7 +578,7 @@ class Renamer(Plugin): self.makeDir(os.path.dirname(dst)) try: - self.moveFile(src, dst, forcemove = not self.downloadIsTorrent(release_download) or self.fileIsAdded(src, group)) + self.moveFile(src, dst, use_default = not is_torrent or self.fileIsAdded(src, group)) group['renamed_files'].append(dst) except: log.error('Failed renaming the file "%s" : %s', (os.path.basename(src), traceback.format_exc())) @@ -590,7 +594,7 @@ class Renamer(Plugin): self.untagRelease(group = group, tag = 'failed_rename') # Tag folder if it is in the 'from' folder and it will not be removed because it is a torrent - if self.movieInFromFolder(media_folder) and self.downloadIsTorrent(release_download): + if self.movieInFromFolder(media_folder) and keep_original: self.tagRelease(group = group, tag = 'renamed_already') # Remove matching releases @@ -601,7 +605,7 @@ class Renamer(Plugin): except: log.error('Failed removing %s: %s', (release, traceback.format_exc())) - if group['dirname'] and group['parentdir'] and not self.downloadIsTorrent(release_download): + if group['dirname'] and group['parentdir'] and not keep_original: if media_folder: # Delete the movie folder group_folder = media_folder @@ -763,10 +767,15 @@ Remove it if you want it to be renamed (again, or at least let it try again) return False - def moveFile(self, old, dest, forcemove = False): + def moveFile(self, old, dest, use_default = False): dest = sp(dest) try: - if forcemove or self.conf('file_action') not in ['copy', 'link']: + + move_type = self.conf('file_action') + if use_default: + move_type = self.conf('default_file_action') + + if move_type not in ['copy', 'link']: try: shutil.move(old, dest) except: @@ -775,16 +784,16 @@ Remove it if you want it to be renamed (again, or at least let it try again) os.unlink(old) else: raise - elif self.conf('file_action') == 'copy': + elif move_type == 'copy': shutil.copy(old, dest) - elif self.conf('file_action') == 'link': + else: # First try to hardlink try: log.debug('Hardlinking file "%s" to "%s"...', (old, dest)) link(old, dest) except: # Try to simlink next - log.debug('Couldn\'t hardlink file "%s" to "%s". Simlinking instead. Error: %s.', (old, dest, traceback.format_exc())) + log.debug('Couldn\'t hardlink file "%s" to "%s". Symlinking instead. Error: %s.', (old, dest, traceback.format_exc())) shutil.copy(old, dest) try: symlink(dest, old + '.link') @@ -1084,6 +1093,9 @@ Remove it if you want it to be renamed (again, or at least let it try again) return False return src in group['before_rename'] + def moveTypeIsLinked(self): + return self.conf('default_file_action') in ['copy', 'link'] + def statusInfoComplete(self, release_download): return release_download.get('id') and release_download.get('downloader') and release_download.get('folder') @@ -1346,13 +1358,22 @@ config = [{ 'description': ('Replace all the spaces with a character.', 'Example: ".", "-" (without quotes). Leave empty to use spaces.'), }, { + 'name': 'default_file_action', + 'label': 'Default File Action', + 'default': 'move', + 'type': 'dropdown', + 'values': [('Link', 'link'), ('Copy', 'copy'), ('Move', 'move')], + 'description': ('Link, Copy or Move after download completed.', + 'Link first tries hard link, then sym link and falls back to Copy.'), + 'advanced': True, + }, + { 'name': 'file_action', 'label': 'Torrent File Action', 'default': 'link', 'type': 'dropdown', 'values': [('Link', 'link'), ('Copy', 'copy'), ('Move', 'move')], - 'description': ('Link, Copy or Move after download completed.', - 'Link first tries hard link, then sym link and falls back to Copy. It is perfered to use link when downloading torrents as it will save you space, while still beeing able to seed.'), + 'description': 'See above. It is prefered to use link when downloading torrents as it will save you space, while still beeing able to seed.', 'advanced': True, }, { From 330e15bbcb56d44766d5bb89ce2905e7841fd22e Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 4 Jul 2014 20:36:48 +0200 Subject: [PATCH 010/202] Snatched not giving enough data to show notification. fix #3564 --- couchpotato/core/notifications/pushover.py | 6 +++--- couchpotato/core/plugins/release/main.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/notifications/pushover.py b/couchpotato/core/notifications/pushover.py index d9ef226..46dc0ad 100644 --- a/couchpotato/core/notifications/pushover.py +++ b/couchpotato/core/notifications/pushover.py @@ -1,7 +1,7 @@ from httplib import HTTPSConnection from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode -from couchpotato.core.helpers.variable import getTitle +from couchpotato.core.helpers.variable import getTitle, getIdentifier from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification @@ -27,9 +27,9 @@ class Pushover(Notification): 'sound': self.conf('sound'), } - if data and data.get('identifier'): + if data and getIdentifier(data): api_data.update({ - 'url': toUnicode('http://www.imdb.com/title/%s/' % data['identifier']), + 'url': toUnicode('http://www.imdb.com/title/%s/' % getIdentifier(data)), 'url_title': toUnicode('%s on IMDb' % getTitle(data)), }) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 196892c..7b47aa9 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -327,7 +327,7 @@ class Release(Plugin): log_movie = '%s (%s) in %s' % (getTitle(media), media['info']['year'], rls['quality']) snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie) log.info(snatch_message) - fireEvent('%s.snatched' % data['type'], message = snatch_message, data = rls) + fireEvent('%s.snatched' % data['type'], message = snatch_message, data = media) # Mark release as snatched if renamer_enabled: From 40a5ce087b52a7f9b940fed25a08c83aa8c94da4 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 6 Jul 2014 22:19:45 +0200 Subject: [PATCH 011/202] Better label for Pushbullet settings --- couchpotato/core/notifications/pushbullet.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/pushbullet.py b/couchpotato/core/notifications/pushbullet.py index 361294e..e9d4605 100644 --- a/couchpotato/core/notifications/pushbullet.py +++ b/couchpotato/core/notifications/pushbullet.py @@ -84,7 +84,8 @@ config = [{ }, { 'name': 'api_key', - 'label': 'User API Key' + 'label': 'Access Token', + 'description': 'Can be found on Account Settings', }, { 'name': 'devices', From 04e22b3966977085cd706bbcb9f1806ee2d4d1fc Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 8 Jul 2014 20:09:24 +0200 Subject: [PATCH 012/202] XBMC error > info2 --- couchpotato/core/notifications/xbmc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/xbmc.py b/couchpotato/core/notifications/xbmc.py index eb0b699..bf5310e 100644 --- a/couchpotato/core/notifications/xbmc.py +++ b/couchpotato/core/notifications/xbmc.py @@ -208,7 +208,7 @@ class XBMC(Notification): log.debug('Returned from request %s: %s', (host, response)) return response - except (MaxRetryError, requests.exceptions.Timeout): + except (MaxRetryError, requests.exceptions.Timeout, ConnectionError): log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off') return [] except: From e0479e79bd7295c549dfc61a70694e8e1e9d30cd Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 8 Jul 2014 20:24:18 +0200 Subject: [PATCH 013/202] AwesomeHD not returning proper size. fix #3587 --- couchpotato/core/media/_base/providers/torrent/awesomehd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/torrent/awesomehd.py b/couchpotato/core/media/_base/providers/torrent/awesomehd.py index 78c4648..bd9e193 100644 --- a/couchpotato/core/media/_base/providers/torrent/awesomehd.py +++ b/couchpotato/core/media/_base/providers/torrent/awesomehd.py @@ -61,7 +61,7 @@ class Base(TorrentProvider): 'name': re.sub('[^A-Za-z0-9\-_ \(\).]+', '', '%s (%s) %s' % (name, year, torrent_desc)), 'url': self.urls['download'] % (torrent_id, authkey, self.conf('passkey')), 'detail_url': self.urls['detail'] % torrent_id, - 'size': self.parseSize(entry.find('size').get_text()), + 'size': tryInt(entry.find('size').get_text()) / 1048576, 'seeders': tryInt(entry.find('seeders').get_text()), 'leechers': tryInt(entry.find('leechers').get_text()), 'score': torrentscore From 32ce93d2e9bccbaa5768324984d956627694eadd Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 11 Jul 2014 10:30:32 +0200 Subject: [PATCH 014/202] Encode video path --- couchpotato/core/plugins/scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/scanner.py b/couchpotato/core/plugins/scanner.py index 6a4d537..3d39b29 100644 --- a/couchpotato/core/plugins/scanner.py +++ b/couchpotato/core/plugins/scanner.py @@ -553,7 +553,7 @@ class Scanner(Plugin): scan_result = [] for p in paths: if not group['is_dvd']: - video = Video.from_path(toUnicode(p)) + video = Video.from_path(sp(p)) video_result = [(video, video.scan())] scan_result.extend(video_result) From 0b48ad5084fdafa56abbe21dccfedabea8b12b0e Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 11 Jul 2014 16:24:45 +0200 Subject: [PATCH 015/202] Change fanart api url --- couchpotato/core/media/movie/providers/info/fanarttv.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/media/movie/providers/info/fanarttv.py b/couchpotato/core/media/movie/providers/info/fanarttv.py index 8bfa92c..fcd3891 100644 --- a/couchpotato/core/media/movie/providers/info/fanarttv.py +++ b/couchpotato/core/media/movie/providers/info/fanarttv.py @@ -14,7 +14,7 @@ autoload = 'FanartTV' class FanartTV(MovieProvider): urls = { - 'api': 'http://api.fanart.tv/webservice/movie/b28b14e9be662e027cfbc7c3dd600405/%s/JSON/all/1/2' + 'api': 'http://webservice.fanart.tv/v3/movies/%s?api_key=b28b14e9be662e027cfbc7c3dd600405' } MAX_EXTRAFANART = 20 @@ -36,9 +36,8 @@ class FanartTV(MovieProvider): fanart_data = self.getJsonData(url) if fanart_data: - name, resource = fanart_data.items()[0] - log.debug('Found images for %s', name) - images = self._parseMovie(resource) + log.debug('Found images for %s', fanart_data.get('name')) + images = self._parseMovie(fanart_data) except: log.error('Failed getting extra art for %s: %s', @@ -95,7 +94,7 @@ class FanartTV(MovieProvider): for image in images: if tryInt(image.get('likes')) > highscore: highscore = tryInt(image.get('likes')) - image_url = image.get('url') + image_url = image.get('url') or image.get('href') return image_url @@ -118,7 +117,9 @@ class FanartTV(MovieProvider): if tryInt(image.get('likes')) > highscore: highscore = tryInt(image.get('likes')) best = image - image_urls.append(best.get('url')) + url = best.get('url') or best.get('href') + if url: + image_urls.append(url) pool.remove(best) return image_urls From 06f49be09007406d1608c04d84ca1cd77df52a7b Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 11 Jul 2014 16:47:15 +0200 Subject: [PATCH 016/202] Don't error out on media.get. fix #3611 --- couchpotato/core/media/_base/media/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 1d3e153..e79e3ea 100644 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -4,7 +4,7 @@ import time import traceback from string import ascii_lowercase -from CodernityDB.database import RecordNotFound +from CodernityDB.database import RecordNotFound, RecordDeleted from couchpotato import tryInt, get_db from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, fireEventAsync, addEvent @@ -146,7 +146,7 @@ class MediaPlugin(MediaBase): return media - except RecordNotFound: + except (RecordNotFound, RecordDeleted): log.error('Media with id "%s" not found', media_id) except: raise From b13df16b53bbd35c2b59c35ce2b309de3f63b1f1 Mon Sep 17 00:00:00 2001 From: peerster Date: Sat, 12 Jul 2014 16:29:42 +0200 Subject: [PATCH 017/202] Added systemd specific instructions --- README.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4dbe75b..05e3c8d 100644 --- a/README.md +++ b/README.md @@ -29,17 +29,23 @@ OS X: * Then do `python CouchPotatoServer/CouchPotato.py` * Your browser should open up, but if it doesn't go to `http://localhost:5050/` -Linux (Ubuntu / Debian): +Linux: -* Install [GIT](http://git-scm.com/) with `apt-get install git-core` +* (Ubuntu / Debian) Install [GIT](http://git-scm.com/) with `apt-get install git-core` +* (Fedora / CentOS) Install [GIT](http://git-scm.com/) with `yum install git` * 'cd' to the folder of your choosing. * Run `git clone https://github.com/RuudBurger/CouchPotatoServer.git` * Then do `python CouchPotatoServer/CouchPotato.py` to start -* To run on boot copy the init script `sudo cp CouchPotatoServer/init/ubuntu /etc/init.d/couchpotato` -* Copy the default paths file `sudo cp CouchPotatoServer/init/ubuntu.default /etc/default/couchpotato` -* Change the paths inside the default file `sudo nano /etc/default/couchpotato` -* Make it executable `sudo chmod +x /etc/init.d/couchpotato` -* Add it to defaults `sudo update-rc.d couchpotato defaults` +* (Ubuntu / Debian) To run on boot copy the init script `sudo cp CouchPotatoServer/init/ubuntu /etc/init.d/couchpotato` +* (Ubuntu / Debian) Copy the default paths file `sudo cp CouchPotatoServer/init/ubuntu.default /etc/default/couchpotato` +* (Ubuntu / Debian) Change the paths inside the default file `sudo nano /etc/default/couchpotato` +* (Ubuntu / Debian) Make it executable `sudo chmod +x /etc/init.d/couchpotato` +* (Ubuntu / Debian) Add it to defaults `sudo update-rc.d couchpotato defaults` +Systemd specific: +* To run on boot copy the systemd config `sudo cp CouchPotatoServer/init/couchpotato.fedora.service /etc/systemd/system/couchpotato.service` +* Update the systemd config file with your user and path to CouchPotato.py +* Enable it at boot with `sudo systemctl enable couchpotato` + * Open your browser and go to `http://localhost:5050/` From bd6690b159b3a510757e262bee8dafa7e7824f87 Mon Sep 17 00:00:00 2001 From: peerster Date: Sat, 12 Jul 2014 16:34:41 +0200 Subject: [PATCH 018/202] layout fix --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 05e3c8d..4d5a39e 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,9 @@ Linux: * (Ubuntu / Debian) Change the paths inside the default file `sudo nano /etc/default/couchpotato` * (Ubuntu / Debian) Make it executable `sudo chmod +x /etc/init.d/couchpotato` * (Ubuntu / Debian) Add it to defaults `sudo update-rc.d couchpotato defaults` -Systemd specific: -* To run on boot copy the systemd config `sudo cp CouchPotatoServer/init/couchpotato.fedora.service /etc/systemd/system/couchpotato.service` -* Update the systemd config file with your user and path to CouchPotato.py -* Enable it at boot with `sudo systemctl enable couchpotato` +* (systemd) To run on boot copy the systemd config `sudo cp CouchPotatoServer/init/couchpotato.fedora.service /etc/systemd/system/couchpotato.service` +* (systemd) Update the systemd config file with your user and path to CouchPotato.py +* (systemd) Enable it at boot with `sudo systemctl enable couchpotato` * Open your browser and go to `http://localhost:5050/` From a99d52392fce799075c230bdef4f93af62344976 Mon Sep 17 00:00:00 2001 From: peerster Date: Sat, 12 Jul 2014 16:40:24 +0200 Subject: [PATCH 019/202] Another layout fix --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 4d5a39e..a38c052 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,6 @@ Linux: * (systemd) To run on boot copy the systemd config `sudo cp CouchPotatoServer/init/couchpotato.fedora.service /etc/systemd/system/couchpotato.service` * (systemd) Update the systemd config file with your user and path to CouchPotato.py * (systemd) Enable it at boot with `sudo systemctl enable couchpotato` - * Open your browser and go to `http://localhost:5050/` From 687221f035587ad0339c1bb8c433c94a993f2b98 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 12 Jul 2014 19:38:49 +0200 Subject: [PATCH 020/202] Update last_edit when tagging with recent --- couchpotato/core/media/_base/media/main.py | 7 +++++-- couchpotato/core/media/movie/searcher.py | 2 +- couchpotato/core/plugins/renamer.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index e79e3ea..ee9e6cc 100644 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -488,18 +488,21 @@ class MediaPlugin(MediaBase): db.update(m) # Tag media as recent - self.tag(media_id, 'recent') + self.tag(media_id, 'recent', update_edited = True) return m['status'] except: log.error('Failed restatus: %s', traceback.format_exc()) - def tag(self, media_id, tag): + def tag(self, media_id, tag, update_edited = False): try: db = get_db() m = db.get('id', media_id) + if update_edited: + m['last_edit'] = int(time.time()) + tags = m.get('tags') or [] if tag not in tags: tags.append(tag) diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index 7d92c57..4bd8c8d 100644 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -240,7 +240,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): break if total_result_count > 0: - fireEvent('media.tag', movie['_id'], 'recent', single = True) + fireEvent('media.tag', movie['_id'], 'recent', update_edited = True, single = True) if len(too_early_to_search) > 0: log.info2('Too early to search for %s, %s', (too_early_to_search, default_title)) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 2bc4787..9f6792a 100644 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -522,7 +522,7 @@ class Renamer(Plugin): # Mark media for dashboard if mark_as_recent: - fireEvent('media.tag', group['media'].get('_id'), 'recent', single = True) + fireEvent('media.tag', group['media'].get('_id'), 'recent', update_edited = True, single = True) # Remove leftover files if not remove_leftovers: # Don't remove anything From 12cda35494770ae4a3d8b1f4eabf4c5a416f68c7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 17 Jul 2014 22:24:48 +0200 Subject: [PATCH 021/202] Try fix migration failure from 2.5.1 --- couchpotato/core/database.py | 30 +++++++++++++++++++++++++++++- couchpotato/runner.py | 7 +++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/database.py b/couchpotato/core/database.py index c7051f5..47be7f5 100644 --- a/couchpotato/core/database.py +++ b/couchpotato/core/database.py @@ -3,10 +3,11 @@ import os import time import traceback +from CodernityDB.database import RecordNotFound from CodernityDB.index import IndexException, IndexNotFoundException, IndexConflict from couchpotato import CPLog from couchpotato.api import addApiView -from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, sp from couchpotato.core.helpers.variable import getImdb, tryInt @@ -226,6 +227,33 @@ class Database(object): from couchpotato import Env db = self.getDB() + + # Try fix for migration failures on desktop + if Env.get('desktop'): + try: + list(db.all('profile', with_doc = True)) + except RecordNotFound: + + failed_location = '%s_failed' % db.path + old_db = os.path.join(Env.get('data_dir'), 'couchpotato.db.old') + + if not os.path.isdir(failed_location) and os.path.isfile(old_db): + db.close() + + # Rename database folder + os.rename(db.path, '%s_failed' % db.path) + + # Rename .old database to try another migrate + os.rename(old_db, old_db[:-4]) + + fireEventAsync('app.restart') + else: + log.error('Migration failed and couldn\'t recover database. Please report on GitHub, with this message.') + db.reindex() + + return + + # Check size and compact if needed size = db.get_db_details().get('size') prop_name = 'last_db_compact' last_check = int(Env.prop(prop_name, default = 0)) diff --git a/couchpotato/runner.py b/couchpotato/runner.py index e5f9bca..5d3f62b 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -87,6 +87,13 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En # Do db stuff db_path = sp(os.path.join(data_dir, 'database')) + old_db_path = os.path.join(data_dir, 'couchpotato.db') + + # Remove database folder if both exists + if os.path.isdir(db_path) and os.path.isfile(old_db_path): + db = SuperThreadSafeDatabase(db_path) + db.open() + db.destroy() # Check if database exists db = SuperThreadSafeDatabase(db_path) From 324920cd8c6f080f391a99610c19d863072a190b Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 17 Jul 2014 23:09:26 +0200 Subject: [PATCH 022/202] Don't migrate when db is closed --- couchpotato/core/database.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/couchpotato/core/database.py b/couchpotato/core/database.py index 47be7f5..10ae26c 100644 --- a/couchpotato/core/database.py +++ b/couchpotato/core/database.py @@ -238,6 +238,7 @@ class Database(object): old_db = os.path.join(Env.get('data_dir'), 'couchpotato.db.old') if not os.path.isdir(failed_location) and os.path.isfile(old_db): + log.error('Corrupt database, trying migrate again') db.close() # Rename database folder @@ -331,6 +332,8 @@ class Database(object): log.info('Getting data took %s', time.time() - migrate_start) db = self.getDB() + if not db.opened: + return # Use properties properties = migrate_data['properties'] From cd0afd20e56571ab82f927e8dd5ba68e169a106e Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 17 Jul 2014 23:58:51 +0200 Subject: [PATCH 023/202] Always reconnect Transmission. fix #3631 --- couchpotato/core/downloaders/transmission.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/downloaders/transmission.py b/couchpotato/core/downloaders/transmission.py index d941cca..53ca9b8 100644 --- a/couchpotato/core/downloaders/transmission.py +++ b/couchpotato/core/downloaders/transmission.py @@ -23,17 +23,14 @@ class Transmission(DownloaderBase): log = CPLog(__name__) trpc = None - def connect(self, reconnect = False): + def connect(self): # Load host from config and split out port. host = cleanHost(self.conf('host'), protocol = False).split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False - if not self.trpc or reconnect: - self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url').strip('/ '), username = self.conf('username'), password = self.conf('password')) - - return self.trpc + self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url').strip('/ '), username = self.conf('username'), password = self.conf('password')) def download(self, data = None, media = None, filedata = None): if not media: media = {} @@ -88,7 +85,7 @@ class Transmission(DownloaderBase): return self.downloadReturnId(remote_torrent['torrent-added']['hashString']) def test(self): - if self.connect(True) and self.trpc.get_session(): + if self.connect() and self.trpc.get_session(): return True return False From 0771aeac3bbeebe9af58db7cd04504bb0ad6e310 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 20 Jul 2014 16:20:12 +0200 Subject: [PATCH 024/202] Update nzbgeek api url --- couchpotato/core/media/_base/providers/nzb/newznab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/nzb/newznab.py b/couchpotato/core/media/_base/providers/nzb/newznab.py index 495463b..b1a8ec6 100644 --- a/couchpotato/core/media/_base/providers/nzb/newznab.py +++ b/couchpotato/core/media/_base/providers/nzb/newznab.py @@ -235,7 +235,7 @@ config = [{ }, { 'name': 'host', - 'default': 'api.nzb.su,api.dognzb.cr,nzbs.org,https://index.nzbgeek.info,https://www.nzbfinder.ws', + 'default': 'api.nzb.su,api.dognzb.cr,nzbs.org,https://api.nzbgeek.info,https://www.nzbfinder.ws', 'description': 'The hostname of your newznab provider', }, { From cd1dc39ef23d41cb018899c30f681b7f0dbbad22 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 27 Jul 2014 21:14:31 +0200 Subject: [PATCH 025/202] Add bd50 torrentleech --- couchpotato/core/media/movie/providers/torrent/torrentleech.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/providers/torrent/torrentleech.py b/couchpotato/core/media/movie/providers/torrent/torrentleech.py index 191ceba..d72f425 100644 --- a/couchpotato/core/media/movie/providers/torrent/torrentleech.py +++ b/couchpotato/core/media/movie/providers/torrent/torrentleech.py @@ -11,7 +11,7 @@ autoload = 'TorrentLeech' class TorrentLeech(MovieProvider, Base): cat_ids = [ - ([13], ['720p', '1080p']), + ([13], ['720p', '1080p', 'bd50']), ([8], ['cam']), ([9], ['ts', 'tc']), ([10], ['r5', 'scr']), From 7b6fa4f0e54cd0be7d818338b926e9329d0bf1d4 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 27 Jul 2014 21:28:06 +0200 Subject: [PATCH 026/202] Transmission failed to download --- couchpotato/core/downloaders/transmission.py | 1 + 1 file changed, 1 insertion(+) diff --git a/couchpotato/core/downloaders/transmission.py b/couchpotato/core/downloaders/transmission.py index 53ca9b8..409efaa 100644 --- a/couchpotato/core/downloaders/transmission.py +++ b/couchpotato/core/downloaders/transmission.py @@ -31,6 +31,7 @@ class Transmission(DownloaderBase): return False self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url').strip('/ '), username = self.conf('username'), password = self.conf('password')) + return self.trpc def download(self, data = None, media = None, filedata = None): if not media: media = {} From 456563eab01447f31271db81b4951f767315ff25 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 28 Jul 2014 16:01:16 +0200 Subject: [PATCH 027/202] Update log post text --- couchpotato/core/plugins/log/static/log.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/log/static/log.js b/couchpotato/core/plugins/log/static/log.js index 11acb5c..fcb7ed5 100644 --- a/couchpotato/core/plugins/log/static/log.js +++ b/couchpotato/core/plugins/log/static/log.js @@ -241,7 +241,7 @@ Running on: ...\n\ 'href': 'https://github.com/RuudBurger/CouchPotatoServer/blob/develop/contributing.md' }), new Element('span', { - 'text': ' before posting, then copy the text below' + 'text': ' before posting (I\'ll close if you don\'t), then copy the text below.' }) ), textarea = new Element('textarea', { From 69e3e36fae5071de3e88a3c53ef5123f90865273 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 28 Jul 2014 16:01:27 +0200 Subject: [PATCH 028/202] Update contribute --- contributing.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contributing.md b/contributing.md index 821212c..e95337d 100644 --- a/contributing.md +++ b/contributing.md @@ -13,6 +13,8 @@ Lastly, for anything related to CouchPotato, feel free to stop by the [forum](ht ## Issues Issues are intended for reporting bugs and weird behaviour or suggesting improvements to CouchPotatoServer. Before you submit an issue, please go through the following checklist: + * FILL IN ALL THE FIELDS ASKED FOR, if you don't tell why you didn't or isn't relevant + * POST MORE THAN A SINGLE LINE, if you do, you'd better have a easy reproducable bug * Search through existing issues (*including closed issues!*) first: you might be able to get your answer there. * Double check your issue manually, because it could be an external issue. * Post logs with your issue: Without seeing what is going on, the developers can't reproduce the error. @@ -25,12 +27,14 @@ Before you submit an issue, please go through the following checklist: * What hardware / OS are you using and what are its limitations? For example: NAS can be slow and maybe have a different version of python installed than when you use CP on OS X or Windows. * Your issue might be marked with the "can't reproduce" tag. Don't ask why your issue was closed if it says so in the tag. * If you're running on a NAS (QNAP, Austor, Synology etc.) with pre-made packages, make sure these are set up to use our source repository (RuudBurger/CouchPotatoServer) and nothing else! + * Do not "bump" issues with "Any updates on this" or whatever. Yes I've seen it, you don't have to remind me of it. There will be an update when the code is done or I need information. If you feel the need to do so, you'd better have more info on the issue. The more relevant information you provide, the more likely that your issue will be resolved. +If you don't follow any of the checks above, I'll close the issue. If you are wondering why (and ask) I'll block you from posting new issues and the repo. ## Pull Requests Pull requests are intended for contributing code or documentation to the project. Before you submit a pull request, consider the following: * Make sure your pull request is made for the *develop* branch (or relevant feature branch). * Have you tested your PR? If not, why? - * Does your PR have any limitations we should know of? + * Does your PR have any limitations I should know of? * Is your PR up-to-date with the branch you're trying to push into? From bed94586041aac7c2fb0094e86f6341c96ae272d Mon Sep 17 00:00:00 2001 From: Ruud Burger Date: Mon, 28 Jul 2014 16:06:38 +0200 Subject: [PATCH 029/202] Update contributing.md --- contributing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contributing.md b/contributing.md index e95337d..3bd42c0 100644 --- a/contributing.md +++ b/contributing.md @@ -13,8 +13,8 @@ Lastly, for anything related to CouchPotato, feel free to stop by the [forum](ht ## Issues Issues are intended for reporting bugs and weird behaviour or suggesting improvements to CouchPotatoServer. Before you submit an issue, please go through the following checklist: - * FILL IN ALL THE FIELDS ASKED FOR, if you don't tell why you didn't or isn't relevant - * POST MORE THAN A SINGLE LINE, if you do, you'd better have a easy reproducable bug + * **FILL IN ALL THE FIELDS ASKED FOR** + * **POST MORE THAN A SINGLE LINE LOG**, if you do, you'd better have a easy reproducable bug * Search through existing issues (*including closed issues!*) first: you might be able to get your answer there. * Double check your issue manually, because it could be an external issue. * Post logs with your issue: Without seeing what is going on, the developers can't reproduce the error. From e749d132cda342533c69a02ea648788cece5e752 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 28 Jul 2014 16:12:54 +0200 Subject: [PATCH 030/202] Better message --- couchpotato/core/plugins/log/static/log.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/log/static/log.js b/couchpotato/core/plugins/log/static/log.js index fcb7ed5..6cb683b 100644 --- a/couchpotato/core/plugins/log/static/log.js +++ b/couchpotato/core/plugins/log/static/log.js @@ -241,7 +241,7 @@ Running on: ...\n\ 'href': 'https://github.com/RuudBurger/CouchPotatoServer/blob/develop/contributing.md' }), new Element('span', { - 'text': ' before posting (I\'ll close if you don\'t), then copy the text below.' + 'text': ' before posting (kittens die if you don\'t), then copy the text below.' }) ), textarea = new Element('textarea', { From 9fb348f3a4f9e360c5addbe958b6c9d7e5d0dea4 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 1 Aug 2014 13:21:15 +0200 Subject: [PATCH 031/202] Don't try to ignore None release --- couchpotato/core/plugins/release/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 7b47aa9..9def40d 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -234,8 +234,9 @@ class Release(Plugin): db = get_db() try: - rel = db.get('id', id, with_doc = True) - self.updateStatus(id, 'available' if rel['status'] in ['ignored', 'failed'] else 'ignored') + if id: + rel = db.get('id', id, with_doc = True) + self.updateStatus(id, 'available' if rel['status'] in ['ignored', 'failed'] else 'ignored') return { 'success': True From 9f12fe263616319e1bd946e082d969f552cd9eee Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 1 Aug 2014 13:23:00 +0200 Subject: [PATCH 032/202] Add edge meta for IE fix #3727 --- couchpotato/templates/index.html | 2 +- couchpotato/templates/login.html | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/couchpotato/templates/index.html b/couchpotato/templates/index.html index 0d8acbc..729f1e3 100644 --- a/couchpotato/templates/index.html +++ b/couchpotato/templates/index.html @@ -5,7 +5,7 @@ - + {% for url in fireEvent('clientscript.get_styles', as_html = True, location = 'front', single = True) %} {% end %} diff --git a/couchpotato/templates/login.html b/couchpotato/templates/login.html index 3562622..e33db2d 100644 --- a/couchpotato/templates/login.html +++ b/couchpotato/templates/login.html @@ -4,22 +4,24 @@ + + {% for url in fireEvent('clientscript.get_styles', as_html = True, location = 'front', single = True) %} {% end %} - + {% for url in fireEvent('clientscript.get_scripts', as_html = True, location = 'front', single = True) %} {% end %} - + CouchPotato @@ -35,4 +37,4 @@ - \ No newline at end of file + From 4126007cac7ab518e22adfefb1f908b35ab1f1f3 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 1 Aug 2014 13:29:39 +0200 Subject: [PATCH 033/202] Don't download 0 seed torrents fix #3728 --- couchpotato/core/plugins/release/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 9def40d..b1ecafd 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -365,7 +365,7 @@ class Release(Plugin): let_through = False filtered_results = [] - # If a single release comes through the "wait for", let through all + # Filter out ignored and other releases we don't want for rel in results: if rel['status'] in ['ignored', 'failed']: @@ -380,6 +380,11 @@ class Release(Plugin): log.info('Ignored, size "%sMB" to low: %s', (rel['size'], rel['name'])) continue + if 'seeders' in rel and rel.get('seeders') <= 0: + log.info('Ignored, no seeders: %s', (rel['name'])) + continue + + # If a single release comes through the "wait for", let through all rel['wait_for'] = False if quality_custom.get('index') != 0 and quality_custom.get('wait_for', 0) > 0 and rel.get('age') <= quality_custom.get('wait_for', 0): rel['wait_for'] = True From accf19bb26e4ab8758727232a1547df07bf5cfb3 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 1 Aug 2014 13:33:59 +0200 Subject: [PATCH 034/202] Different log level --- couchpotato/core/plugins/base.py | 2 +- couchpotato/core/plugins/renamer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index bc66123..2c7f9e1 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -157,7 +157,7 @@ class Plugin(object): os.rmdir(full_path) except: if show_error: - log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc())) + log.info2('Couldn\'t remove directory %s: %s', (full_path, traceback.format_exc())) try: os.rmdir(folder) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 9f6792a..ad72086 100644 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -807,7 +807,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) if os.name == 'nt' and self.conf('ntfs_permission'): os.popen('icacls "' + dest + '"* /reset /T') except: - log.error('Failed setting permissions for file: %s, %s', (dest, traceback.format_exc(1))) + log.debug('Failed setting permissions for file: %s, %s', (dest, traceback.format_exc(1))) except: log.error('Couldn\'t move file "%s" to "%s": %s', (old, dest, traceback.format_exc())) raise From 6897dab6477f0ce2e1ba05c6849272b772ea3ab7 Mon Sep 17 00:00:00 2001 From: fenduru Date: Sat, 9 Aug 2014 11:20:23 -0400 Subject: [PATCH 035/202] Give higher weight to freeleech torrents --- .../core/media/_base/providers/torrent/passthepopcorn.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/couchpotato/core/media/_base/providers/torrent/passthepopcorn.py b/couchpotato/core/media/_base/providers/torrent/passthepopcorn.py index 609ef2d..40a5567 100644 --- a/couchpotato/core/media/_base/providers/torrent/passthepopcorn.py +++ b/couchpotato/core/media/_base/providers/torrent/passthepopcorn.py @@ -64,6 +64,10 @@ class Base(TorrentProvider): torrentdesc += ' HQ' if self.conf('prefer_golden'): torrentscore += 5000 + if 'FreeleechType' in torrent: + torrentdesc += ' Freeleech' + if self.conf('prefer_freeleech'): + torrentscore += 7000 if 'Scene' in torrent and torrent['Scene']: torrentdesc += ' Scene' if self.conf('prefer_scene'): @@ -224,6 +228,14 @@ config = [{ 'description': 'Favors Golden Popcorn-releases over all other releases.' }, { + 'name': 'prefer_freeleech', + 'advanced': True, + 'type': 'bool', + 'label': 'Prefer Freeleech', + 'default': 1, + 'description': 'Favors torrents marked as freeleech over all other releases.' + }, + { 'name': 'prefer_scene', 'advanced': True, 'type': 'bool', From dd7de31e9f5c5f81dc7c1fede40da30672d1010a Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 14 Aug 2014 21:28:17 +0200 Subject: [PATCH 036/202] Update TorrentShack url fix #3797 --- .../core/media/_base/providers/torrent/torrentshack.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentshack.py b/couchpotato/core/media/_base/providers/torrent/torrentshack.py index 1af7e55..226993c 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentshack.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentshack.py @@ -13,12 +13,12 @@ log = CPLog(__name__) class Base(TorrentProvider): urls = { - 'test': 'https://torrentshack.net/', - 'login': 'https://torrentshack.net/login.php', - 'login_check': 'https://torrentshack.net/inbox.php', - 'detail': 'https://torrentshack.net/torrent/%s', - 'search': 'https://torrentshack.net/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1', - 'download': 'https://torrentshack.net/%s', + 'test': 'http://torrentshack.eu/', + 'login': 'http://torrentshack.eu/login.php', + 'login_check': 'http://torrentshack.eu/inbox.php', + 'detail': 'http://torrentshack.eu/torrent/%s', + 'search': 'http://torrentshack.eu/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1', + 'download': 'http://torrentshack.eu/%s', } http_time_between_calls = 1 # Seconds @@ -80,7 +80,7 @@ config = [{ 'tab': 'searcher', 'list': 'torrent_providers', 'name': 'TorrentShack', - 'description': 'TorrentShack', + 'description': 'TorrentShack', 'wizard': True, 'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAABmElEQVQoFQXBzY2cVRiE0afqvd84CQiAnxWWtyxsS6ThINBYg2Dc7mZBMEjE4mzs6e9WcY5+ePNuVFJJodQAoLo+SaWCy9rcV8cmjah3CI6iYu7oRU30kE5xxELRfamklY3k1NL19sSm7vPzP/ZdNZzKVDaY2sPZJBh9fv5ITrmG2+Vp4e1sPchVqTCQZJnVXi+/L4uuAJGly1+Pw8CprLbi8Om7tbT19/XRqJUk11JP9uHj9ulxhXbvJbI9qJvr5YkGXFG2IBT8tXczt+sfzDZCp3765f3t9tHEHGEDACma77+8o4oATKk+/PfW9YmHruRFjWoVSFsVsGu1YSKq6Oc37+n98unPZSRlY7vsKDqN+92X3yR9+PdXee3iJNKMStqdcZqoTJbUSi5JOkpfRlhSI0mSpEmCFKoU7FqSNOLAk54uGwCStMUCgLrVic62g7oDoFmmdI+P3S0pDe1xvDqb6XrZqbtzShWNoh9fv/XQHaDdM9OqrZi2M7M3UrB2vlkPS1IbdEBk7UiSoD6VlZ6aKWer4aH4f/AvKoHUTjuyAAAAAElFTkSuQmCC', 'options': [ From bdadd00d93247ec364b820d1e1d66a90118f0b69 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 16 Aug 2014 12:44:52 +0200 Subject: [PATCH 037/202] Don't add & on url creation --- couchpotato/core/media/_base/providers/nzb/newznab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/_base/providers/nzb/newznab.py b/couchpotato/core/media/_base/providers/nzb/newznab.py index b1a8ec6..81622ad 100644 --- a/couchpotato/core/media/_base/providers/nzb/newznab.py +++ b/couchpotato/core/media/_base/providers/nzb/newznab.py @@ -45,7 +45,7 @@ class Base(NZBProvider, RSS): def _searchOnHost(self, host, media, quality, results): query = self.buildUrl(media, host) - url = '%s&%s' % (self.getUrl(host['host']), query) + url = '%s%s' % (self.getUrl(host['host']), query) nzbs = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()}) for nzb in nzbs: @@ -83,7 +83,7 @@ class Base(NZBProvider, RSS): try: # Get details for extended description to retrieve passwords query = self.buildDetailsUrl(nzb_id, host['api_key']) - url = '%s&%s' % (self.getUrl(host['host']), query) + url = '%s%s' % (self.getUrl(host['host']), query) nzb_details = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})[0] description = self.getTextElement(nzb_details, 'description') From e3414fe91f8c155adecf13f4754448508995f3e0 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 16 Aug 2014 13:29:34 +0200 Subject: [PATCH 038/202] Remove movie if no releases are left after delete --- couchpotato/core/media/_base/media/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index ee9e6cc..036badc 100644 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -415,7 +415,7 @@ class MediaPlugin(MediaBase): db.delete(release) total_deleted += 1 - if (total_releases == total_deleted and media['status'] != 'active') or (total_releases == 0 and not new_media_status) or (not new_media_status and delete_from == 'late'): + if (total_releases == total_deleted) or (total_releases == 0 and not new_media_status) or (not new_media_status and delete_from == 'late'): db.delete(media) deleted = True elif new_media_status: From 89f3b6624e148d707c0abd795592480d916806a1 Mon Sep 17 00:00:00 2001 From: dkboy Date: Sun, 17 Aug 2014 14:52:43 +1200 Subject: [PATCH 039/202] Update bitsoup.py They've rearranged the table --- couchpotato/core/media/_base/providers/torrent/bitsoup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/bitsoup.py b/couchpotato/core/media/_base/providers/torrent/bitsoup.py index 9519e58..392706c 100644 --- a/couchpotato/core/media/_base/providers/torrent/bitsoup.py +++ b/couchpotato/core/media/_base/providers/torrent/bitsoup.py @@ -49,9 +49,9 @@ class Base(TorrentProvider): torrent_name = torrent.getText() - torrent_size = self.parseSize(all_cells[7].getText()) - torrent_seeders = tryInt(all_cells[9].getText()) - torrent_leechers = tryInt(all_cells[10].getText()) + torrent_size = self.parseSize(all_cells[8].getText()) + torrent_seeders = tryInt(all_cells[10].getText()) + torrent_leechers = tryInt(all_cells[11].getText()) torrent_url = self.urls['baseurl'] % download['href'] torrent_detail_url = self.urls['baseurl'] % torrent['href'] From e7be5c7809ec1db9e0589e685741a1b83b74a938 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Fri, 25 Jul 2014 10:52:53 +1200 Subject: [PATCH 040/202] Added "library.related" event and "library.query", "library.related" API calls --- couchpotato/core/media/_base/library/main.py | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/couchpotato/core/media/_base/library/main.py b/couchpotato/core/media/_base/library/main.py index a723de5..2eccfce 100644 --- a/couchpotato/core/media/_base/library/main.py +++ b/couchpotato/core/media/_base/library/main.py @@ -1,10 +1,35 @@ +from couchpotato import get_db +from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.logger import CPLog from couchpotato.core.media._base.library.base import LibraryBase +log = CPLog(__name__) + class Library(LibraryBase): def __init__(self): addEvent('library.title', self.title) + addEvent('library.related', self.related) + + addApiView('library.query', self.queryView) + addApiView('library.related', self.relatedView) + + def queryView(self, media_id, **kwargs): + db = get_db() + media = db.get('id', media_id) + + return { + 'result': fireEvent('library.query', media, single = True) + } + + def relatedView(self, media_id, **kwargs): + db = get_db() + media = db.get('id', media_id) + + return { + 'result': fireEvent('library.related', media, single = True) + } def title(self, library): return fireEvent( @@ -16,3 +41,16 @@ class Library(LibraryBase): include_identifier = False, single = True ) + + def related(self, media): + result = {media['type']: media} + + db = get_db() + cur = media + + while cur and cur.get('parent_id'): + cur = db.get('id', cur['parent_id']) + + result[cur['type']] = cur + + return result From a1ce3e0d6ba5633992ad5f69f90c8b31ebfda8e0 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Wed, 20 Aug 2014 14:54:24 +1200 Subject: [PATCH 041/202] Added "library.root" event, fixes to "matcher", "release" and "score" to use "library.root" + handle missing "year" --- couchpotato/core/media/_base/library/main.py | 10 ++++++++++ couchpotato/core/media/_base/matcher/main.py | 6 +++--- couchpotato/core/plugins/release/main.py | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/media/_base/library/main.py b/couchpotato/core/media/_base/library/main.py index 2eccfce..b90f4c3 100644 --- a/couchpotato/core/media/_base/library/main.py +++ b/couchpotato/core/media/_base/library/main.py @@ -11,6 +11,7 @@ class Library(LibraryBase): def __init__(self): addEvent('library.title', self.title) addEvent('library.related', self.related) + addEvent('library.root', self.root) addApiView('library.query', self.queryView) addApiView('library.related', self.relatedView) @@ -54,3 +55,12 @@ class Library(LibraryBase): result[cur['type']] = cur return result + + def root(self, media): + db = get_db() + cur = media + + while cur and cur.get('parent_id'): + cur = db.get('id', cur['parent_id']) + + return cur diff --git a/couchpotato/core/media/_base/matcher/main.py b/couchpotato/core/media/_base/matcher/main.py index 2034249..64e13ae 100644 --- a/couchpotato/core/media/_base/matcher/main.py +++ b/couchpotato/core/media/_base/matcher/main.py @@ -40,7 +40,7 @@ class Matcher(MatcherBase): return False def correctTitle(self, chain, media): - root_library = media['library']['root_library'] + root = fireEvent('library.root', media, single = True) if 'show_name' not in chain.info or not len(chain.info['show_name']): log.info('Wrong: missing show name in parsed result') @@ -50,10 +50,10 @@ class Matcher(MatcherBase): chain_words = [x.lower() for x in chain.info['show_name']] # Build a list of possible titles of the media we are searching for - titles = root_library['info']['titles'] + titles = root['info']['titles'] # Add year suffix titles (will result in ['', ' ', '', ...]) - suffixes = [None, root_library['info']['year']] + suffixes = [None, root['info']['year']] titles = [ title + ((' %s' % suffix) if suffix else '') diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index b1ecafd..e4bfdc2 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -325,7 +325,7 @@ class Release(Plugin): rls['download_info'] = download_result db.update(rls) - log_movie = '%s (%s) in %s' % (getTitle(media), media['info']['year'], rls['quality']) + log_movie = '%s (%s) in %s' % (getTitle(media), media['info'].get('year'), rls['quality']) snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie) log.info(snatch_message) fireEvent('%s.snatched' % data['type'], message = snatch_message, data = media) From a821d85bf2c80b3c30e0bfb75e29c6e2694e3e5f Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Fri, 25 Jul 2014 13:44:18 +1200 Subject: [PATCH 042/202] Fixed MediaBase.getPoster(), switched MovieBase to use this generic method --- couchpotato/core/media/__init__.py | 9 ++++++--- couchpotato/core/media/movie/_base/main.py | 32 +++--------------------------- 2 files changed, 9 insertions(+), 32 deletions(-) mode change 100644 => 100755 couchpotato/core/media/__init__.py mode change 100644 => 100755 couchpotato/core/media/movie/_base/main.py diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py old mode 100644 new mode 100755 index 4a3eb68..4e319fc --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -65,10 +65,13 @@ class MediaBase(Plugin): return def_title or 'UNKNOWN' - def getPoster(self, image_urls, existing_files): - image_type = 'poster' + def getPoster(self, media, image_urls): + if 'files' not in media: + media['files'] = {} + + existing_files = media['files'] - # Remove non-existing files + image_type = 'poster' file_type = 'image_%s' % image_type # Make existing unique diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py old mode 100644 new mode 100755 index 336d803..3afe1e8 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -312,37 +312,11 @@ class MovieBase(MovieTypeBase): media['title'] = def_title # Files - images = info.get('images', []) - media['files'] = media.get('files', {}) - for image_type in ['poster']: - - # Remove non-existing files - file_type = 'image_%s' % image_type - existing_files = list(set(media['files'].get(file_type, []))) - for ef in media['files'].get(file_type, []): - if not os.path.isfile(ef): - existing_files.remove(ef) - - # Replace new files list - media['files'][file_type] = existing_files - if len(existing_files) == 0: - del media['files'][file_type] - - # Loop over type - for image in images.get(image_type, []): - if not isinstance(image, (str, unicode)): - continue - - if file_type not in media['files'] or len(media['files'].get(file_type, [])) == 0: - file_path = fireEvent('file.download', url = image, single = True) - if file_path: - media['files'][file_type] = [file_path] - break - else: - break + image_urls = info.get('images', []) - db.update(media) + self.getPoster(media, image_urls) + db.update(media) return media except: log.error('Failed update media: %s', traceback.format_exc()) From a366d57278c8dd6c95953a4d989011b16fffb2cb Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Fri, 25 Jul 2014 16:56:09 +1200 Subject: [PATCH 043/202] Added "library.tree" event/api call --- couchpotato/core/media/_base/library/main.py | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) mode change 100644 => 100755 couchpotato/core/media/_base/library/main.py diff --git a/couchpotato/core/media/_base/library/main.py b/couchpotato/core/media/_base/library/main.py old mode 100644 new mode 100755 index b90f4c3..d20342f --- a/couchpotato/core/media/_base/library/main.py +++ b/couchpotato/core/media/_base/library/main.py @@ -11,10 +11,13 @@ class Library(LibraryBase): def __init__(self): addEvent('library.title', self.title) addEvent('library.related', self.related) + addEvent('library.tree', self.tree) + addEvent('library.root', self.root) addApiView('library.query', self.queryView) addApiView('library.related', self.relatedView) + addApiView('library.tree', self.treeView) def queryView(self, media_id, **kwargs): db = get_db() @@ -32,6 +35,14 @@ class Library(LibraryBase): 'result': fireEvent('library.related', media, single = True) } + def treeView(self, media_id, **kwargs): + db = get_db() + media = db.get('id', media_id) + + return { + 'result': fireEvent('library.tree', media, single = True) + } + def title(self, library): return fireEvent( 'library.query', @@ -64,3 +75,33 @@ class Library(LibraryBase): cur = db.get('id', cur['parent_id']) return cur + + def tree(self, media): + result = media + + db = get_db() + + # TODO this probably should be using an index? + items = [ + item['doc'] + for item in db.all('media', with_doc = True) + if item['doc'].get('parent_id') == media['_id'] + ] + + keys = [] + + for item in items: + key = item['type'] + 's' + + if key not in result: + result[key] = {} + + if key not in keys: + keys.append(key) + + result[key][item['_id']] = fireEvent('library.tree', item, single = True) + + for key in keys: + result[key] = result[key].values() + + return result From f2b0d3f80bcd41f062ee3a68e9de4b959c5ec719 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sat, 26 Jul 2014 13:54:55 +1200 Subject: [PATCH 044/202] Switched "library.tree" to use "media_children" index --- couchpotato/core/media/_base/library/main.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/couchpotato/core/media/_base/library/main.py b/couchpotato/core/media/_base/library/main.py index d20342f..c0e4464 100755 --- a/couchpotato/core/media/_base/library/main.py +++ b/couchpotato/core/media/_base/library/main.py @@ -80,18 +80,12 @@ class Library(LibraryBase): result = media db = get_db() - - # TODO this probably should be using an index? - items = [ - item['doc'] - for item in db.all('media', with_doc = True) - if item['doc'].get('parent_id') == media['_id'] - ] + items = db.get_many('media_children', media['_id'], with_doc = True) keys = [] for item in items: - key = item['type'] + 's' + key = item['doc']['type'] + 's' if key not in result: result[key] = {} @@ -99,7 +93,7 @@ class Library(LibraryBase): if key not in keys: keys.append(key) - result[key][item['_id']] = fireEvent('library.tree', item, single = True) + result[key][item['_id']] = fireEvent('library.tree', item['doc'], single = True) for key in keys: result[key] = result[key].values() From d75f58f5ecb5042e58ac6cd59af3877ea2be9c34 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sat, 26 Jul 2014 22:29:16 +1200 Subject: [PATCH 045/202] Fixed "library.related" and "libary.tree" to work with "show.episode", 'show.season" media types --- couchpotato/core/media/_base/library/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/_base/library/main.py b/couchpotato/core/media/_base/library/main.py index c0e4464..b565d16 100755 --- a/couchpotato/core/media/_base/library/main.py +++ b/couchpotato/core/media/_base/library/main.py @@ -63,7 +63,9 @@ class Library(LibraryBase): while cur and cur.get('parent_id'): cur = db.get('id', cur['parent_id']) - result[cur['type']] = cur + parts = cur['type'].split('.') + + result[parts[-1]] = cur return result @@ -85,7 +87,8 @@ class Library(LibraryBase): keys = [] for item in items: - key = item['doc']['type'] + 's' + parts = item['doc']['type'].split('.') + key = parts[-1] + 's' if key not in result: result[key] = {} From cd836f36606f0666de844ff430e41f8f904f930a Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sun, 27 Jul 2014 01:03:58 +1200 Subject: [PATCH 046/202] Include releases in "library.tree" --- couchpotato/core/media/_base/library/main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/library/main.py b/couchpotato/core/media/_base/library/main.py index b565d16..2f5629d 100755 --- a/couchpotato/core/media/_base/library/main.py +++ b/couchpotato/core/media/_base/library/main.py @@ -82,10 +82,12 @@ class Library(LibraryBase): result = media db = get_db() - items = db.get_many('media_children', media['_id'], with_doc = True) + # Find children + items = db.get_many('media_children', media['_id'], with_doc = True) keys = [] + # Build children arrays for item in items: parts = item['doc']['type'].split('.') key = parts[-1] + 's' @@ -98,7 +100,11 @@ class Library(LibraryBase): result[key][item['_id']] = fireEvent('library.tree', item['doc'], single = True) + # Unique children for key in keys: result[key] = result[key].values() + # Include releases + result['releases'] = fireEvent('release.for_media', media['_id'], single = True) + return result From 058846f54fabd42529c8172280c46a5189251d16 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Mon, 28 Jul 2014 16:06:05 +1200 Subject: [PATCH 047/202] Added "find" helper function --- couchpotato/core/helpers/variable.py | 8 ++++++++ 1 file changed, 8 insertions(+) mode change 100644 => 100755 couchpotato/core/helpers/variable.py diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py old mode 100644 new mode 100755 index fc844aa..db68da2 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -380,3 +380,11 @@ def getFreeSpace(directories): free_space[folder] = size return free_space + + +def find(func, iterable): + for item in iterable: + if func(item): + return item + + return None From 3869e350bf14735298eaafc9026fd5ef11276a7d Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Mon, 28 Jul 2014 16:06:51 +1200 Subject: [PATCH 048/202] Added "media_id" parameter to "library.tree" event --- couchpotato/core/media/_base/library/main.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/media/_base/library/main.py b/couchpotato/core/media/_base/library/main.py index 2f5629d..da526ec 100755 --- a/couchpotato/core/media/_base/library/main.py +++ b/couchpotato/core/media/_base/library/main.py @@ -78,13 +78,18 @@ class Library(LibraryBase): return cur - def tree(self, media): - result = media - + def tree(self, media = None, media_id = None): db = get_db() + if media: + result = media + elif media_id: + result = db.get('id', media_id, with_doc = True) + else: + return None + # Find children - items = db.get_many('media_children', media['_id'], with_doc = True) + items = db.get_many('media_children', result['_id'], with_doc = True) keys = [] # Build children arrays @@ -94,6 +99,8 @@ class Library(LibraryBase): if key not in result: result[key] = {} + elif type(result[key]) is not dict: + result[key] = {} if key not in keys: keys.append(key) @@ -105,6 +112,6 @@ class Library(LibraryBase): result[key] = result[key].values() # Include releases - result['releases'] = fireEvent('release.for_media', media['_id'], single = True) + result['releases'] = fireEvent('release.for_media', result['_id'], single = True) return result From 072b6d09fad404ddba9b97fbac5f78bd9afe9dcf Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Mon, 28 Jul 2014 19:54:06 +1200 Subject: [PATCH 049/202] Renamed "[media].update_info" event to "[media].update" --- couchpotato/core/media/_base/media/main.py | 2 +- couchpotato/core/media/movie/_base/main.py | 8 ++++---- couchpotato/core/media/movie/providers/metadata/base.py | 2 +- couchpotato/core/media/movie/searcher.py | 2 +- couchpotato/core/plugins/manage.py | 2 +- couchpotato/core/plugins/renamer.py | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) mode change 100644 => 100755 couchpotato/core/media/_base/media/main.py mode change 100644 => 100755 couchpotato/core/media/movie/providers/metadata/base.py mode change 100644 => 100755 couchpotato/core/media/movie/searcher.py mode change 100644 => 100755 couchpotato/core/plugins/manage.py mode change 100644 => 100755 couchpotato/core/plugins/renamer.py diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py old mode 100644 new mode 100755 index ee9e6cc..a5a1bbd --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -109,7 +109,7 @@ class MediaPlugin(MediaBase): try: media = get_db().get('id', media_id) - event = '%s.update_info' % media.get('type') + event = '%s.update' % media.get('type') def handler(): fireEvent(event, media_id = media_id, on_complete = self.createOnComplete(media_id)) diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 3afe1e8..8a04d0b 100755 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -46,7 +46,7 @@ class MovieBase(MovieTypeBase): }) addEvent('movie.add', self.add) - addEvent('movie.update_info', self.updateInfo) + addEvent('movie.update', self.update) addEvent('movie.update_release_dates', self.updateReleaseDate) def add(self, params = None, force_readd = True, search_after = True, update_after = True, notify_after = True, status = None): @@ -172,7 +172,7 @@ class MovieBase(MovieTypeBase): # Trigger update info if added and update_after: # Do full update to get images etc - fireEventAsync('movie.update_info', m['_id'], default_title = params.get('title'), on_complete = onComplete) + fireEventAsync('movie.update', m['_id'], default_title = params.get('title'), on_complete = onComplete) # Remove releases for rel in fireEvent('release.for_media', m['_id'], single = True): @@ -256,7 +256,7 @@ class MovieBase(MovieTypeBase): 'success': False, } - def updateInfo(self, media_id = None, identifier = None, default_title = None, extended = False): + def update(self, media_id = None, identifier = None, default_title = None, extended = False): """ Update movie information inside media['doc']['info'] @@ -337,7 +337,7 @@ class MovieBase(MovieTypeBase): media = db.get('id', media_id) if not media.get('info'): - media = self.updateInfo(media_id) + media = self.update(media_id) dates = media.get('info', {}).get('release_date') else: dates = media.get('info').get('release_date') diff --git a/couchpotato/core/media/movie/providers/metadata/base.py b/couchpotato/core/media/movie/providers/metadata/base.py old mode 100644 new mode 100755 index 7968000..cc914af --- a/couchpotato/core/media/movie/providers/metadata/base.py +++ b/couchpotato/core/media/movie/providers/metadata/base.py @@ -28,7 +28,7 @@ class MovieMetaData(MetaDataBase): # Update library to get latest info try: - group['media'] = fireEvent('movie.update_info', group['media'].get('_id'), identifier = getIdentifier(group['media']), extended = True, single = True) + group['media'] = fireEvent('movie.update', group['media'].get('_id'), identifier = getIdentifier(group['media']), extended = True, single = True) except: log.error('Failed to update movie, before creating metadata: %s', traceback.format_exc()) diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py old mode 100644 new mode 100755 index 4bd8c8d..e943c21 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -94,7 +94,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): self.single(media, search_protocols, manual = manual) except IndexError: log.error('Forcing library update for %s, if you see this often, please report: %s', (getIdentifier(media), traceback.format_exc())) - fireEvent('movie.update_info', media_id) + fireEvent('movie.update', media_id) except: log.error('Search failed for %s: %s', (getIdentifier(media), traceback.format_exc())) diff --git a/couchpotato/core/plugins/manage.py b/couchpotato/core/plugins/manage.py old mode 100644 new mode 100755 index c8d53ea..d389e22 --- a/couchpotato/core/plugins/manage.py +++ b/couchpotato/core/plugins/manage.py @@ -219,7 +219,7 @@ class Manage(Plugin): # Add it to release and update the info fireEvent('release.add', group = group, update_info = False) - fireEvent('movie.update_info', identifier = group['identifier'], on_complete = self.createAfterUpdate(folder, group['identifier'])) + fireEvent('movie.update', identifier = group['identifier'], on_complete = self.createAfterUpdate(folder, group['identifier'])) return addToLibrary diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py old mode 100644 new mode 100755 index ad72086..d47d87f --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -247,7 +247,7 @@ class Renamer(Plugin): 'profile_id': None }, search_after = False, status = 'done', single = True) else: - group['media'] = fireEvent('movie.update_info', media_id = group['media'].get('_id'), single = True) + group['media'] = fireEvent('movie.update', media_id = group['media'].get('_id'), single = True) if not group['media'] or not group['media'].get('_id'): log.error('Could not rename, no library item to work with: %s', group_identifier) From 7df92f2882716b95f1fba4160439b4a6bb120e74 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Mon, 28 Jul 2014 20:33:50 +1200 Subject: [PATCH 050/202] Fix possible dashboard error, add "types" parameter to "media.with_status", limit suggestions to movies (for now) --- couchpotato/core/media/_base/media/main.py | 9 ++++++++- couchpotato/core/media/movie/suggestion/main.py | 2 +- couchpotato/core/plugins/dashboard.py | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) mode change 100644 => 100755 couchpotato/core/media/movie/suggestion/main.py diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index a5a1bbd..bcb8402 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -160,10 +160,13 @@ class MediaPlugin(MediaBase): 'media': media, } - def withStatus(self, status, with_doc = True): + def withStatus(self, status, types = None, with_doc = True): db = get_db() + if types and not isinstance(types, (list, tuple)): + types = [types] + status = list(status if isinstance(status, (list, tuple)) else [status]) for s in status: @@ -171,6 +174,10 @@ class MediaPlugin(MediaBase): if with_doc: try: doc = db.get('id', ms['_id']) + + if types and doc.get('type') not in types: + continue + yield doc except RecordNotFound: log.debug('Record not found, skipping: %s', ms['_id']) diff --git a/couchpotato/core/media/movie/suggestion/main.py b/couchpotato/core/media/movie/suggestion/main.py old mode 100644 new mode 100755 index 146a6a0..3df67ab --- a/couchpotato/core/media/movie/suggestion/main.py +++ b/couchpotato/core/media/movie/suggestion/main.py @@ -27,7 +27,7 @@ class Suggestion(Plugin): else: if not movies or len(movies) == 0: - active_movies = fireEvent('media.with_status', ['active', 'done'], single = True) + active_movies = fireEvent('media.with_status', ['active', 'done'], 'movie', single = True) movies = [getIdentifier(x) for x in active_movies] if not ignored or len(ignored) == 0: diff --git a/couchpotato/core/plugins/dashboard.py b/couchpotato/core/plugins/dashboard.py index 776f24e..d8ea4c9 100644 --- a/couchpotato/core/plugins/dashboard.py +++ b/couchpotato/core/plugins/dashboard.py @@ -62,7 +62,7 @@ class Dashboard(Plugin): for media_id in active_ids: media = db.get('id', media_id) - pp = profile_pre.get(media['profile_id']) + pp = profile_pre.get(media.get('profile_id')) if not pp: continue eta = media['info'].get('release_date', {}) or {} From e2df3a4dfd6fc8e39a93e07e232799962aff4e10 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sun, 3 Aug 2014 22:04:49 +1200 Subject: [PATCH 051/202] Added children to "library.related" --- couchpotato/core/media/_base/library/main.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/media/_base/library/main.py b/couchpotato/core/media/_base/library/main.py index da526ec..9e614fb 100755 --- a/couchpotato/core/media/_base/library/main.py +++ b/couchpotato/core/media/_base/library/main.py @@ -55,7 +55,7 @@ class Library(LibraryBase): ) def related(self, media): - result = {media['type']: media} + result = {self.key(media['type']): media} db = get_db() cur = media @@ -63,9 +63,17 @@ class Library(LibraryBase): while cur and cur.get('parent_id'): cur = db.get('id', cur['parent_id']) - parts = cur['type'].split('.') + result[self.key(cur['type'])] = cur - result[parts[-1]] = cur + children = db.get_many('media_children', media['_id'], with_doc = True) + + for item in children: + key = self.key(item['doc']['type']) + 's' + + if key not in result: + result[key] = [] + + result[key].append(item['doc']) return result @@ -94,8 +102,7 @@ class Library(LibraryBase): # Build children arrays for item in items: - parts = item['doc']['type'].split('.') - key = parts[-1] + 's' + key = self.key(item['doc']['type']) + 's' if key not in result: result[key] = {} @@ -115,3 +122,7 @@ class Library(LibraryBase): result['releases'] = fireEvent('release.for_media', result['_id'], single = True) return result + + def key(self, media_type): + parts = media_type.split('.') + return parts[-1] From 81d4d9a4e255de40bae424cb50eb428be0471227 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Thu, 7 Aug 2014 14:05:05 +1200 Subject: [PATCH 052/202] Changed "media.with_identifiers" to remove "No media found with..." messages --- couchpotato/core/media/_base/media/main.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index bcb8402..7021f82 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -185,17 +185,15 @@ class MediaPlugin(MediaBase): yield ms def withIdentifiers(self, identifiers, with_doc = False): - db = get_db() for x in identifiers: try: - media = db.get('media', '%s-%s' % (x, identifiers[x]), with_doc = with_doc) - return media + return db.get('media', '%s-%s' % (x, identifiers[x]), with_doc = with_doc) except: pass - log.debug('No media found with identifiers: %s', identifiers) + return False def list(self, types = None, status = None, release_status = None, status_or = False, limit_offset = None, with_tags = None, starts_with = None, search = None): From c381b719b111ff41974d778014b5b2ecd673bd8d Mon Sep 17 00:00:00 2001 From: seedzero Date: Tue, 29 Jul 2014 18:13:02 +1000 Subject: [PATCH 053/202] Stop movie searcher searching for TV shows and hosing episodes --- couchpotato/core/media/movie/searcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index e943c21..a8d6fe5 100755 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -74,7 +74,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): self.in_progress = True fireEvent('notify.frontend', type = 'movie.searcher.started', data = True, message = 'Full search started') - medias = [x['_id'] for x in fireEvent('media.with_status', 'active', with_doc = False, single = True)] + medias = [x['_id'] for x in fireEvent('media.with_status', 'active', 'movie', single = True)] random.shuffle(medias) total = len(medias) From 436883a96d09bbe9007c4eea8baf711b1ed28537 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Wed, 20 Aug 2014 14:59:57 +1200 Subject: [PATCH 054/202] Fixed media.types & addSingleListView addSingleCharView, addSingleDeleteView Conflicts: couchpotato/core/media/show/_base/main.py --- couchpotato/core/media/_base/media/main.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 7021f82..8adc666 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -312,8 +312,7 @@ class MediaPlugin(MediaBase): def addSingleListView(self): for media_type in fireEvent('media.types', merge = True): - def tempList(*args, **kwargs): - return self.listView(types = media_type, **kwargs) + tempList = lambda media_type = media_type, *args, **kwargs : self.listView(type = media_type, **kwargs) addApiView('%s.list' % media_type, tempList) def availableChars(self, types = None, status = None, release_status = None): @@ -381,8 +380,7 @@ class MediaPlugin(MediaBase): def addSingleCharView(self): for media_type in fireEvent('media.types', merge = True): - def tempChar(*args, **kwargs): - return self.charView(types = media_type, **kwargs) + tempChar = lambda media_type = media_type, *args, **kwargs : self.charView(type = media_type, **kwargs) addApiView('%s.available_chars' % media_type, tempChar) def delete(self, media_id, delete_from = None): @@ -451,8 +449,7 @@ class MediaPlugin(MediaBase): def addSingleDeleteView(self): for media_type in fireEvent('media.types', merge = True): - def tempDelete(*args, **kwargs): - return self.deleteView(types = media_type, *args, **kwargs) + tempDelete = lambda media_type = media_type, *args, **kwargs : self.deleteView(type = media_type, **kwargs) addApiView('%s.delete' % media_type, tempDelete) def restatus(self, media_id): From cfd92b826869791b46142bff956642d7fbd5c876 Mon Sep 17 00:00:00 2001 From: seedzero Date: Tue, 19 Aug 2014 01:36:12 +1000 Subject: [PATCH 055/202] Documentation added for media type .list & .delete APIs --- couchpotato/core/media/_base/media/main.py | 36 +++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 8adc666..26140f3 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -44,15 +44,15 @@ class MediaPlugin(MediaBase): 'desc': 'List media', 'params': { 'type': {'type': 'string', 'desc': 'Media type to filter on.'}, - 'status': {'type': 'array or csv', 'desc': 'Filter movie by status. Example:"active,done"'}, - 'release_status': {'type': 'array or csv', 'desc': 'Filter movie by status of its releases. Example:"snatched,available"'}, - 'limit_offset': {'desc': 'Limit and offset the movie list. Examples: "50" or "50,30"'}, - 'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all movies starting with the letter "a"'}, - 'search': {'desc': 'Search movie title'}, + 'status': {'type': 'array or csv', 'desc': 'Filter media by status. Example:"active,done"'}, + 'release_status': {'type': 'array or csv', 'desc': 'Filter media by status of its releases. Example:"snatched,available"'}, + 'limit_offset': {'desc': 'Limit and offset the media list. Examples: "50" or "50,30"'}, + 'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all media starting with the letter "a"'}, + 'search': {'desc': 'Search media title'}, }, 'return': {'type': 'object', 'example': """{ 'success': True, - 'empty': bool, any movies returned or not, + 'empty': bool, any media returned or not, 'media': array, media found, }"""} }) @@ -313,7 +313,21 @@ class MediaPlugin(MediaBase): for media_type in fireEvent('media.types', merge = True): tempList = lambda media_type = media_type, *args, **kwargs : self.listView(type = media_type, **kwargs) - addApiView('%s.list' % media_type, tempList) + addApiView('%s.list' % media_type, tempList, docs = { + 'desc': 'List media', + 'params': { + 'status': {'type': 'array or csv', 'desc': 'Filter ' + media_type + ' by status. Example:"active,done"'}, + 'release_status': {'type': 'array or csv', 'desc': 'Filter ' + media_type + ' by status of its releases. Example:"snatched,available"'}, + 'limit_offset': {'desc': 'Limit and offset the ' + media_type + ' list. Examples: "50" or "50,30"'}, + 'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all ' + media_type + 's starting with the letter "a"'}, + 'search': {'desc': 'Search ' + media_type + ' title'}, + }, + 'return': {'type': 'object', 'example': """{ + 'success': True, + 'empty': bool, any """ + media_type + """s returned or not, + 'media': array, media found, + }"""} + }) def availableChars(self, types = None, status = None, release_status = None): @@ -450,7 +464,13 @@ class MediaPlugin(MediaBase): for media_type in fireEvent('media.types', merge = True): tempDelete = lambda media_type = media_type, *args, **kwargs : self.deleteView(type = media_type, **kwargs) - addApiView('%s.delete' % media_type, tempDelete) + addApiView('%s.delete' % media_type, tempDelete, docs = { + 'desc': 'Delete a ' + media_type + ' from the wanted list', + 'params': { + 'id': {'desc': 'Media ID(s) you want to delete.', 'type': 'int (comma separated)'}, + 'delete_from': {'desc': 'Delete ' + media_type + ' from this page', 'type': 'string: all (default), wanted, manage'}, + } + }) def restatus(self, media_id): From c0492a41d90bb3c336642054b732412fe57d3164 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Fri, 22 Aug 2014 11:58:29 +0930 Subject: [PATCH 056/202] Ignore separator at end of string. Fixes #3823 --- couchpotato/core/plugins/renamer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index ad72086..238f150 100644 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -847,7 +847,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) replaces = [ ('\.+', '.'), ('_+', '_'), ('-+', '-'), ('\s+', ' '), (' \\\\', '\\\\'), (' /', '/'), - ('(\s\.)+', '.'), ('(-\.)+', '.'), ('(\s-)+', '-'), + ('(\s\.)+', '.'), ('(-\.)+', '.'), ('(\s-)+', '-'), ('([\s\.\,\_\-\/\\]$)', ''), ] for r in replaces: From c545c9aab1dd0c809d6e89689ae13e30796eff92 Mon Sep 17 00:00:00 2001 From: Andrew Parker Date: Mon, 25 Aug 2014 09:43:06 -0600 Subject: [PATCH 057/202] Couchpotato will identify a release with 'hdtv' in the name as brrip. The passthepopcorn provider searched for brrip with the parameters media=Blu-ray. This created a condition where a 720p hdtv movie would not be snatched because the 720p quality would skip it due to being marked as 'brrip', and br-rip quality would not find it because it was not from a Blu-ray source. This changes the search parameters for brrip to resolution=anyhd to remove the requirement that it's source media is Blu-ray. --- couchpotato/core/media/movie/providers/torrent/passthepopcorn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/movie/providers/torrent/passthepopcorn.py b/couchpotato/core/media/movie/providers/torrent/passthepopcorn.py index bbaea26..2b577ad 100644 --- a/couchpotato/core/media/movie/providers/torrent/passthepopcorn.py +++ b/couchpotato/core/media/movie/providers/torrent/passthepopcorn.py @@ -13,7 +13,7 @@ class PassThePopcorn(MovieProvider, Base): 'bd50': {'media': 'Blu-ray', 'format': 'BD50'}, '1080p': {'resolution': '1080p'}, '720p': {'resolution': '720p'}, - 'brrip': {'media': 'Blu-ray'}, + 'brrip': {'resolution': 'anyhd'}, 'dvdr': {'resolution': 'anysd'}, 'dvdrip': {'media': 'DVD'}, 'scr': {'media': 'DVD-Screener'}, @@ -27,7 +27,7 @@ class PassThePopcorn(MovieProvider, Base): 'bd50': {'Codec': ['BD50']}, '1080p': {'Resolution': ['1080p']}, '720p': {'Resolution': ['720p']}, - 'brrip': {'Source': ['Blu-ray'], 'Quality': ['High Definition'], 'Container': ['!ISO']}, + 'brrip': {'Quality': ['High Definition'], 'Container': ['!ISO']}, 'dvdr': {'Codec': ['DVD5', 'DVD9']}, 'dvdrip': {'Source': ['DVD'], 'Codec': ['!DVD5', '!DVD9']}, 'scr': {'Source': ['DVD-Screener']}, From b4a15f344d0a4437ac5a3575938274b6c4419567 Mon Sep 17 00:00:00 2001 From: Michael Walker Date: Tue, 26 Aug 2014 15:53:10 +0100 Subject: [PATCH 058/202] Updated URL's Replaced TPB links with working URL's --- .../core/media/_base/providers/torrent/thepiratebay.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/thepiratebay.py b/couchpotato/core/media/_base/providers/torrent/thepiratebay.py index 57bcfbd..6a1de8d 100644 --- a/couchpotato/core/media/_base/providers/torrent/thepiratebay.py +++ b/couchpotato/core/media/_base/providers/torrent/thepiratebay.py @@ -24,16 +24,16 @@ class Base(TorrentMagnetProvider): http_time_between_calls = 0 proxy_list = [ - 'https://nobay.net', + 'https://www.dieroschtibay.org', 'https://thebay.al', 'https://thepiratebay.se', - 'http://thepiratebay.cd', + 'http://thepiratebay.se.net', 'http://thebootlegbay.com', - 'http://www.tpb.gr', - 'http://tpbproxy.co.uk', + 'http://tpb.ninja.so', + 'http://proxybay.fr', 'http://pirateproxy.in', - 'http://www.getpirate.com', - 'http://piratebay.io', + 'http://piratebay.skey.sk', + 'http://pirateproxy.be', 'http://bayproxy.li', 'http://proxybay.pw', ] From 416c9eabdedd7a7ae509c72503d9c65f8c415721 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 27 Aug 2014 19:35:32 +0200 Subject: [PATCH 059/202] Update charts every 3 days --- couchpotato/core/media/movie/charts/__init__.py | 7 ------- couchpotato/core/media/movie/charts/main.py | 5 +++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/couchpotato/core/media/movie/charts/__init__.py b/couchpotato/core/media/movie/charts/__init__.py index 361da51..cc17d97 100644 --- a/couchpotato/core/media/movie/charts/__init__.py +++ b/couchpotato/core/media/movie/charts/__init__.py @@ -22,13 +22,6 @@ config = [{ 'description': 'Maximum number of items displayed from each chart.', }, { - 'name': 'update_interval', - 'default': 12, - 'type': 'int', - 'advanced': True, - 'description': '(hours)', - }, - { 'name': 'hide_wanted', 'default': False, 'type': 'bool', diff --git a/couchpotato/core/media/movie/charts/main.py b/couchpotato/core/media/movie/charts/main.py index fe6ddc0..28d5d68 100644 --- a/couchpotato/core/media/movie/charts/main.py +++ b/couchpotato/core/media/movie/charts/main.py @@ -13,13 +13,14 @@ log = CPLog(__name__) class Charts(Plugin): update_in_progress = False + update_interval = 72 # hours def __init__(self): addApiView('charts.view', self.automationView) addEvent('app.load', self.setCrons) def setCrons(self): - fireEvent('schedule.interval', 'charts.update_cache', self.updateViewCache, hours = self.conf('update_interval', default = 12)) + fireEvent('schedule.interval', 'charts.update_cache', self.updateViewCache, hours = self.update_interval) def automationView(self, force_update = False, **kwargs): @@ -52,7 +53,7 @@ class Charts(Plugin): for chart in charts: chart['hide_wanted'] = self.conf('hide_wanted') chart['hide_library'] = self.conf('hide_library') - self.setCache('charts_cached', charts, timeout = 7200 * tryInt(self.conf('update_interval', default = 12))) + self.setCache('charts_cached', charts, timeout = self.update_interval * 3600) except: log.error('Failed refreshing charts') From 282f6fb73a8c04fc5228d7e18627fcd2872e4567 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 27 Aug 2014 19:35:52 +0200 Subject: [PATCH 060/202] Only get fanart with extended info --- couchpotato/core/media/movie/providers/info/fanarttv.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/media/movie/providers/info/fanarttv.py b/couchpotato/core/media/movie/providers/info/fanarttv.py index fcd3891..49d944e 100644 --- a/couchpotato/core/media/movie/providers/info/fanarttv.py +++ b/couchpotato/core/media/movie/providers/info/fanarttv.py @@ -23,10 +23,9 @@ class FanartTV(MovieProvider): def __init__(self): addEvent('movie.info', self.getArt, priority = 1) - def getArt(self, identifier = None, **kwargs): + def getArt(self, identifier = None, extended = True, **kwargs): - log.debug("Getting Extra Artwork from Fanart.tv...") - if not identifier: + if not identifier or not extended: return {} images = {} From 42d728f71e99e9e65c0f2aa48900f927dc573b33 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 27 Aug 2014 19:40:28 +0200 Subject: [PATCH 061/202] Update ubuntu start script closes #3846 --- init/ubuntu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/init/ubuntu b/init/ubuntu index 1d2eb57..8c5d556 100755 --- a/init/ubuntu +++ b/init/ubuntu @@ -46,7 +46,7 @@ DESC=CouchPotato # Run CP as username RUN_AS=${CP_USER-couchpotato} -# Path to app +# Path to app # CP_HOME=path_to_app_CouchPotato.py APP_PATH=${CP_HOME-/opt/couchpotato/} @@ -100,12 +100,12 @@ case "$1" in ;; stop) echo "Stopping $DESC" - start-stop-daemon --stop --pidfile $PID_FILE --retry 15 + start-stop-daemon --stop --pidfile $PID_FILE --retry 15 --oknodo ;; restart|force-reload) echo "Restarting $DESC" - start-stop-daemon --stop --pidfile $PID_FILE --retry 15 + start-stop-daemon --stop --pidfile $PID_FILE --retry 15 --oknodo start-stop-daemon -d $APP_PATH -c $RUN_AS $EXTRA_SSD_OPTS --start --pidfile $PID_FILE --exec $DAEMON -- $DAEMON_OPTS ;; From 91c24105cc78265aa429eb187274564fae111f1a Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 27 Aug 2014 19:45:46 +0200 Subject: [PATCH 062/202] Dockers readme install --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a38c052..1ccf689 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ Linux: * (systemd) Enable it at boot with `sudo systemctl enable couchpotato` * Open your browser and go to `http://localhost:5050/` +Docker: +* You can use [razorgirl's Dockerfile](https://github.com/razorgirl/docker-couchpotato) to quickly build your own isolated app container. It's based on the Linux instructions above. For more info about Docker check out the [official website](https://www.docker.com). FreeBSD : From 1daedb72590d868972de9dcbe0fb3887080328a6 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 27 Aug 2014 20:07:27 +0200 Subject: [PATCH 063/202] Update bitsoup tables closes #3807 --- couchpotato/core/media/_base/providers/torrent/bitsoup.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/bitsoup.py b/couchpotato/core/media/_base/providers/torrent/bitsoup.py index 392706c..ad60357 100644 --- a/couchpotato/core/media/_base/providers/torrent/bitsoup.py +++ b/couchpotato/core/media/_base/providers/torrent/bitsoup.py @@ -21,7 +21,10 @@ class Base(TorrentProvider): http_time_between_calls = 1 # Seconds only_tables_tags = SoupStrainer('table') - + + torrent_name_cell = 1; + torrent_download_cell = 2; + def _searchOnTitle(self, title, movie, quality, results): url = self.urls['search'] % self.buildUrl(title, movie, quality) @@ -40,8 +43,8 @@ class Base(TorrentProvider): all_cells = result.find_all('td') - torrent = all_cells[1].find('a') - download = all_cells[3].find('a') + torrent = all_cells[torrent_name_cell].find('a') + download = all_cells[torrent_download_cell].find('a') torrent_id = torrent['href'] torrent_id = torrent_id.replace('details.php?id=', '') From 77e602f3591326ef93115ebabcec1fa0c3363258 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 27 Aug 2014 21:45:53 +0200 Subject: [PATCH 064/202] Use proper bitsoup variable --- couchpotato/core/media/_base/providers/torrent/bitsoup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/bitsoup.py b/couchpotato/core/media/_base/providers/torrent/bitsoup.py index ad60357..f4e256e 100644 --- a/couchpotato/core/media/_base/providers/torrent/bitsoup.py +++ b/couchpotato/core/media/_base/providers/torrent/bitsoup.py @@ -21,10 +21,10 @@ class Base(TorrentProvider): http_time_between_calls = 1 # Seconds only_tables_tags = SoupStrainer('table') - - torrent_name_cell = 1; - torrent_download_cell = 2; - + + torrent_name_cell = 1 + torrent_download_cell = 2 + def _searchOnTitle(self, title, movie, quality, results): url = self.urls['search'] % self.buildUrl(title, movie, quality) @@ -43,8 +43,8 @@ class Base(TorrentProvider): all_cells = result.find_all('td') - torrent = all_cells[torrent_name_cell].find('a') - download = all_cells[torrent_download_cell].find('a') + torrent = all_cells[self.torrent_name_cell].find('a') + download = all_cells[self.torrent_download_cell].find('a') torrent_id = torrent['href'] torrent_id = torrent_id.replace('details.php?id=', '') From 1991792291f866643939938ad1ca336c13bd260f Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 27 Aug 2014 22:04:33 +0200 Subject: [PATCH 065/202] Remove ending seperator --- couchpotato/core/plugins/renamer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 238f150..7e4814b 100644 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -847,13 +847,15 @@ Remove it if you want it to be renamed (again, or at least let it try again) replaces = [ ('\.+', '.'), ('_+', '_'), ('-+', '-'), ('\s+', ' '), (' \\\\', '\\\\'), (' /', '/'), - ('(\s\.)+', '.'), ('(-\.)+', '.'), ('(\s-)+', '-'), ('([\s\.\,\_\-\/\\]$)', ''), + ('(\s\.)+', '.'), ('(-\.)+', '.'), ('(\s-)+', '-'), ] for r in replaces: reg, replace_with = r string = re.sub(reg, replace_with, string) + string = string.rstrip(',_-/\\ ') + return string def checkSnatched(self, fire_scan = True): From afa782194d639340a6043d4b2065112383b59bdc Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 27 Aug 2014 22:04:33 +0200 Subject: [PATCH 066/202] Remove ending seperator --- couchpotato/core/plugins/renamer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 98a3a8c..fdc4329 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -847,13 +847,15 @@ Remove it if you want it to be renamed (again, or at least let it try again) replaces = [ ('\.+', '.'), ('_+', '_'), ('-+', '-'), ('\s+', ' '), (' \\\\', '\\\\'), (' /', '/'), - ('(\s\.)+', '.'), ('(-\.)+', '.'), ('(\s-)+', '-'), ('([\s\.\,\_\-\/\\]$)', ''), + ('(\s\.)+', '.'), ('(-\.)+', '.'), ('(\s-)+', '-'), ] for r in replaces: reg, replace_with = r string = re.sub(reg, replace_with, string) + string = string.rstrip(',_-/\\ ') + return string def checkSnatched(self, fire_scan = True): From ef8cd1aa40087278e9a15f78cfc680dfb72d64aa Mon Sep 17 00:00:00 2001 From: Michael Walker Date: Thu, 28 Aug 2014 01:59:37 +0100 Subject: [PATCH 067/202] URL Fix Domain is missing WWW recond casing an ISSUE. --- couchpotato/core/media/_base/providers/torrent/thepiratebay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/torrent/thepiratebay.py b/couchpotato/core/media/_base/providers/torrent/thepiratebay.py index 6a1de8d..796ade3 100644 --- a/couchpotato/core/media/_base/providers/torrent/thepiratebay.py +++ b/couchpotato/core/media/_base/providers/torrent/thepiratebay.py @@ -24,7 +24,7 @@ class Base(TorrentMagnetProvider): http_time_between_calls = 0 proxy_list = [ - 'https://www.dieroschtibay.org', + 'https://dieroschtibay.org', 'https://thebay.al', 'https://thepiratebay.se', 'http://thepiratebay.se.net', From 5f0543ba42217d5d0a09ad57c849d3ae40659614 Mon Sep 17 00:00:00 2001 From: Michael Walker Date: Thu, 28 Aug 2014 02:10:30 +0100 Subject: [PATCH 068/202] Updated URL's Removed dead links --- couchpotato/core/media/_base/providers/torrent/kickasstorrents.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py b/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py index 730bb60..59b2982 100644 --- a/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py +++ b/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py @@ -34,8 +34,8 @@ class Base(TorrentMagnetProvider): 'http://kickass.pw', 'http://kickassto.come.in', 'http://katproxy.ws', - 'http://www.kickassunblock.info', - 'http://www.kickassproxy.info', + 'http://kickass.bitproxy.eu', + 'http://kickass.to.prx.websiteproxy.co.uk', 'http://katph.eu', 'http://kickassto.come.in', ] From 43af091b02e282863d678f85040c0ebbc2dbde0a Mon Sep 17 00:00:00 2001 From: Michael Walker Date: Thu, 28 Aug 2014 02:13:22 +0100 Subject: [PATCH 069/202] SSL'd Links SSL'd Links --- .../core/media/_base/providers/torrent/torrentleech.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentleech.py b/couchpotato/core/media/_base/providers/torrent/torrentleech.py index 5f59dab..83eb5f1 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentleech.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentleech.py @@ -13,12 +13,12 @@ log = CPLog(__name__) class Base(TorrentProvider): urls = { - 'test': 'http://www.torrentleech.org/', - 'login': 'http://www.torrentleech.org/user/account/login/', - 'login_check': 'http://torrentleech.org/user/messages', - 'detail': 'http://www.torrentleech.org/torrent/%s', - 'search': 'http://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d', - 'download': 'http://www.torrentleech.org%s', + 'test': 'https://www.torrentleech.org/', + 'login': 'https://www.torrentleech.org/user/account/login/', + 'login_check': 'https://torrentleech.org/user/messages', + 'detail': 'https://www.torrentleech.org/torrent/%s', + 'search': 'https://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d', + 'download': 'https://www.torrentleech.org%s', } http_time_between_calls = 1 # Seconds From b07f91d6a591aac88e8f8b46ad275f4595594900 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 28 Aug 2014 13:41:41 +0200 Subject: [PATCH 070/202] Wrap exceptions around score calculations Fix #3859 --- couchpotato/core/plugins/score/scores.py | 101 +++++++++++++++++++------------ 1 file changed, 63 insertions(+), 38 deletions(-) diff --git a/couchpotato/core/plugins/score/scores.py b/couchpotato/core/plugins/score/scores.py index a53608c..f53f69a 100644 --- a/couchpotato/core/plugins/score/scores.py +++ b/couchpotato/core/plugins/score/scores.py @@ -33,33 +33,43 @@ name_scores = [ def nameScore(name, year, preferred_words): """ Calculate score for words in the NZB name """ - score = 0 - name = name.lower() + try: + score = 0 + name = name.lower() - # give points for the cool stuff - for value in name_scores: - v = value.split(':') - add = int(v.pop()) - if v.pop() in name: - score += add + # give points for the cool stuff + for value in name_scores: + v = value.split(':') + add = int(v.pop()) + if v.pop() in name: + score += add - # points if the year is correct - if str(year) in name: - score += 5 + # points if the year is correct + if str(year) in name: + score += 5 - # Contains preferred word - nzb_words = re.split('\W+', simplifyString(name)) - score += 100 * len(list(set(nzb_words) & set(preferred_words))) + # Contains preferred word + nzb_words = re.split('\W+', simplifyString(name)) + score += 100 * len(list(set(nzb_words) & set(preferred_words))) - return score + return score + except: + log.error('Failed doing nameScore: %s', traceback.format_exc()) + + return 0 def nameRatioScore(nzb_name, movie_name): - nzb_words = re.split('\W+', fireEvent('scanner.create_file_identifier', nzb_name, single = True)) - movie_words = re.split('\W+', simplifyString(movie_name)) + try: + nzb_words = re.split('\W+', fireEvent('scanner.create_file_identifier', nzb_name, single = True)) + movie_words = re.split('\W+', simplifyString(movie_name)) - left_over = set(nzb_words) - set(movie_words) - return 10 - len(left_over) + left_over = set(nzb_words) - set(movie_words) + return 10 - len(left_over) + except: + log.error('Failed doing nameRatioScore: %s', traceback.format_exc()) + + return 0 def namePositionScore(nzb_name, movie_name): @@ -134,38 +144,53 @@ def providerScore(provider): def duplicateScore(nzb_name, movie_name): - nzb_words = re.split('\W+', simplifyString(nzb_name)) - movie_words = re.split('\W+', simplifyString(movie_name)) + try: + nzb_words = re.split('\W+', simplifyString(nzb_name)) + movie_words = re.split('\W+', simplifyString(movie_name)) - # minus for duplicates - duplicates = [x for i, x in enumerate(nzb_words) if nzb_words[i:].count(x) > 1] + # minus for duplicates + duplicates = [x for i, x in enumerate(nzb_words) if nzb_words[i:].count(x) > 1] - return len(list(set(duplicates) - set(movie_words))) * -4 + return len(list(set(duplicates) - set(movie_words))) * -4 + except: + log.error('Failed doing duplicateScore: %s', traceback.format_exc()) + + return 0 def partialIgnoredScore(nzb_name, movie_name, ignored_words): - nzb_name = nzb_name.lower() - movie_name = movie_name.lower() + try: + nzb_name = nzb_name.lower() + movie_name = movie_name.lower() - score = 0 - for ignored_word in ignored_words: - if ignored_word in nzb_name and ignored_word not in movie_name: - score -= 5 + score = 0 + for ignored_word in ignored_words: + if ignored_word in nzb_name and ignored_word not in movie_name: + score -= 5 - return score + return score + except: + log.error('Failed doing partialIgnoredScore: %s', traceback.format_exc()) + + return 0 def halfMultipartScore(nzb_name): - wrong_found = 0 - for nr in [1, 2, 3, 4, 5, 'i', 'ii', 'iii', 'iv', 'v', 'a', 'b', 'c', 'd', 'e']: - for wrong in ['cd', 'part', 'dis', 'disc', 'dvd']: - if '%s%s' % (wrong, nr) in nzb_name.lower(): - wrong_found += 1 + try: + wrong_found = 0 + for nr in [1, 2, 3, 4, 5, 'i', 'ii', 'iii', 'iv', 'v', 'a', 'b', 'c', 'd', 'e']: + for wrong in ['cd', 'part', 'dis', 'disc', 'dvd']: + if '%s%s' % (wrong, nr) in nzb_name.lower(): + wrong_found += 1 + + if wrong_found == 1: + return -30 - if wrong_found == 1: - return -30 + return 0 + except: + log.error('Failed doing halfMultipartScore: %s', traceback.format_exc()) return 0 From 8e0d1520e8a3c1e4f056bc9b3c929504d93ba159 Mon Sep 17 00:00:00 2001 From: Michael Walker Date: Thu, 28 Aug 2014 12:44:24 +0100 Subject: [PATCH 071/202] Removed URL URL had a captcha --- couchpotato/core/media/_base/providers/torrent/kickasstorrents.py | 1 - 1 file changed, 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py b/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py index 59b2982..d6e3ee7 100644 --- a/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py +++ b/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py @@ -35,7 +35,6 @@ class Base(TorrentMagnetProvider): 'http://kickassto.come.in', 'http://katproxy.ws', 'http://kickass.bitproxy.eu', - 'http://kickass.to.prx.websiteproxy.co.uk', 'http://katph.eu', 'http://kickassto.come.in', ] From 1d60d9caf17bdfddd409c25970e013b62d5f3589 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 28 Aug 2014 15:23:01 +0200 Subject: [PATCH 072/202] Blu-ray backlog not working fixes #3826 --- .../media/movie/providers/automation/bluray.py | 53 ++++++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/couchpotato/core/media/movie/providers/automation/bluray.py b/couchpotato/core/media/movie/providers/automation/bluray.py index 0501c60..a39fddc 100644 --- a/couchpotato/core/media/movie/providers/automation/bluray.py +++ b/couchpotato/core/media/movie/providers/automation/bluray.py @@ -1,3 +1,5 @@ +import traceback + from bs4 import BeautifulSoup from couchpotato import fireEvent from couchpotato.core.helpers.rss import RSS @@ -5,6 +7,7 @@ from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.media.movie.providers.automation.base import Automation + log = CPLog(__name__) autoload = 'Bluray' @@ -34,27 +37,47 @@ class Bluray(Automation, RSS): try: # Stop if the release year is before the minimal year - page_year = soup.body.find_all('center')[3].table.tr.find_all('td', recursive = False)[3].h3.get_text().split(', ')[1] - if tryInt(page_year) < self.getMinimal('year'): + brk = False + h3s = soup.body.find_all('h3') + for h3 in h3s: + if h3.parent.name != 'a': + + try: + page_year = tryInt(h3.get_text()[-4:]) + if page_year > 0 and page_year < self.getMinimal('year'): + brk = True + except: + break + + if brk: break - for table in soup.body.find_all('center')[3].table.tr.find_all('td', recursive = False)[3].find_all('table')[1:20]: - name = table.h3.get_text().lower().split('blu-ray')[0].strip() - year = table.small.get_text().split('|')[1].strip() + for h3 in h3s: + try: + if h3.parent.name == 'a': + name = h3.get_text().lower().split('blu-ray')[0].strip() + + if not name.find('/') == -1: # make sure it is not a double movie release + continue + + if not h3.parent.parent.small: # ignore non-movie tables + continue - if not name.find('/') == -1: # make sure it is not a double movie release - continue + year = h3.parent.parent.small.get_text().split('|')[1].strip() - if tryInt(year) < self.getMinimal('year'): - continue + if tryInt(year) < self.getMinimal('year'): + continue - imdb = self.search(name, year) + imdb = self.search(name, year) - if imdb: - if self.isMinimalMovie(imdb): - movies.append(imdb['imdb']) + if imdb: + if self.isMinimalMovie(imdb): + movies.append(imdb['imdb']) + except: + log.debug('Error parsing movie html: %s', traceback.format_exc()) + break except: - log.debug('Error loading page: %s', page) + log.debug('Error loading page %s: %s', (page, traceback.format_exc())) break self.conf('backlog', value = False) @@ -134,7 +157,7 @@ config = [{ { 'name': 'backlog', 'advanced': True, - 'description': 'Parses the history until the minimum movie year is reached. (Will be disabled once it has completed)', + 'description': ('Parses the history until the minimum movie year is reached. (Takes a while)', 'Will be disabled once it has completed'), 'default': False, 'type': 'bool', }, From 3cfe90d581a2351bb10081b2efba94a450c947d1 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 28 Aug 2014 15:25:25 +0200 Subject: [PATCH 073/202] break backlog on error --- couchpotato/core/media/movie/providers/automation/bluray.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/media/movie/providers/automation/bluray.py b/couchpotato/core/media/movie/providers/automation/bluray.py index a39fddc..31df78b 100644 --- a/couchpotato/core/media/movie/providers/automation/bluray.py +++ b/couchpotato/core/media/movie/providers/automation/bluray.py @@ -47,6 +47,8 @@ class Bluray(Automation, RSS): if page_year > 0 and page_year < self.getMinimal('year'): brk = True except: + log.error('Failed determining page year: %s', traceback.format_exc()) + brk = True break if brk: From b0ff526c95243814b32ca1bbbf2e7d896c46fc24 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 28 Aug 2014 16:33:59 +0200 Subject: [PATCH 074/202] Improved quality matching fixes #3829 --- couchpotato/core/plugins/quality/main.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 2687cb1..20e6b07 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -187,14 +187,15 @@ class QualityPlugin(Plugin): return False - def guess(self, files, extra = None, size = None): + def guess(self, files, extra = None, size = None, use_cache = True): if not extra: extra = {} # Create hash for cache cache_key = str([f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files]) - cached = self.getCache(cache_key) - if cached and len(extra) == 0: - return cached + if use_cache: + cached = self.getCache(cache_key) + if cached and len(extra) == 0: + return cached qualities = self.all() @@ -234,7 +235,7 @@ class QualityPlugin(Plugin): # Add additional size score if only 1 size validated if len(size_scores) == 1: - self.calcScore(score, size_scores[0], 10, penalty = False) + self.calcScore(score, size_scores[0], 8, penalty = False) del size_scores # Return nothing if all scores are <= 0 @@ -259,7 +260,7 @@ class QualityPlugin(Plugin): def containsTagScore(self, quality, words, cur_file = ''): cur_file = ss(cur_file) - score = 0 + score = 0.0 extension = words[-1] words = words[:-1] @@ -267,7 +268,7 @@ class QualityPlugin(Plugin): points = { 'identifier': 10, 'label': 10, - 'alternative': 9, + 'alternative': 10, 'tags': 9, 'ext': 3, } @@ -285,7 +286,7 @@ class QualityPlugin(Plugin): if isinstance(alt, (str, unicode)) and ss(alt.lower()) in words: log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) - score += points.get(tag_type) / 2 + score += points.get(tag_type) if list(set(qualities) & set(words)): log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) @@ -374,9 +375,9 @@ class QualityPlugin(Plugin): for allow in quality.get('allow', []): score[allow]['score'] -= 40 if self.cached_order[allow] < self.cached_order[quality['identifier']] else 5 - # Give panelty for all lower qualities - for q in self.qualities[self.order.index(quality.get('identifier'))+1:]: - if score.get(q.get('identifier')): + # Give panelty for all other qualities + for q in self.qualities: + if quality.get('identifier') != q.get('identifier') and score.get(q.get('identifier')): score[q.get('identifier')]['score'] -= 1 def isFinish(self, quality, profile, release_age = 0): @@ -456,12 +457,15 @@ class QualityPlugin(Plugin): 'Movie Name (2014).mp4': {'size': 750, 'quality': 'brrip'}, 'Moviename.2014.720p.R6.WEB-DL.x264.AC3-xyz': {'size': 750, 'quality': 'r5'}, 'Movie name 2014 New Source 720p HDCAM x264 AC3 xyz': {'size': 750, 'quality': 'cam'}, - 'Movie.Name.2014.720p.HD.TS.AC3.x264': {'size': 750, 'quality': 'ts'} + 'Movie.Name.2014.720p.HD.TS.AC3.x264': {'size': 750, 'quality': 'ts'}, + # 'Movie.Name.2014.1080p.HDrip.x264.aac-ReleaseGroup': {'size': 7500, 'quality': 'brrip'}, + 'Movie.Name.2014.HDCam.Chinese.Subs-ReleaseGroup': {'size': 15000, 'quality': 'cam'}, + 'Movie Name 2014 HQ DVDRip X264 AC3 (bla)': {'size': 0, 'quality': 'dvdrip'}, } correct = 0 for name in tests: - test_quality = self.guess(files = [name], extra = tests[name].get('extra', None), size = tests[name].get('size', None)) or {} + test_quality = self.guess(files = [name], extra = tests[name].get('extra', None), size = tests[name].get('size', None), use_cache = False) or {} success = test_quality.get('identifier') == tests[name]['quality'] and test_quality.get('is_3d') == tests[name].get('is_3d', False) if not success: log.error('%s failed check, thinks it\'s "%s" expecting "%s"', (name, From 73d37584add8b3c28a7480da31d3177d473aea8a Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 30 Aug 2014 14:00:57 +0200 Subject: [PATCH 075/202] Cleanup import --- CouchPotato.py | 1 - 1 file changed, 1 deletion(-) diff --git a/CouchPotato.py b/CouchPotato.py index 7049cda..afc46aa 100755 --- a/CouchPotato.py +++ b/CouchPotato.py @@ -10,7 +10,6 @@ import socket import subprocess import sys import traceback -import time # Root path base_path = dirname(os.path.abspath(__file__)) From e0dffe20a412af2cd32518a139d84fb646a0ca5e Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 30 Aug 2014 14:01:12 +0200 Subject: [PATCH 076/202] Update lib: Tornado --- libs/tornado/__init__.py | 4 +- libs/tornado/auth.py | 16 +- libs/tornado/concurrent.py | 6 +- libs/tornado/curl_httpclient.py | 27 +-- libs/tornado/gen.py | 69 ++------ libs/tornado/http1connection.py | 76 ++++++-- libs/tornado/httpclient.py | 86 ++++++--- libs/tornado/httpserver.py | 11 +- libs/tornado/httputil.py | 28 +-- libs/tornado/ioloop.py | 148 +++++++++++----- libs/tornado/iostream.py | 57 ++++-- libs/tornado/log.py | 2 +- libs/tornado/netutil.py | 3 + libs/tornado/platform/asyncio.py | 36 ++-- libs/tornado/platform/twisted.py | 31 ++-- libs/tornado/simple_httpclient.py | 35 +--- libs/tornado/testing.py | 6 +- libs/tornado/util.py | 5 + libs/tornado/web.py | 98 +++++------ libs/tornado/websocket.py | 361 ++++++++++---------------------------- libs/tornado/wsgi.py | 8 +- 21 files changed, 506 insertions(+), 607 deletions(-) diff --git a/libs/tornado/__init__.py b/libs/tornado/__init__.py index 0517408..eefe0f2 100755 --- a/libs/tornado/__init__.py +++ b/libs/tornado/__init__.py @@ -25,5 +25,5 @@ from __future__ import absolute_import, division, print_function, with_statement # is zero for an official release, positive for a development branch, # or negative for a release candidate or beta (after the base version # number has been incremented) -version = "3.3.dev1" -version_info = (3, 3, 0, -100) +version = "4.0.1" +version_info = (4, 0, 1, -100) diff --git a/libs/tornado/auth.py b/libs/tornado/auth.py index f15413e..7bd3fa1 100755 --- a/libs/tornado/auth.py +++ b/libs/tornado/auth.py @@ -51,7 +51,7 @@ Example usage for Google OpenID:: response_type='code', extra_params={'approval_prompt': 'auto'}) -.. versionchanged:: 3.3 +.. versionchanged:: 4.0 All of the callback interfaces in this module are now guaranteed to run their callback with an argument of ``None`` on error. Previously some functions would do this while others would simply @@ -883,9 +883,10 @@ class FriendFeedMixin(OAuthMixin): class GoogleMixin(OpenIdMixin, OAuthMixin): """Google Open ID / OAuth authentication. - *Deprecated:* New applications should use `GoogleOAuth2Mixin` - below instead of this class. As of May 19, 2014, Google has stopped - supporting registration-free authentication. + .. deprecated:: 4.0 + New applications should use `GoogleOAuth2Mixin` + below instead of this class. As of May 19, 2014, Google has stopped + supporting registration-free authentication. No application registration is necessary to use Google for authentication or to access Google resources on behalf of a user. @@ -1053,9 +1054,10 @@ class GoogleOAuth2Mixin(OAuth2Mixin): class FacebookMixin(object): """Facebook Connect authentication. - *Deprecated:* New applications should use `FacebookGraphMixin` - below instead of this class. This class does not support the - Future-based interface seen on other classes in this module. + .. deprecated:: 1.1 + New applications should use `FacebookGraphMixin` + below instead of this class. This class does not support the + Future-based interface seen on other classes in this module. To authenticate with Facebook, register your application with Facebook at http://www.facebook.com/developers/apps.php. Then diff --git a/libs/tornado/concurrent.py b/libs/tornado/concurrent.py index 63b0a8c..702aa35 100755 --- a/libs/tornado/concurrent.py +++ b/libs/tornado/concurrent.py @@ -60,7 +60,7 @@ class Future(object): This functionality was previously available in a separate class ``TracebackFuture``, which is now a deprecated alias for this class. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 `tornado.concurrent.Future` is always a thread-unsafe ``Future`` with support for the ``exc_info`` methods. Previously it would be an alias for the thread-safe `concurrent.futures.Future` @@ -152,7 +152,7 @@ class Future(object): def exc_info(self): """Returns a tuple in the same format as `sys.exc_info` or None. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ return self._exc_info @@ -161,7 +161,7 @@ class Future(object): Preserves tracebacks on Python 2. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ self._exc_info = exc_info self.set_exception(exc_info[1]) diff --git a/libs/tornado/curl_httpclient.py b/libs/tornado/curl_httpclient.py index fc7d7f2..3da59a4 100755 --- a/libs/tornado/curl_httpclient.py +++ b/libs/tornado/curl_httpclient.py @@ -51,18 +51,6 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): self._fds = {} self._timeout = None - try: - self._socket_action = self._multi.socket_action - except AttributeError: - # socket_action is found in pycurl since 7.18.2 (it's been - # in libcurl longer than that but wasn't accessible to - # python). - gen_log.warning("socket_action method missing from pycurl; " - "falling back to socket_all. Upgrading " - "libcurl and pycurl will improve performance") - self._socket_action = \ - lambda fd, action: self._multi.socket_all() - # libcurl has bugs that sometimes cause it to not report all # relevant file descriptors and timeouts to TIMERFUNCTION/ # SOCKETFUNCTION. Mitigate the effects of such bugs by @@ -87,7 +75,6 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): for curl in self._curls: curl.close() self._multi.close() - self._closed = True super(CurlAsyncHTTPClient, self).close() def fetch_impl(self, request, callback): @@ -143,7 +130,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): action |= pycurl.CSELECT_OUT while True: try: - ret, num_handles = self._socket_action(fd, action) + ret, num_handles = self._multi.socket_action(fd, action) except pycurl.error as e: ret = e.args[0] if ret != pycurl.E_CALL_MULTI_PERFORM: @@ -156,7 +143,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): self._timeout = None while True: try: - ret, num_handles = self._socket_action( + ret, num_handles = self._multi.socket_action( pycurl.SOCKET_TIMEOUT, 0) except pycurl.error as e: ret = e.args[0] @@ -224,11 +211,6 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): "callback": callback, "curl_start_time": time.time(), } - # Disable IPv6 to mitigate the effects of this bug - # on curl versions <= 7.21.0 - # http://sourceforge.net/tracker/?func=detail&aid=3017819&group_id=976&atid=100976 - if pycurl.version_info()[2] <= 0x71500: # 7.21.0 - curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) _curl_setup_request(curl, request, curl.info["buffer"], curl.info["headers"]) self._multi.add_handle(curl) @@ -350,7 +332,7 @@ def _curl_setup_request(curl, request, buffer, headers): curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)") if request.network_interface: curl.setopt(pycurl.INTERFACE, request.network_interface) - if request.use_gzip: + if request.decompress_response: curl.setopt(pycurl.ENCODING, "gzip,deflate") else: curl.setopt(pycurl.ENCODING, "none") @@ -384,7 +366,6 @@ def _curl_setup_request(curl, request, buffer, headers): if request.allow_ipv6 is False: # Curl behaves reasonably when DNS resolution gives an ipv6 address # that we can't reach, so allow ipv6 unless the user asks to disable. - # (but see version check in _process_queue above) curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) else: curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER) @@ -474,7 +455,7 @@ def _curl_header_callback(headers, header_line): try: (__, __, reason) = httputil.parse_response_start_line(header_line) header_line = "X-Http-Reason: %s" % reason - except httputil.HTTPInputException: + except httputil.HTTPInputError: return if not header_line: return diff --git a/libs/tornado/gen.py b/libs/tornado/gen.py index 4d1dc6e..06f2715 100755 --- a/libs/tornado/gen.py +++ b/libs/tornado/gen.py @@ -29,16 +29,7 @@ could be written with ``gen`` as:: Most asynchronous functions in Tornado return a `.Future`; yielding this object returns its `~.Future.result`. -For functions that do not return ``Futures``, `Task` works with any -function that takes a ``callback`` keyword argument (most Tornado functions -can be used in either style, although the ``Future`` style is preferred -since it is both shorter and provides better exception handling):: - - @gen.coroutine - def get(self): - yield gen.Task(AsyncHTTPClient().fetch, "http://example.com") - -You can also yield a list or dict of ``Futures`` and/or ``Tasks``, which will be +You can also yield a list or dict of ``Futures``, which will be started at the same time and run in parallel; a list or dict of results will be returned when they are all finished:: @@ -54,30 +45,6 @@ be returned when they are all finished:: .. versionchanged:: 3.2 Dict support added. - -For more complicated interfaces, `Task` can be split into two parts: -`Callback` and `Wait`:: - - class GenAsyncHandler2(RequestHandler): - @gen.coroutine - def get(self): - http_client = AsyncHTTPClient() - http_client.fetch("http://example.com", - callback=(yield gen.Callback("key"))) - response = yield gen.Wait("key") - do_something_with_response(response) - self.render("template.html") - -The ``key`` argument to `Callback` and `Wait` allows for multiple -asynchronous operations to be started at different times and proceed -in parallel: yield several callbacks with different keys, then wait -for them once all the async operations have started. - -The result of a `Wait` or `Task` yield expression depends on how the callback -was run. If it was called with no arguments, the result is ``None``. If -it was called with one argument, the result is that argument. If it was -called with more than one argument or any keyword arguments, the result -is an `Arguments` object, which is a named tuple ``(args, kwargs)``. """ from __future__ import absolute_import, division, print_function, with_statement @@ -252,8 +219,8 @@ class Return(Exception): class YieldPoint(object): """Base class for objects that may be yielded from the generator. - Applications do not normally need to use this class, but it may be - subclassed to provide additional yielding behavior. + .. deprecated:: 4.0 + Use `Futures <.Future>` instead. """ def start(self, runner): """Called by the runner after the generator has yielded. @@ -289,6 +256,9 @@ class Callback(YieldPoint): The callback may be called with zero or one arguments; if an argument is given it will be returned by `Wait`. + + .. deprecated:: 4.0 + Use `Futures <.Future>` instead. """ def __init__(self, key): self.key = key @@ -305,7 +275,11 @@ class Callback(YieldPoint): class Wait(YieldPoint): - """Returns the argument passed to the result of a previous `Callback`.""" + """Returns the argument passed to the result of a previous `Callback`. + + .. deprecated:: 4.0 + Use `Futures <.Future>` instead. + """ def __init__(self, key): self.key = key @@ -326,6 +300,9 @@ class WaitAll(YieldPoint): a list of results in the same order. `WaitAll` is equivalent to yielding a list of `Wait` objects. + + .. deprecated:: 4.0 + Use `Futures <.Future>` instead. """ def __init__(self, keys): self.keys = keys @@ -341,21 +318,13 @@ class WaitAll(YieldPoint): def Task(func, *args, **kwargs): - """Runs a single asynchronous operation. + """Adapts a callback-based asynchronous function for use in coroutines. Takes a function (and optional additional arguments) and runs it with those arguments plus a ``callback`` keyword argument. The argument passed to the callback is returned as the result of the yield expression. - A `Task` is equivalent to a `Callback`/`Wait` pair (with a unique - key generated automatically):: - - result = yield gen.Task(func, args) - - func(args, callback=(yield gen.Callback(key))) - result = yield gen.Wait(key) - - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 ``gen.Task`` is now a function that returns a `.Future`, instead of a subclass of `YieldPoint`. It still behaves the same way when yielded. @@ -464,7 +433,7 @@ def multi_future(children): This function is faster than the `Multi` `YieldPoint` because it does not require the creation of a stack context. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ if isinstance(children, dict): keys = list(children.keys()) @@ -520,7 +489,7 @@ def with_timeout(timeout, future, io_loop=None): Currently only supports Futures, not other `YieldPoint` classes. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ # TODO: allow yield points in addition to futures? # Tricky to do with stack_context semantics. @@ -564,7 +533,7 @@ coroutines that are likely to yield Futures that are ready instantly. Usage: ``yield gen.moment`` -.. versionadded:: 3.3 +.. versionadded:: 4.0 """ moment.set_result(None) diff --git a/libs/tornado/http1connection.py b/libs/tornado/http1connection.py index edaa5d9..1ac24f5 100644 --- a/libs/tornado/http1connection.py +++ b/libs/tornado/http1connection.py @@ -16,11 +16,13 @@ """Client and server implementations of HTTP/1.x. -.. versionadded:: 3.3 +.. versionadded:: 4.0 """ from __future__ import absolute_import, division, print_function, with_statement +import re + from tornado.concurrent import Future from tornado.escape import native_str, utf8 from tornado import gen @@ -56,7 +58,7 @@ class HTTP1ConnectionParameters(object): """ def __init__(self, no_keep_alive=False, chunk_size=None, max_header_size=None, header_timeout=None, max_body_size=None, - body_timeout=None, use_gzip=False): + body_timeout=None, decompress=False): """ :arg bool no_keep_alive: If true, always close the connection after one request. @@ -65,7 +67,8 @@ class HTTP1ConnectionParameters(object): :arg float header_timeout: how long to wait for all headers (seconds) :arg int max_body_size: maximum amount of data for body :arg float body_timeout: how long to wait while reading body (seconds) - :arg bool use_gzip: if true, decode incoming ``Content-Encoding: gzip`` + :arg bool decompress: if true, decode incoming + ``Content-Encoding: gzip`` """ self.no_keep_alive = no_keep_alive self.chunk_size = chunk_size or 65536 @@ -73,7 +76,7 @@ class HTTP1ConnectionParameters(object): self.header_timeout = header_timeout self.max_body_size = max_body_size self.body_timeout = body_timeout - self.use_gzip = use_gzip + self.decompress = decompress class HTTP1Connection(httputil.HTTPConnection): @@ -141,7 +144,7 @@ class HTTP1Connection(httputil.HTTPConnection): Returns a `.Future` that resolves to None after the full response has been read. """ - if self.params.use_gzip: + if self.params.decompress: delegate = _GzipMessageDelegate(delegate, self.params.chunk_size) return self._read_message(delegate) @@ -190,8 +193,17 @@ class HTTP1Connection(httputil.HTTPConnection): skip_body = True code = start_line.code if code == 304: + # 304 responses may include the content-length header + # but do not actually have a body. + # http://tools.ietf.org/html/rfc7230#section-3.3 skip_body = True if code >= 100 and code < 200: + # 1xx responses should never indicate the presence of + # a body. + if ('Content-Length' in headers or + 'Transfer-Encoding' in headers): + raise httputil.HTTPInputError( + "Response code %d cannot have body" % code) # TODO: client delegates will get headers_received twice # in the case of a 100-continue. Document or change? yield self._read_message(delegate) @@ -200,7 +212,8 @@ class HTTP1Connection(httputil.HTTPConnection): not self._write_finished): self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n") if not skip_body: - body_future = self._read_body(headers, delegate) + body_future = self._read_body( + start_line.code if self.is_client else 0, headers, delegate) if body_future is not None: if self._body_timeout is None: yield body_future @@ -231,7 +244,7 @@ class HTTP1Connection(httputil.HTTPConnection): self.close() if self.stream is None: raise gen.Return(False) - except httputil.HTTPInputException as e: + except httputil.HTTPInputError as e: gen_log.info("Malformed HTTP message from %s: %s", self.context, e) self.close() @@ -258,7 +271,7 @@ class HTTP1Connection(httputil.HTTPConnection): def set_close_callback(self, callback): """Sets a callback that will be run when the connection is closed. - .. deprecated:: 3.3 + .. deprecated:: 4.0 Use `.HTTPMessageDelegate.on_connection_close` instead. """ self._close_callback = stack_context.wrap(callback) @@ -377,7 +390,7 @@ class HTTP1Connection(httputil.HTTPConnection): if self._expected_content_remaining < 0: # Close the stream now to stop further framing errors. self.stream.close() - raise httputil.HTTPOutputException( + raise httputil.HTTPOutputError( "Tried to write more data than Content-Length") if self._chunking_output and chunk: # Don't write out empty chunks because that means END-OF-STREAM @@ -412,7 +425,7 @@ class HTTP1Connection(httputil.HTTPConnection): self._expected_content_remaining != 0 and not self.stream.closed()): self.stream.close() - raise httputil.HTTPOutputException( + raise httputil.HTTPOutputError( "Tried to write %d bytes less than Content-Length" % self._expected_content_remaining) if self._chunking_output: @@ -477,16 +490,40 @@ class HTTP1Connection(httputil.HTTPConnection): headers = httputil.HTTPHeaders.parse(data[eol:]) except ValueError: # probably form split() if there was no ':' in the line - raise httputil.HTTPInputException("Malformed HTTP headers: %r" % - data[eol:100]) + raise httputil.HTTPInputError("Malformed HTTP headers: %r" % + data[eol:100]) return start_line, headers - def _read_body(self, headers, delegate): - content_length = headers.get("Content-Length") - if content_length: - content_length = int(content_length) + def _read_body(self, code, headers, delegate): + if "Content-Length" in headers: + if "," in headers["Content-Length"]: + # Proxies sometimes cause Content-Length headers to get + # duplicated. If all the values are identical then we can + # use them but if they differ it's an error. + pieces = re.split(r',\s*', headers["Content-Length"]) + if any(i != pieces[0] for i in pieces): + raise httputil.HTTPInputError( + "Multiple unequal Content-Lengths: %r" % + headers["Content-Length"]) + headers["Content-Length"] = pieces[0] + content_length = int(headers["Content-Length"]) + if content_length > self._max_body_size: - raise httputil.HTTPInputException("Content-Length too long") + raise httputil.HTTPInputError("Content-Length too long") + else: + content_length = None + + if code == 204: + # This response code is not allowed to have a non-empty body, + # and has an implicit length of zero instead of read-until-close. + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3 + if ("Transfer-Encoding" in headers or + content_length not in (None, 0)): + raise httputil.HTTPInputError( + "Response with code %d should not have body" % code) + content_length = 0 + + if content_length is not None: return self._read_fixed_body(content_length, delegate) if headers.get("Transfer-Encoding") == "chunked": return self._read_chunked_body(delegate) @@ -515,7 +552,7 @@ class HTTP1Connection(httputil.HTTPConnection): return total_size += chunk_len if total_size > self._max_body_size: - raise httputil.HTTPInputException("chunked body too large") + raise httputil.HTTPInputError("chunked body too large") bytes_to_read = chunk_len while bytes_to_read: chunk = yield self.stream.read_bytes( @@ -581,6 +618,9 @@ class _GzipMessageDelegate(httputil.HTTPMessageDelegate): self._delegate.data_received(tail) return self._delegate.finish() + def on_connection_close(self): + return self._delegate.on_connection_close() + class HTTP1ServerConnection(object): """An HTTP/1.x server.""" diff --git a/libs/tornado/httpclient.py b/libs/tornado/httpclient.py index 94a4593..c8ecf47 100755 --- a/libs/tornado/httpclient.py +++ b/libs/tornado/httpclient.py @@ -22,14 +22,20 @@ to switch to ``curl_httpclient`` for reasons such as the following: * ``curl_httpclient`` was the default prior to Tornado 2.0. -Note that if you are using ``curl_httpclient``, it is highly recommended that -you use a recent version of ``libcurl`` and ``pycurl``. Currently the minimum -supported version is 7.18.2, and the recommended version is 7.21.1 or newer. -It is highly recommended that your ``libcurl`` installation is built with -asynchronous DNS resolver (threaded or c-ares), otherwise you may encounter -various problems with request timeouts (for more information, see +Note that if you are using ``curl_httpclient``, it is highly +recommended that you use a recent version of ``libcurl`` and +``pycurl``. Currently the minimum supported version of libcurl is +7.21.1, and the minimum version of pycurl is 7.18.2. It is highly +recommended that your ``libcurl`` installation is built with +asynchronous DNS resolver (threaded or c-ares), otherwise you may +encounter various problems with request timeouts (for more +information, see http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUTMS and comments in curl_httpclient.py). + +To select ``curl_httpclient``, call `AsyncHTTPClient.configure` at startup:: + + AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") """ from __future__ import absolute_import, division, print_function, with_statement @@ -110,10 +116,21 @@ class AsyncHTTPClient(Configurable): actually creates an instance of an implementation-specific subclass, and instances are reused as a kind of pseudo-singleton (one per `.IOLoop`). The keyword argument ``force_instance=True`` - can be used to suppress this singleton behavior. Constructor - arguments other than ``io_loop`` and ``force_instance`` are - deprecated. The implementation subclass as well as arguments to - its constructor can be set with the static method `configure()` + can be used to suppress this singleton behavior. Unless + ``force_instance=True`` is used, no arguments other than + ``io_loop`` should be passed to the `AsyncHTTPClient` constructor. + The implementation subclass as well as arguments to its + constructor can be set with the static method `configure()` + + All `AsyncHTTPClient` implementations support a ``defaults`` + keyword argument, which can be used to set default values for + `HTTPRequest` attributes. For example:: + + AsyncHTTPClient.configure( + None, defaults=dict(user_agent="MyUserAgent")) + # or with force_instance: + client = AsyncHTTPClient(force_instance=True, + defaults=dict(user_agent="MyUserAgent")) """ @classmethod def configurable_base(cls): @@ -133,12 +150,21 @@ class AsyncHTTPClient(Configurable): def __new__(cls, io_loop=None, force_instance=False, **kwargs): io_loop = io_loop or IOLoop.current() - if io_loop in cls._async_clients() and not force_instance: - return cls._async_clients()[io_loop] + if force_instance: + instance_cache = None + else: + instance_cache = cls._async_clients() + if instance_cache is not None and io_loop in instance_cache: + return instance_cache[io_loop] instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop, **kwargs) - if not force_instance: - cls._async_clients()[io_loop] = instance + # Make sure the instance knows which cache to remove itself from. + # It can't simply call _async_clients() because we may be in + # __new__(AsyncHTTPClient) but instance.__class__ may be + # SimpleAsyncHTTPClient. + instance._instance_cache = instance_cache + if instance_cache is not None: + instance_cache[instance.io_loop] = instance return instance def initialize(self, io_loop, defaults=None): @@ -146,6 +172,7 @@ class AsyncHTTPClient(Configurable): self.defaults = dict(HTTPRequest._DEFAULTS) if defaults is not None: self.defaults.update(defaults) + self._closed = False def close(self): """Destroys this HTTP client, freeing any file descriptors used. @@ -160,8 +187,13 @@ class AsyncHTTPClient(Configurable): ``close()``. """ - if self._async_clients().get(self.io_loop) is self: - del self._async_clients()[self.io_loop] + if self._closed: + return + self._closed = True + if self._instance_cache is not None: + if self._instance_cache.get(self.io_loop) is not self: + raise RuntimeError("inconsistent AsyncHTTPClient cache") + del self._instance_cache[self.io_loop] def fetch(self, request, callback=None, **kwargs): """Executes a request, asynchronously returning an `HTTPResponse`. @@ -179,6 +211,8 @@ class AsyncHTTPClient(Configurable): Instead, you must check the response's ``error`` attribute or call its `~HTTPResponse.rethrow` method. """ + if self._closed: + raise RuntimeError("fetch() called on closed AsyncHTTPClient") if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) # We may modify this (to add Host, Accept-Encoding, etc), @@ -248,7 +282,7 @@ class HTTPRequest(object): request_timeout=20.0, follow_redirects=True, max_redirects=5, - use_gzip=True, + decompress_response=True, proxy_password='', allow_nonstandard_methods=False, validate_cert=True) @@ -265,7 +299,7 @@ class HTTPRequest(object): validate_cert=None, ca_certs=None, allow_ipv6=None, client_key=None, client_cert=None, body_producer=None, - expect_100_continue=False): + expect_100_continue=False, decompress_response=None): r"""All parameters except ``url`` are optional. :arg string url: URL to fetch @@ -284,7 +318,7 @@ class HTTPRequest(object): ``curl_httpclient``. When using ``body_producer`` it is recommended to pass a ``Content-Length`` in the headers as otherwise chunked encoding will be used, and many servers do not support chunked - encoding on requests. New in Tornado 3.3 + encoding on requests. New in Tornado 4.0 :arg string auth_username: Username for HTTP authentication :arg string auth_password: Password for HTTP authentication :arg string auth_mode: Authentication mode; default is "basic". @@ -299,7 +333,11 @@ class HTTPRequest(object): or return the 3xx response? :arg int max_redirects: Limit for ``follow_redirects`` :arg string user_agent: String to send as ``User-Agent`` header - :arg bool use_gzip: Request gzip encoding from the server + :arg bool decompress_response: Request a compressed response from + the server and decompress it after downloading. Default is True. + New in Tornado 4.0. + :arg bool use_gzip: Deprecated alias for ``decompress_response`` + since Tornado 4.0. :arg string network_interface: Network interface to use for request. ``curl_httpclient`` only; see note below. :arg callable streaming_callback: If set, ``streaming_callback`` will @@ -342,7 +380,6 @@ class HTTPRequest(object): before sending the request body. Only supported with simple_httpclient. - .. note:: When using ``curl_httpclient`` certain options may be @@ -358,7 +395,7 @@ class HTTPRequest(object): .. versionadded:: 3.1 The ``auth_mode`` argument. - .. versionadded:: 3.3 + .. versionadded:: 4.0 The ``body_producer`` and ``expect_100_continue`` arguments. """ # Note that some of these attributes go through property setters @@ -383,7 +420,10 @@ class HTTPRequest(object): self.follow_redirects = follow_redirects self.max_redirects = max_redirects self.user_agent = user_agent - self.use_gzip = use_gzip + if decompress_response is not None: + self.decompress_response = decompress_response + else: + self.decompress_response = use_gzip self.network_interface = network_interface self.streaming_callback = streaming_callback self.header_callback = header_callback diff --git a/libs/tornado/httpserver.py b/libs/tornado/httpserver.py index 469374e..03b5fc7 100755 --- a/libs/tornado/httpserver.py +++ b/libs/tornado/httpserver.py @@ -20,7 +20,7 @@ Typical applications have little direct interaction with the `HTTPServer` class except to start a server at the beginning of the process (and even that is often done indirectly via `tornado.web.Application.listen`). -.. versionchanged:: 3.3 +.. versionchanged:: 4.0 The ``HTTPRequest`` class that used to live in this module has been moved to `tornado.httputil.HTTPServerRequest`. The old name remains as an alias. @@ -128,14 +128,15 @@ class HTTPServer(TCPServer, httputil.HTTPServerConnectionDelegate): servers if you want to create your listening sockets in some way other than `tornado.netutil.bind_sockets`. - .. versionchanged:: 3.3 - Added ``gzip``, ``chunk_size``, ``max_header_size``, + .. versionchanged:: 4.0 + Added ``decompress_request``, ``chunk_size``, ``max_header_size``, ``idle_connection_timeout``, ``body_timeout``, ``max_body_size`` arguments. Added support for `.HTTPServerConnectionDelegate` instances as ``request_callback``. """ def __init__(self, request_callback, no_keep_alive=False, io_loop=None, - xheaders=False, ssl_options=None, protocol=None, gzip=False, + xheaders=False, ssl_options=None, protocol=None, + decompress_request=False, chunk_size=None, max_header_size=None, idle_connection_timeout=None, body_timeout=None, max_body_size=None, max_buffer_size=None): @@ -144,7 +145,7 @@ class HTTPServer(TCPServer, httputil.HTTPServerConnectionDelegate): self.xheaders = xheaders self.protocol = protocol self.conn_params = HTTP1ConnectionParameters( - use_gzip=gzip, + decompress=decompress_request, chunk_size=chunk_size, max_header_size=max_header_size, header_timeout=idle_connection_timeout or 3600, diff --git a/libs/tornado/httputil.py b/libs/tornado/httputil.py index 6e110d9..a674897 100755 --- a/libs/tornado/httputil.py +++ b/libs/tornado/httputil.py @@ -319,7 +319,7 @@ class HTTPServerRequest(object): are typically kept open in HTTP/1.1, multiple requests can be handled sequentially on a single connection. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Moved from ``tornado.httpserver.HTTPRequest``. """ def __init__(self, method=None, uri=None, version="HTTP/1.0", headers=None, @@ -352,7 +352,7 @@ class HTTPServerRequest(object): def supports_http_1_1(self): """Returns True if this request supports HTTP/1.1 semantics. - .. deprecated:: 3.3 + .. deprecated:: 4.0 Applications are less likely to need this information with the introduction of `.HTTPConnection`. If you still need it, access the ``version`` attribute directly. @@ -375,7 +375,7 @@ class HTTPServerRequest(object): def write(self, chunk, callback=None): """Writes the given chunk to the response stream. - .. deprecated:: 3.3 + .. deprecated:: 4.0 Use ``request.connection`` and the `.HTTPConnection` methods to write the response. """ @@ -385,7 +385,7 @@ class HTTPServerRequest(object): def finish(self): """Finishes this HTTP request on the open connection. - .. deprecated:: 3.3 + .. deprecated:: 4.0 Use ``request.connection`` and the `.HTTPConnection` methods to write the response. """ @@ -445,19 +445,19 @@ class HTTPServerRequest(object): self.__class__.__name__, args, dict(self.headers)) -class HTTPInputException(Exception): +class HTTPInputError(Exception): """Exception class for malformed HTTP requests or responses from remote sources. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ pass -class HTTPOutputException(Exception): +class HTTPOutputError(Exception): """Exception class for errors in HTTP output. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ pass @@ -465,7 +465,7 @@ class HTTPOutputException(Exception): class HTTPServerConnectionDelegate(object): """Implement this interface to handle requests from `.HTTPServer`. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ def start_request(self, server_conn, request_conn): """This method is called by the server when a new request has started. @@ -491,7 +491,7 @@ class HTTPServerConnectionDelegate(object): class HTTPMessageDelegate(object): """Implement this interface to handle an HTTP request or response. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ def headers_received(self, start_line, headers): """Called when the HTTP headers have been received and parsed. @@ -531,7 +531,7 @@ class HTTPMessageDelegate(object): class HTTPConnection(object): """Applications use this interface to write their responses. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ def write_headers(self, start_line, headers, chunk=None, callback=None): """Write an HTTP header block. @@ -774,9 +774,9 @@ def parse_request_start_line(line): try: method, path, version = line.split(" ") except ValueError: - raise HTTPInputException("Malformed HTTP request line") + raise HTTPInputError("Malformed HTTP request line") if not version.startswith("HTTP/"): - raise HTTPInputException( + raise HTTPInputError( "Malformed HTTP version in HTTP Request-Line: %r" % version) return RequestStartLine(method, path, version) @@ -796,7 +796,7 @@ def parse_response_start_line(line): line = native_str(line) match = re.match("(HTTP/1.[01]) ([0-9]+) ([^\r]*)", line) if not match: - raise HTTPInputException("Error parsing response start line") + raise HTTPInputError("Error parsing response start line") return ResponseStartLine(match.group(1), int(match.group(2)), match.group(3)) diff --git a/libs/tornado/ioloop.py b/libs/tornado/ioloop.py index cd59bfe..e15252d 100755 --- a/libs/tornado/ioloop.py +++ b/libs/tornado/ioloop.py @@ -45,8 +45,7 @@ import traceback from tornado.concurrent import TracebackFuture, is_future from tornado.log import app_log, gen_log from tornado import stack_context -from tornado.util import Configurable -from tornado.util import errno_from_exception +from tornado.util import Configurable, errno_from_exception, timedelta_to_seconds try: import signal @@ -162,7 +161,7 @@ class IOLoop(Configurable): def clear_instance(): """Clear the global `IOLoop` instance. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ if hasattr(IOLoop, "_instance"): del IOLoop._instance @@ -267,7 +266,7 @@ class IOLoop(Configurable): When an event occurs, ``handler(fd, events)`` will be run. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Added the ability to pass file-like objects in addition to raw file descriptors. """ @@ -276,7 +275,7 @@ class IOLoop(Configurable): def update_handler(self, fd, events): """Changes the events we listen for ``fd``. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Added the ability to pass file-like objects in addition to raw file descriptors. """ @@ -285,7 +284,7 @@ class IOLoop(Configurable): def remove_handler(self, fd): """Stop listening for events on ``fd``. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Added the ability to pass file-like objects in addition to raw file descriptors. """ @@ -433,7 +432,7 @@ class IOLoop(Configurable): """ return time.time() - def add_timeout(self, deadline, callback): + def add_timeout(self, deadline, callback, *args, **kwargs): """Runs the ``callback`` at the time ``deadline`` from the I/O loop. Returns an opaque handle that may be passed to @@ -442,13 +441,59 @@ class IOLoop(Configurable): ``deadline`` may be a number denoting a time (on the same scale as `IOLoop.time`, normally `time.time`), or a `datetime.timedelta` object for a deadline relative to the - current time. + current time. Since Tornado 4.0, `call_later` is a more + convenient alternative for the relative case since it does not + require a timedelta object. Note that it is not safe to call `add_timeout` from other threads. Instead, you must use `add_callback` to transfer control to the `IOLoop`'s thread, and then call `add_timeout` from there. + + Subclasses of IOLoop must implement either `add_timeout` or + `call_at`; the default implementations of each will call + the other. `call_at` is usually easier to implement, but + subclasses that wish to maintain compatibility with Tornado + versions prior to 4.0 must use `add_timeout` instead. + + .. versionchanged:: 4.0 + Now passes through ``*args`` and ``**kwargs`` to the callback. """ - raise NotImplementedError() + if isinstance(deadline, numbers.Real): + return self.call_at(deadline, callback, *args, **kwargs) + elif isinstance(deadline, datetime.timedelta): + return self.call_at(self.time() + timedelta_to_seconds(deadline), + callback, *args, **kwargs) + else: + raise TypeError("Unsupported deadline %r" % deadline) + + def call_later(self, delay, callback, *args, **kwargs): + """Runs the ``callback`` after ``delay`` seconds have passed. + + Returns an opaque handle that may be passed to `remove_timeout` + to cancel. Note that unlike the `asyncio` method of the same + name, the returned object does not have a ``cancel()`` method. + + See `add_timeout` for comments on thread-safety and subclassing. + + .. versionadded:: 4.0 + """ + return self.call_at(self.time() + delay, callback, *args, **kwargs) + + def call_at(self, when, callback, *args, **kwargs): + """Runs the ``callback`` at the absolute time designated by ``when``. + + ``when`` must be a number using the same reference point as + `IOLoop.time`. + + Returns an opaque handle that may be passed to `remove_timeout` + to cancel. Note that unlike the `asyncio` method of the same + name, the returned object does not have a ``cancel()`` method. + + See `add_timeout` for comments on thread-safety and subclassing. + + .. versionadded:: 4.0 + """ + return self.add_timeout(when, callback, *args, **kwargs) def remove_timeout(self, timeout): """Cancels a pending timeout. @@ -486,6 +531,19 @@ class IOLoop(Configurable): """ raise NotImplementedError() + def spawn_callback(self, callback, *args, **kwargs): + """Calls the given callback on the next IOLoop iteration. + + Unlike all other callback-related methods on IOLoop, + ``spawn_callback`` does not associate the callback with its caller's + ``stack_context``, so it is suitable for fire-and-forget callbacks + that should not interfere with the caller. + + .. versionadded:: 4.0 + """ + with stack_context.NullContext(): + self.add_callback(callback, *args, **kwargs) + def add_future(self, future, callback): """Schedules a callback on the ``IOLoop`` when the given `.Future` is finished. @@ -504,7 +562,13 @@ class IOLoop(Configurable): For use in subclasses. """ try: - callback() + ret = callback() + if ret is not None and is_future(ret): + # Functions that return Futures typically swallow all + # exceptions and store them in the Future. If a Future + # makes it out to the IOLoop, ensure its exception (if any) + # gets logged too. + self.add_future(ret, lambda f: f.result()) except Exception: self.handle_callback_exception(callback) @@ -534,7 +598,7 @@ class IOLoop(Configurable): This method is provided for use by `IOLoop` subclasses and should not generally be used by application code. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ try: return fd.fileno(), fd @@ -551,7 +615,7 @@ class IOLoop(Configurable): implementations of ``IOLoop.close(all_fds=True)`` and should not generally be used by application code. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ try: try: @@ -587,7 +651,7 @@ class PollIOLoop(IOLoop): self._thread_ident = None self._blocking_signal_threshold = None self._timeout_counter = itertools.count() - + # Create a pipe that we send bogus data to when we want to wake # the I/O loop when it is idle self._waker = Waker() @@ -680,19 +744,16 @@ class PollIOLoop(IOLoop): try: while True: - poll_timeout = _POLL_TIMEOUT - # Prevent IO event starvation by delaying new callbacks # to the next iteration of the event loop. with self._callback_lock: callbacks = self._callbacks self._callbacks = [] - for callback in callbacks: - self._run_callback(callback) - # Closures may be holding on to a lot of memory, so allow - # them to be freed before we go into our poll wait. - callbacks = callback = None + # Add any timeouts that have come due to the callback list. + # Do not run anything until we have determined which ones + # are ready, so timeouts that call add_timeout cannot + # schedule anything in this iteration. if self._timeouts: now = self.time() while self._timeouts: @@ -702,11 +763,9 @@ class PollIOLoop(IOLoop): self._cancellations -= 1 elif self._timeouts[0].deadline <= now: timeout = heapq.heappop(self._timeouts) - self._run_callback(timeout.callback) + callbacks.append(timeout.callback) del timeout else: - seconds = self._timeouts[0].deadline - now - poll_timeout = min(seconds, poll_timeout) break if (self._cancellations > 512 and self._cancellations > (len(self._timeouts) >> 1)): @@ -717,10 +776,25 @@ class PollIOLoop(IOLoop): if x.callback is not None] heapq.heapify(self._timeouts) + for callback in callbacks: + self._run_callback(callback) + # Closures may be holding on to a lot of memory, so allow + # them to be freed before we go into our poll wait. + callbacks = callback = None + if self._callbacks: # If any callbacks or timeouts called add_callback, # we don't want to wait in poll() before we run them. poll_timeout = 0.0 + elif self._timeouts: + # If there are any timeouts, schedule the first one. + # Use self.time() instead of 'now' to account for time + # spent running callbacks. + poll_timeout = self._timeouts[0].deadline - self.time() + poll_timeout = max(0, min(poll_timeout, _POLL_TIMEOUT)) + else: + # No timeouts and no callbacks, so use the default. + poll_timeout = _POLL_TIMEOUT if not self._running: break @@ -784,8 +858,11 @@ class PollIOLoop(IOLoop): def time(self): return self.time_func() - def add_timeout(self, deadline, callback): - timeout = _Timeout(deadline, stack_context.wrap(callback), self) + def call_at(self, deadline, callback, *args, **kwargs): + timeout = _Timeout( + deadline, + functools.partial(stack_context.wrap(callback), *args, **kwargs), + self) heapq.heappush(self._timeouts, timeout) return timeout @@ -840,24 +917,12 @@ class _Timeout(object): __slots__ = ['deadline', 'callback', 'tiebreaker'] def __init__(self, deadline, callback, io_loop): - if isinstance(deadline, numbers.Real): - self.deadline = deadline - elif isinstance(deadline, datetime.timedelta): - now = io_loop.time() - try: - self.deadline = now + deadline.total_seconds() - except AttributeError: # py2.6 - self.deadline = now + _Timeout.timedelta_to_seconds(deadline) - else: + if not isinstance(deadline, numbers.Real): raise TypeError("Unsupported deadline %r" % deadline) + self.deadline = deadline self.callback = callback self.tiebreaker = next(io_loop._timeout_counter) - @staticmethod - def timedelta_to_seconds(td): - """Equivalent to td.total_seconds() (introduced in python 2.7).""" - return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / float(10 ** 6) - # Comparison methods to sort by deadline, with object id as a tiebreaker # to guarantee a consistent ordering. The heapq module uses __le__ # in python2.5, and __lt__ in 2.6+ (sort() and most other comparisons @@ -904,10 +969,11 @@ class PeriodicCallback(object): if not self._running: return try: - self.callback() + return self.callback() except Exception: self.io_loop.handle_callback_exception(self.callback) - self._schedule_next() + finally: + self._schedule_next() def _schedule_next(self): if self._running: diff --git a/libs/tornado/iostream.py b/libs/tornado/iostream.py index 3874bf7..99c681d 100755 --- a/libs/tornado/iostream.py +++ b/libs/tornado/iostream.py @@ -57,11 +57,24 @@ except ImportError: # some they differ. _ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN) +if hasattr(errno, "WSAEWOULDBLOCK"): + _ERRNO_WOULDBLOCK += (errno.WSAEWOULDBLOCK,) + # These errnos indicate that a connection has been abruptly terminated. # They should be caught and handled less noisily than other errors. -_ERRNO_CONNRESET = (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE) +_ERRNO_CONNRESET = (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE, + errno.ETIMEDOUT) + +if hasattr(errno, "WSAECONNRESET"): + _ERRNO_CONNRESET += (errno.WSAECONNRESET, errno.WSAECONNABORTED, errno.WSAETIMEDOUT) + +# More non-portable errnos: +_ERRNO_INPROGRESS = (errno.EINPROGRESS,) +if hasattr(errno, "WSAEINPROGRESS"): + _ERRNO_INPROGRESS += (errno.WSAEINPROGRESS,) +####################################################### class StreamClosedError(IOError): """Exception raised by `IOStream` methods when the stream is closed. @@ -116,7 +129,7 @@ class BaseIOStream(object): :arg max_write_buffer_size: Amount of outgoing data to buffer; defaults to unlimited. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Add the ``max_write_buffer_size`` parameter. Changed default ``read_chunk_size`` to 64KB. """ @@ -203,7 +216,7 @@ class BaseIOStream(object): if more than ``max_bytes`` bytes have been read and the regex is not satisfied. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Added the ``max_bytes`` argument. The ``callback`` argument is now optional and a `.Future` will be returned if it is omitted. """ @@ -230,7 +243,7 @@ class BaseIOStream(object): if more than ``max_bytes`` bytes have been read and the delimiter is not found. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Added the ``max_bytes`` argument. The ``callback`` argument is now optional and a `.Future` will be returned if it is omitted. """ @@ -259,7 +272,7 @@ class BaseIOStream(object): If ``partial`` is true, the callback is run as soon as we have any bytes to return (but never more than ``num_bytes``) - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Added the ``partial`` argument. The callback argument is now optional and a `.Future` will be returned if it is omitted. """ @@ -280,7 +293,7 @@ class BaseIOStream(object): If a callback is given, it will be run with the data as an argument; if not, this method returns a `.Future`. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 The callback argument is now optional and a `.Future` will be returned if it is omitted. """ @@ -308,7 +321,7 @@ class BaseIOStream(object): completed. If `write` is called again before that `.Future` has resolved, the previous future will be orphaned and will never resolve. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Now returns a `.Future` if no callback is given. """ assert isinstance(data, bytes_type) @@ -492,7 +505,7 @@ class BaseIOStream(object): def wrapper(): self._pending_callbacks -= 1 try: - callback(*args) + return callback(*args) except Exception: app_log.error("Uncaught exception, closing connection.", exc_info=True) @@ -504,7 +517,8 @@ class BaseIOStream(object): # Re-raise the exception so that IOLoop.handle_callback_exception # can see it and log the error raise - self._maybe_add_error_listener() + finally: + self._maybe_add_error_listener() # We schedule callbacks to be run on the next IOLoop iteration # rather than running them directly for several reasons: # * Prevents unbounded stack growth when a callback calls an @@ -949,11 +963,19 @@ class IOStream(BaseIOStream): May only be called if the socket passed to the constructor was not previously connected. The address parameter is in the - same format as for `socket.connect `, - i.e. a ``(host, port)`` tuple. If ``callback`` is specified, - it will be called with no arguments when the connection is - completed; if not this method returns a `.Future` (whose result - after a successful connection will be the stream itself). + same format as for `socket.connect ` for + the type of socket passed to the IOStream constructor, + e.g. an ``(ip, port)`` tuple. Hostnames are accepted here, + but will be resolved synchronously and block the IOLoop. + If you have a hostname instead of an IP address, the `.TCPClient` + class is recommended instead of calling this method directly. + `.TCPClient` will do asynchronous DNS resolution and handle + both IPv4 and IPv6. + + If ``callback`` is specified, it will be called with no + arguments when the connection is completed; if not this method + returns a `.Future` (whose result after a successful + connection will be the stream itself). If specified, the ``server_hostname`` parameter will be used in SSL connections for certificate validation (if requested in @@ -966,8 +988,9 @@ class IOStream(BaseIOStream): is ready. Calling `IOStream` read methods before the socket is connected works on some platforms but is non-portable. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 If no callback is given, returns a `.Future`. + """ self._connecting = True try: @@ -980,7 +1003,7 @@ class IOStream(BaseIOStream): # returned immediately when attempting to connect to # localhost, so handle them the same way as an error # reported later in _handle_connect. - if (errno_from_exception(e) != errno.EINPROGRESS and + if (errno_from_exception(e) not in _ERRNO_INPROGRESS and errno_from_exception(e) not in _ERRNO_WOULDBLOCK): gen_log.warning("Connect error on fd %s: %s", self.socket.fileno(), e) @@ -1021,7 +1044,7 @@ class IOStream(BaseIOStream): If a close callback is defined on this stream, it will be transferred to the new stream. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ if (self._read_callback or self._read_future or self._write_callback or self._write_future or diff --git a/libs/tornado/log.py b/libs/tornado/log.py index 7066466..374071d 100755 --- a/libs/tornado/log.py +++ b/libs/tornado/log.py @@ -179,7 +179,7 @@ class LogFormatter(logging.Formatter): def enable_pretty_logging(options=None, logger=None): """Turns on formatted logging output as configured. - This is called automaticaly by `tornado.options.parse_command_line` + This is called automatically by `tornado.options.parse_command_line` and `tornado.options.parse_config_file`. """ if options is None: diff --git a/libs/tornado/netutil.py b/libs/tornado/netutil.py index a9e05d1..336c806 100755 --- a/libs/tornado/netutil.py +++ b/libs/tornado/netutil.py @@ -57,6 +57,9 @@ u('foo').encode('idna') # some they differ. _ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN) +if hasattr(errno, "WSAEWOULDBLOCK"): + _ERRNO_WOULDBLOCK += (errno.WSAEWOULDBLOCK,) + def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags=None): """Creates listening sockets bound to the given port and address. diff --git a/libs/tornado/platform/asyncio.py b/libs/tornado/platform/asyncio.py index 552476b..b40f014 100644 --- a/libs/tornado/platform/asyncio.py +++ b/libs/tornado/platform/asyncio.py @@ -13,9 +13,9 @@ from __future__ import absolute_import, division, print_function, with_statement import datetime import functools -# _Timeout is used for its timedelta_to_seconds method for py26 compatibility. -from tornado.ioloop import IOLoop, _Timeout +from tornado.ioloop import IOLoop from tornado import stack_context +from tornado.util import timedelta_to_seconds try: # Import the real asyncio module for py33+ first. Older versions of the @@ -109,21 +109,13 @@ class BaseAsyncIOLoop(IOLoop): def stop(self): self.asyncio_loop.stop() - def _run_callback(self, callback, *args, **kwargs): - try: - callback(*args, **kwargs) - except Exception: - self.handle_callback_exception(callback) - - def add_timeout(self, deadline, callback): - if isinstance(deadline, (int, float)): - delay = max(deadline - self.time(), 0) - elif isinstance(deadline, datetime.timedelta): - delay = _Timeout.timedelta_to_seconds(deadline) - else: - raise TypeError("Unsupported deadline %r", deadline) - return self.asyncio_loop.call_later(delay, self._run_callback, - stack_context.wrap(callback)) + def call_at(self, when, callback, *args, **kwargs): + # asyncio.call_at supports *args but not **kwargs, so bind them here. + # We do not synchronize self.time and asyncio_loop.time, so + # convert from absolute to relative. + return self.asyncio_loop.call_later( + max(0, when - self.time()), self._run_callback, + functools.partial(stack_context.wrap(callback), *args, **kwargs)) def remove_timeout(self, timeout): timeout.cancel() @@ -131,13 +123,9 @@ class BaseAsyncIOLoop(IOLoop): def add_callback(self, callback, *args, **kwargs): if self.closing: raise RuntimeError("IOLoop is closing") - if kwargs: - self.asyncio_loop.call_soon_threadsafe(functools.partial( - self._run_callback, stack_context.wrap(callback), - *args, **kwargs)) - else: - self.asyncio_loop.call_soon_threadsafe( - self._run_callback, stack_context.wrap(callback), *args) + self.asyncio_loop.call_soon_threadsafe( + self._run_callback, + functools.partial(stack_context.wrap(callback), *args, **kwargs)) add_callback_from_signal = add_callback diff --git a/libs/tornado/platform/twisted.py b/libs/tornado/platform/twisted.py index 889aa3c..b271dfc 100755 --- a/libs/tornado/platform/twisted.py +++ b/libs/tornado/platform/twisted.py @@ -68,6 +68,7 @@ from __future__ import absolute_import, division, print_function, with_statement import datetime import functools +import numbers import socket import twisted.internet.abstract @@ -90,11 +91,7 @@ from tornado.log import app_log from tornado.netutil import Resolver from tornado.stack_context import NullContext, wrap from tornado.ioloop import IOLoop - -try: - long # py2 -except NameError: - long = int # py3 +from tornado.util import timedelta_to_seconds @implementer(IDelayedCall) @@ -475,28 +472,28 @@ class TwistedIOLoop(tornado.ioloop.IOLoop): def stop(self): self.reactor.crash() - def _run_callback(self, callback, *args, **kwargs): - try: - callback(*args, **kwargs) - except Exception: - self.handle_callback_exception(callback) - - def add_timeout(self, deadline, callback): - if isinstance(deadline, (int, long, float)): + def add_timeout(self, deadline, callback, *args, **kwargs): + # This method could be simplified (since tornado 4.0) by + # overriding call_at instead of add_timeout, but we leave it + # for now as a test of backwards-compatibility. + if isinstance(deadline, numbers.Real): delay = max(deadline - self.time(), 0) elif isinstance(deadline, datetime.timedelta): - delay = tornado.ioloop._Timeout.timedelta_to_seconds(deadline) + delay = timedelta_to_seconds(deadline) else: raise TypeError("Unsupported deadline %r") - return self.reactor.callLater(delay, self._run_callback, wrap(callback)) + return self.reactor.callLater( + delay, self._run_callback, + functools.partial(wrap(callback), *args, **kwargs)) def remove_timeout(self, timeout): if timeout.active(): timeout.cancel() def add_callback(self, callback, *args, **kwargs): - self.reactor.callFromThread(self._run_callback, - wrap(callback), *args, **kwargs) + self.reactor.callFromThread( + self._run_callback, + functools.partial(wrap(callback), *args, **kwargs)) def add_callback_from_signal(self, callback, *args, **kwargs): self.add_callback(callback, *args, **kwargs) diff --git a/libs/tornado/simple_httpclient.py b/libs/tornado/simple_httpclient.py index 06d7ecf..516dc20 100755 --- a/libs/tornado/simple_httpclient.py +++ b/libs/tornado/simple_httpclient.py @@ -277,7 +277,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): stream.close() return self.stream = stream - self.stream.set_close_callback(self._on_close) + self.stream.set_close_callback(self.on_connection_close) self._remove_timeout() if self.final_callback is None: return @@ -338,7 +338,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): if (self.request.method == "POST" and "Content-Type" not in self.request.headers): self.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - if self.request.use_gzip: + if self.request.decompress_response: self.request.headers["Accept-Encoding"] = "gzip" req_path = ((self.parsed.path or '/') + (('?' + self.parsed.query) if self.parsed.query else '')) @@ -348,7 +348,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): HTTP1ConnectionParameters( no_keep_alive=True, max_header_size=self.max_header_size, - use_gzip=self.request.use_gzip), + decompress=self.request.decompress_response), self._sockaddr) start_line = httputil.RequestStartLine(self.request.method, req_path, 'HTTP/1.1') @@ -418,12 +418,15 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): # pass it along, unless it's just the stream being closed. return isinstance(value, StreamClosedError) - def _on_close(self): + def on_connection_close(self): if self.final_callback is not None: message = "Connection closed" if self.stream.error: raise self.stream.error - raise HTTPError(599, message) + try: + raise HTTPError(599, message) + except HTTPError: + self._handle_exception(*sys.exc_info()) def headers_received(self, first_line, headers): if self.request.expect_100_continue and first_line.code == 100: @@ -433,20 +436,6 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): self.code = first_line.code self.reason = first_line.reason - if "Content-Length" in self.headers: - if "," in self.headers["Content-Length"]: - # Proxies sometimes cause Content-Length headers to get - # duplicated. If all the values are identical then we can - # use them but if they differ it's an error. - pieces = re.split(r',\s*', self.headers["Content-Length"]) - if any(i != pieces[0] for i in pieces): - raise ValueError("Multiple unequal Content-Lengths: %r" % - self.headers["Content-Length"]) - self.headers["Content-Length"] = pieces[0] - content_length = int(self.headers["Content-Length"]) - else: - content_length = None - if self.request.header_callback is not None: # Reassemble the start line. self.request.header_callback('%s %s %s\r\n' % first_line) @@ -454,14 +443,6 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): self.request.header_callback("%s: %s\r\n" % (k, v)) self.request.header_callback('\r\n') - if 100 <= self.code < 200 or self.code == 204: - # These response codes never have bodies - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3 - if ("Transfer-Encoding" in self.headers or - content_length not in (None, 0)): - raise ValueError("Response with code %d should not have body" % - self.code) - def finish(self): data = b''.join(self.chunks) self._remove_timeout() diff --git a/libs/tornado/testing.py b/libs/tornado/testing.py index dc30e94..b4bfb27 100755 --- a/libs/tornado/testing.py +++ b/libs/tornado/testing.py @@ -70,8 +70,8 @@ def get_unused_port(): only that a series of get_unused_port calls in a single process return distinct ports. - **Deprecated**. Use bind_unused_port instead, which is guaranteed - to find an unused port. + .. deprecated:: + Use bind_unused_port instead, which is guaranteed to find an unused port. """ global _next_port port = _next_port @@ -459,7 +459,7 @@ def gen_test(func=None, timeout=None): The ``timeout`` argument and ``ASYNC_TEST_TIMEOUT`` environment variable. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 The wrapper now passes along ``*args, **kwargs`` so it can be used on functions with arguments. """ diff --git a/libs/tornado/util.py b/libs/tornado/util.py index 49eea2c..b6e06c6 100755 --- a/libs/tornado/util.py +++ b/libs/tornado/util.py @@ -311,6 +311,11 @@ class ArgReplacer(object): return old_value, args, kwargs +def timedelta_to_seconds(td): + """Equivalent to td.total_seconds() (introduced in python 2.7).""" + return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / float(10 ** 6) + + def _websocket_mask_python(mask, data): """Websocket masking function. diff --git a/libs/tornado/web.py b/libs/tornado/web.py index dd2b5ef..25ac56e 100755 --- a/libs/tornado/web.py +++ b/libs/tornado/web.py @@ -35,8 +35,7 @@ Here is a simple "Hello, world" example app:: application.listen(8888) tornado.ioloop.IOLoop.instance().start() -See the :doc:`Tornado overview ` for more details and a good getting -started guide. +See the :doc:`guide` for additional information. Thread-safety notes ------------------- @@ -48,6 +47,7 @@ not thread-safe. In particular, methods such as you use multiple threads it is important to use `.IOLoop.add_callback` to transfer control back to the main thread before finishing the request. + """ from __future__ import absolute_import, division, print_function, with_statement @@ -820,7 +820,7 @@ class RequestHandler(object): if another flush occurs before the previous flush's callback has been run, the previous callback will be discarded. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Now returns a `.Future` if no callback is given. """ chunk = b"".join(self._write_buffer) @@ -943,26 +943,7 @@ class RequestHandler(object): ``kwargs["exc_info"]``. Note that this exception may not be the "current" exception for purposes of methods like ``sys.exc_info()`` or ``traceback.format_exc``. - - For historical reasons, if a method ``get_error_html`` exists, - it will be used instead of the default ``write_error`` implementation. - ``get_error_html`` returned a string instead of producing output - normally, and had different semantics for exception handling. - Users of ``get_error_html`` are encouraged to convert their code - to override ``write_error`` instead. - """ - if hasattr(self, 'get_error_html'): - if 'exc_info' in kwargs: - exc_info = kwargs.pop('exc_info') - kwargs['exception'] = exc_info[1] - try: - # Put the traceback into sys.exc_info() - raise_exc_info(exc_info) - except Exception: - self.finish(self.get_error_html(status_code, **kwargs)) - else: - self.finish(self.get_error_html(status_code, **kwargs)) - return + """ if self.settings.get("serve_traceback") and "exc_info" in kwargs: # in debug mode, try to send a traceback self.set_header('Content-Type', 'text/plain') @@ -1147,14 +1128,15 @@ class RequestHandler(object): else: # Treat unknown versions as not present instead of failing. return None, None, None - elif len(cookie) == 32: + else: version = 1 - token = binascii.a2b_hex(utf8(cookie)) + try: + token = binascii.a2b_hex(utf8(cookie)) + except (binascii.Error, TypeError): + token = utf8(cookie) # We don't have a usable timestamp in older versions. timestamp = int(time.time()) return (version, token, timestamp) - else: - return None, None, None def check_xsrf_cookie(self): """Verifies that the ``_xsrf`` cookie matches the ``_xsrf`` argument. @@ -1242,27 +1224,6 @@ class RequestHandler(object): return base + get_url(self.settings, path, **kwargs) - def async_callback(self, callback, *args, **kwargs): - """Obsolete - catches exceptions from the wrapped function. - - This function is unnecessary since Tornado 1.1. - """ - if callback is None: - return None - if args or kwargs: - callback = functools.partial(callback, *args, **kwargs) - - def wrapper(*args, **kwargs): - try: - return callback(*args, **kwargs) - except Exception as e: - if self._headers_written: - app_log.error("Exception after headers written", - exc_info=True) - else: - self._handle_request_exception(e) - return wrapper - def require_setting(self, name, feature="this feature"): """Raises an exception if the given app setting is not defined.""" if not self.application.settings.get(name): @@ -1405,6 +1366,11 @@ class RequestHandler(object): " (" + self.request.remote_ip + ")" def _handle_request_exception(self, e): + if isinstance(e, Finish): + # Not an error; just finish the request without logging. + if not self._finished: + self.finish() + return self.log_exception(*sys.exc_info()) if self._finished: # Extra errors after the request has been finished should @@ -1662,7 +1628,7 @@ class Application(httputil.HTTPServerConnectionDelegate): **settings): if transforms is None: self.transforms = [] - if settings.get("gzip"): + if settings.get("compress_response") or settings.get("gzip"): self.transforms.append(GZipContentEncoding) else: self.transforms = transforms @@ -1959,6 +1925,9 @@ class HTTPError(Exception): `RequestHandler.send_error` since it automatically ends the current function. + To customize the response sent with an `HTTPError`, override + `RequestHandler.write_error`. + :arg int status_code: HTTP status code. Must be listed in `httplib.responses ` unless the ``reason`` keyword argument is given. @@ -1987,6 +1956,25 @@ class HTTPError(Exception): return message +class Finish(Exception): + """An exception that ends the request without producing an error response. + + When `Finish` is raised in a `RequestHandler`, the request will end + (calling `RequestHandler.finish` if it hasn't already been called), + but the outgoing response will not be modified and the error-handling + methods (including `RequestHandler.write_error`) will not be called. + + This can be a more convenient way to implement custom error pages + than overriding ``write_error`` (especially in library code):: + + if self.current_user is None: + self.set_status(401) + self.set_header('WWW-Authenticate', 'Basic realm="something"') + raise Finish() + """ + pass + + class MissingArgumentError(HTTPError): """Exception raised by `RequestHandler.get_argument`. @@ -2367,7 +2355,7 @@ class StaticFileHandler(RequestHandler): .. versionadded:: 3.1 - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 This method is now always called, instead of only when partial results are requested. """ @@ -2514,9 +2502,9 @@ class FallbackHandler(RequestHandler): class OutputTransform(object): """A transform modifies the result of an HTTP request (e.g., GZip encoding) - A new transform instance is created for every request. See the - GZipContentEncoding example below if you want to implement a - new Transform. + Applications are not expected to create their own OutputTransforms + or interact with them directly; the framework chooses which transforms + (if any) to apply. """ def __init__(self, request): pass @@ -2533,7 +2521,7 @@ class GZipContentEncoding(OutputTransform): See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11 - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Now compresses all mime types beginning with ``text/``, instead of just a whitelist. (the whitelist is still used for certain non-text mime types). @@ -2767,7 +2755,7 @@ class URLSpec(object): in the regex will be passed in to the handler's get/post/etc methods as arguments. - * ``handler_class``: `RequestHandler` subclass to be invoked. + * ``handler``: `RequestHandler` subclass to be invoked. * ``kwargs`` (optional): A dictionary of additional arguments to be passed to the handler's constructor. diff --git a/libs/tornado/websocket.py b/libs/tornado/websocket.py index c0065c7..ed520d5 100755 --- a/libs/tornado/websocket.py +++ b/libs/tornado/websocket.py @@ -3,18 +3,17 @@ `WebSockets `_ allow for bidirectional communication between the browser and server. -.. warning:: - - The WebSocket protocol was recently finalized as `RFC 6455 - `_ and is not yet supported in - all browsers. Refer to http://caniuse.com/websockets for details - on compatibility. In addition, during development the protocol - went through several incompatible versions, and some browsers only - support older versions. By default this module only supports the - latest version of the protocol, but optional support for an older - version (known as "draft 76" or "hixie-76") can be enabled by - overriding `WebSocketHandler.allow_draft76` (see that method's - documentation for caveats). +WebSockets are supported in the current versions of all major browsers, +although older versions that do not support WebSockets are still in use +(refer to http://caniuse.com/websockets for details). + +This module implements the final version of the WebSocket protocol as +defined in `RFC 6455 `_. Certain +browser versions (notably Safari 5.x) implemented an earlier draft of +the protocol (known as "draft 76") and are not compatible with this module. + +.. versionchanged:: 4.0 + Removed support for the draft 76 protocol version. """ from __future__ import absolute_import, division, print_function, with_statement @@ -22,11 +21,9 @@ from __future__ import absolute_import, division, print_function, with_statement import base64 import collections -import functools import hashlib import os import struct -import time import tornado.escape import tornado.web @@ -38,7 +35,7 @@ from tornado.iostream import StreamClosedError from tornado.log import gen_log, app_log from tornado import simple_httpclient from tornado.tcpclient import TCPClient -from tornado.util import bytes_type, unicode_type, _websocket_mask +from tornado.util import bytes_type, _websocket_mask try: from urllib.parse import urlparse # py2 @@ -108,6 +105,21 @@ class WebSocketHandler(tornado.web.RequestHandler): }; This script pops up an alert box that says "You said: Hello, world". + + Web browsers allow any site to open a websocket connection to any other, + instead of using the same-origin policy that governs other network + access from javascript. This can be surprising and is a potential + security hole, so since Tornado 4.0 `WebSocketHandler` requires + applications that wish to receive cross-origin websockets to opt in + by overriding the `~WebSocketHandler.check_origin` method (see that + method's docs for details). Failure to do so is the most likely + cause of 403 errors when making a websocket connection. + + When using a secure websocket connection (``wss://``) with a self-signed + certificate, the connection from a browser may fail because it wants + to show the "accept this certificate" dialog but has nowhere to show it. + You must first visit a regular HTML page using the same certificate + to accept it before the websocket connection will succeed. """ def __init__(self, application, request, **kwargs): tornado.web.RequestHandler.__init__(self, application, request, @@ -115,22 +127,17 @@ class WebSocketHandler(tornado.web.RequestHandler): self.ws_connection = None self.close_code = None self.close_reason = None + self.stream = None @tornado.web.asynchronous def get(self, *args, **kwargs): self.open_args = args self.open_kwargs = kwargs - self.stream = self.request.connection.detach() - self.stream.set_close_callback(self.on_connection_close) - # Upgrade header should be present and should be equal to WebSocket if self.request.headers.get("Upgrade", "").lower() != 'websocket': - self.stream.write(tornado.escape.utf8( - "HTTP/1.1 400 Bad Request\r\n\r\n" - "Can \"Upgrade\" only to \"WebSocket\"." - )) - self.stream.close() + self.set_status(400) + self.finish("Can \"Upgrade\" only to \"WebSocket\".") return # Connection header should be upgrade. Some proxy servers/load balancers @@ -138,11 +145,8 @@ class WebSocketHandler(tornado.web.RequestHandler): headers = self.request.headers connection = map(lambda s: s.strip().lower(), headers.get("Connection", "").split(",")) if 'upgrade' not in connection: - self.stream.write(tornado.escape.utf8( - "HTTP/1.1 400 Bad Request\r\n\r\n" - "\"Connection\" must be \"Upgrade\"." - )) - self.stream.close() + self.set_status(400) + self.finish("\"Connection\" must be \"Upgrade\".") return # Handle WebSocket Origin naming convention differences @@ -159,19 +163,16 @@ class WebSocketHandler(tornado.web.RequestHandler): # according to check_origin. When the origin is None, we assume it # did not come from a browser and that it can be passed on. if origin is not None and not self.check_origin(origin): - self.stream.write(tornado.escape.utf8( - "HTTP/1.1 403 Cross Origin Websockets Disabled\r\n\r\n" - )) - self.stream.close() + self.set_status(403) + self.finish("Cross origin websockets not allowed") return + self.stream = self.request.connection.detach() + self.stream.set_close_callback(self.on_connection_close) + if self.request.headers.get("Sec-WebSocket-Version") in ("7", "8", "13"): self.ws_connection = WebSocketProtocol13(self) self.ws_connection.accept_connection() - elif (self.allow_draft76() and - "Sec-WebSocket-Version" not in self.request.headers): - self.ws_connection = WebSocketProtocol76(self) - self.ws_connection.accept_connection() else: self.stream.write(tornado.escape.utf8( "HTTP/1.1 426 Upgrade Required\r\n" @@ -245,7 +246,7 @@ class WebSocketHandler(tornado.web.RequestHandler): phrase was supplied, these values will be available as the attributes ``self.close_code`` and ``self.close_reason``. - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Added ``close_code`` and ``close_reason`` attributes. """ @@ -263,10 +264,7 @@ class WebSocketHandler(tornado.web.RequestHandler): closing. These values are made available to the client, but are not otherwise interpreted by the websocket protocol. - The ``code`` and ``reason`` arguments are ignored in the "draft76" - protocol version. - - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Added the ``code`` and ``reason`` arguments. """ @@ -292,7 +290,20 @@ class WebSocketHandler(tornado.web.RequestHandler): browsers, since WebSockets are allowed to bypass the usual same-origin policies and don't use CORS headers. - .. versionadded:: 3.3 + To accept all cross-origin traffic (which was the default prior to + Tornado 4.0), simply override this method to always return true:: + + def check_origin(self, origin): + return True + + To allow connections from any subdomain of your site, you might + do something like:: + + def check_origin(self, origin): + parsed_origin = urllib.parse.urlparse(origin) + return parsed_origin.netloc.endswith(".mydomain.com") + + .. versionadded:: 4.0 """ parsed_origin = urlparse(origin) origin = parsed_origin.netloc @@ -303,21 +314,6 @@ class WebSocketHandler(tornado.web.RequestHandler): # Check to see that origin matches host directly, including ports return origin == host - def allow_draft76(self): - """Override to enable support for the older "draft76" protocol. - - The draft76 version of the websocket protocol is disabled by - default due to security concerns, but it can be enabled by - overriding this method to return True. - - Connections using the draft76 protocol do not support the - ``binary=True`` flag to `write_message`. - - Support for the draft76 protocol is deprecated and will be - removed in a future version of Tornado. - """ - return False - def set_nodelay(self, value): """Set the no-delay flag for this stream. @@ -334,29 +330,6 @@ class WebSocketHandler(tornado.web.RequestHandler): """ self.stream.set_nodelay(value) - def get_websocket_scheme(self): - """Return the url scheme used for this request, either "ws" or "wss". - - This is normally decided by HTTPServer, but applications - may wish to override this if they are using an SSL proxy - that does not provide the X-Scheme header as understood - by HTTPServer. - - Note that this is only used by the draft76 protocol. - """ - return "wss" if self.request.protocol == "https" else "ws" - - def async_callback(self, callback, *args, **kwargs): - """Obsolete - catches exceptions from the wrapped function. - - This function is normally unncecessary thanks to - `tornado.stack_context`. - """ - return self.ws_connection.async_callback(callback, *args, **kwargs) - - def _not_supported(self, *args, **kwargs): - raise Exception("Method not supported for Web Sockets") - def on_connection_close(self): if self.ws_connection: self.ws_connection.on_connection_close() @@ -364,9 +337,17 @@ class WebSocketHandler(tornado.web.RequestHandler): self.on_close() +def _wrap_method(method): + def _disallow_for_websocket(self, *args, **kwargs): + if self.stream is None: + method(self, *args, **kwargs) + else: + raise RuntimeError("Method not supported for Web Sockets") + return _disallow_for_websocket for method in ["write", "redirect", "set_header", "send_error", "set_cookie", "set_status", "flush", "finish"]: - setattr(WebSocketHandler, method, WebSocketHandler._not_supported) + setattr(WebSocketHandler, method, + _wrap_method(getattr(WebSocketHandler, method))) class WebSocketProtocol(object): @@ -379,23 +360,17 @@ class WebSocketProtocol(object): self.client_terminated = False self.server_terminated = False - def async_callback(self, callback, *args, **kwargs): - """Wrap callbacks with this if they are used on asynchronous requests. + def _run_callback(self, callback, *args, **kwargs): + """Runs the given callback with exception handling. - Catches exceptions properly and closes this WebSocket if an exception - is uncaught. + On error, aborts the websocket connection and returns False. """ - if args or kwargs: - callback = functools.partial(callback, *args, **kwargs) - - def wrapper(*args, **kwargs): - try: - return callback(*args, **kwargs) - except Exception: - app_log.error("Uncaught exception in %s", - self.request.path, exc_info=True) - self._abort() - return wrapper + try: + callback(*args, **kwargs) + except Exception: + app_log.error("Uncaught exception in %s", + self.request.path, exc_info=True) + self._abort() def on_connection_close(self): self._abort() @@ -408,174 +383,6 @@ class WebSocketProtocol(object): self.close() # let the subclass cleanup -class WebSocketProtocol76(WebSocketProtocol): - """Implementation of the WebSockets protocol, version hixie-76. - - This class provides basic functionality to process WebSockets requests as - specified in - http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-76 - """ - def __init__(self, handler): - WebSocketProtocol.__init__(self, handler) - self.challenge = None - self._waiting = None - - def accept_connection(self): - try: - self._handle_websocket_headers() - except ValueError: - gen_log.debug("Malformed WebSocket request received") - self._abort() - return - - scheme = self.handler.get_websocket_scheme() - - # draft76 only allows a single subprotocol - subprotocol_header = '' - subprotocol = self.request.headers.get("Sec-WebSocket-Protocol", None) - if subprotocol: - selected = self.handler.select_subprotocol([subprotocol]) - if selected: - assert selected == subprotocol - subprotocol_header = "Sec-WebSocket-Protocol: %s\r\n" % selected - - # Write the initial headers before attempting to read the challenge. - # This is necessary when using proxies (such as HAProxy), which - # need to see the Upgrade headers before passing through the - # non-HTTP traffic that follows. - self.stream.write(tornado.escape.utf8( - "HTTP/1.1 101 WebSocket Protocol Handshake\r\n" - "Upgrade: WebSocket\r\n" - "Connection: Upgrade\r\n" - "Server: TornadoServer/%(version)s\r\n" - "Sec-WebSocket-Origin: %(origin)s\r\n" - "Sec-WebSocket-Location: %(scheme)s://%(host)s%(uri)s\r\n" - "%(subprotocol)s" - "\r\n" % (dict( - version=tornado.version, - origin=self.request.headers["Origin"], - scheme=scheme, - host=self.request.host, - uri=self.request.uri, - subprotocol=subprotocol_header)))) - self.stream.read_bytes(8, self._handle_challenge) - - def challenge_response(self, challenge): - """Generates the challenge response that's needed in the handshake - - The challenge parameter should be the raw bytes as sent from the - client. - """ - key_1 = self.request.headers.get("Sec-Websocket-Key1") - key_2 = self.request.headers.get("Sec-Websocket-Key2") - try: - part_1 = self._calculate_part(key_1) - part_2 = self._calculate_part(key_2) - except ValueError: - raise ValueError("Invalid Keys/Challenge") - return self._generate_challenge_response(part_1, part_2, challenge) - - def _handle_challenge(self, challenge): - try: - challenge_response = self.challenge_response(challenge) - except ValueError: - gen_log.debug("Malformed key data in WebSocket request") - self._abort() - return - self._write_response(challenge_response) - - def _write_response(self, challenge): - self.stream.write(challenge) - self.async_callback(self.handler.open)(*self.handler.open_args, **self.handler.open_kwargs) - self._receive_message() - - def _handle_websocket_headers(self): - """Verifies all invariant- and required headers - - If a header is missing or have an incorrect value ValueError will be - raised - """ - fields = ("Origin", "Host", "Sec-Websocket-Key1", - "Sec-Websocket-Key2") - if not all(map(lambda f: self.request.headers.get(f), fields)): - raise ValueError("Missing/Invalid WebSocket headers") - - def _calculate_part(self, key): - """Processes the key headers and calculates their key value. - - Raises ValueError when feed invalid key.""" - # pyflakes complains about variable reuse if both of these lines use 'c' - number = int(''.join(c for c in key if c.isdigit())) - spaces = len([c2 for c2 in key if c2.isspace()]) - try: - key_number = number // spaces - except (ValueError, ZeroDivisionError): - raise ValueError - return struct.pack(">I", key_number) - - def _generate_challenge_response(self, part_1, part_2, part_3): - m = hashlib.md5() - m.update(part_1) - m.update(part_2) - m.update(part_3) - return m.digest() - - def _receive_message(self): - self.stream.read_bytes(1, self._on_frame_type) - - def _on_frame_type(self, byte): - frame_type = ord(byte) - if frame_type == 0x00: - self.stream.read_until(b"\xff", self._on_end_delimiter) - elif frame_type == 0xff: - self.stream.read_bytes(1, self._on_length_indicator) - else: - self._abort() - - def _on_end_delimiter(self, frame): - if not self.client_terminated: - self.async_callback(self.handler.on_message)( - frame[:-1].decode("utf-8", "replace")) - if not self.client_terminated: - self._receive_message() - - def _on_length_indicator(self, byte): - if ord(byte) != 0x00: - self._abort() - return - self.client_terminated = True - self.close() - - def write_message(self, message, binary=False): - """Sends the given message to the client of this Web Socket.""" - if binary: - raise ValueError( - "Binary messages not supported by this version of websockets") - if isinstance(message, unicode_type): - message = message.encode("utf-8") - assert isinstance(message, bytes_type) - self.stream.write(b"\x00" + message + b"\xff") - - def write_ping(self, data): - """Send ping frame.""" - raise ValueError("Ping messages not supported by this version of websockets") - - def close(self, code=None, reason=None): - """Closes the WebSocket connection.""" - if not self.server_terminated: - if not self.stream.closed(): - self.stream.write("\xff\x00") - self.server_terminated = True - if self.client_terminated: - if self._waiting is not None: - self.stream.io_loop.remove_timeout(self._waiting) - self._waiting = None - self.stream.close() - elif self._waiting is None: - self._waiting = self.stream.io_loop.add_timeout( - time.time() + 5, self._abort) - - class WebSocketProtocol13(WebSocketProtocol): """Implementation of the WebSocket protocol from RFC 6455. @@ -645,7 +452,8 @@ class WebSocketProtocol13(WebSocketProtocol): "%s" "\r\n" % (self._challenge_response(), subprotocol_header))) - self.async_callback(self.handler.open)(*self.handler.open_args, **self.handler.open_kwargs) + self._run_callback(self.handler.open, *self.handler.open_args, + **self.handler.open_kwargs) self._receive_frame() def _write_frame(self, fin, opcode, data): @@ -803,10 +611,10 @@ class WebSocketProtocol13(WebSocketProtocol): except UnicodeDecodeError: self._abort() return - self.async_callback(self.handler.on_message)(decoded) + self._run_callback(self.handler.on_message, decoded) elif opcode == 0x2: # Binary data - self.async_callback(self.handler.on_message)(data) + self._run_callback(self.handler.on_message, data) elif opcode == 0x8: # Close self.client_terminated = True @@ -820,7 +628,7 @@ class WebSocketProtocol13(WebSocketProtocol): self._write_frame(True, 0xA, data) elif opcode == 0xA: # Pong - self.async_callback(self.handler.on_pong)(data) + self._run_callback(self.handler.on_pong, data) else: self._abort() @@ -885,7 +693,7 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): .. versionadded:: 3.2 - .. versionchanged:: 3.3 + .. versionchanged:: 4.0 Added the ``code`` and ``reason`` arguments. """ @@ -893,10 +701,12 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): self.protocol.close(code, reason) self.protocol = None - def _on_close(self): + def on_connection_close(self): + if not self.connect_future.done(): + self.connect_future.set_exception(StreamClosedError()) self.on_message(None) - self.resolver.close() - super(WebSocketClientConnection, self)._on_close() + self.tcp_client.close() + super(WebSocketClientConnection, self).on_connection_close() def _on_http_response(self, response): if not self.connect_future.done(): @@ -925,7 +735,12 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): self._timeout = None self.stream = self.connection.detach() - self.stream.set_close_callback(self._on_close) + self.stream.set_close_callback(self.on_connection_close) + # Once we've taken over the connection, clear the final callback + # we set on the http request. This deactivates the error handling + # in simple_httpclient that would otherwise interfere with our + # ability to see exceptions. + self.final_callback = None self.connect_future.set_result(self) diff --git a/libs/tornado/wsgi.py b/libs/tornado/wsgi.py index 47a0590..6e115e1 100755 --- a/libs/tornado/wsgi.py +++ b/libs/tornado/wsgi.py @@ -77,7 +77,7 @@ else: class WSGIApplication(web.Application): """A WSGI equivalent of `tornado.web.Application`. - .. deprecated: 3.3:: + .. deprecated:: 4.0 Use a regular `.Application` and wrap it in `WSGIAdapter` instead. """ @@ -126,7 +126,7 @@ class _WSGIConnection(httputil.HTTPConnection): if self._expected_content_remaining is not None: self._expected_content_remaining -= len(chunk) if self._expected_content_remaining < 0: - self._error = httputil.HTTPOutputException( + self._error = httputil.HTTPOutputError( "Tried to write more data than Content-Length") raise self._error self._write_buffer.append(chunk) @@ -137,7 +137,7 @@ class _WSGIConnection(httputil.HTTPConnection): def finish(self): if (self._expected_content_remaining is not None and self._expected_content_remaining != 0): - self._error = httputil.HTTPOutputException( + self._error = httputil.HTTPOutputError( "Tried to write %d bytes less than Content-Length" % self._expected_content_remaining) raise self._error @@ -183,7 +183,7 @@ class WSGIAdapter(object): that it is not possible to use `.AsyncHTTPClient`, or the `tornado.auth` or `tornado.websocket` modules. - .. versionadded:: 3.3 + .. versionadded:: 4.0 """ def __init__(self, application): if isinstance(application, WSGIApplication): From 87e97cd8a5874f248646fa5d9157039772cb571f Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 30 Aug 2014 17:06:51 +0200 Subject: [PATCH 077/202] Only show poster when available fix #3866 --- couchpotato/core/media/movie/_base/static/movie.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/_base/static/movie.js b/couchpotato/core/media/movie/_base/static/movie.js index 4788008..669546b 100644 --- a/couchpotato/core/media/movie/_base/static/movie.js +++ b/couchpotato/core/media/movie/_base/static/movie.js @@ -159,7 +159,7 @@ var Movie = new Class({ } } }), - self.thumbnail = (self.data.files && self.data.files.image_poster) ? new Element('img', { + self.thumbnail = (self.data.files && self.data.files.image_poster && self.data.files.image_poster.length > 0) ? new Element('img', { 'class': 'type_image poster', 'src': Api.createUrl('file.cache') + self.data.files.image_poster[0].split(Api.getOption('path_sep')).pop() }): null, From 4a1f70da0978ce4ad860c73f904b111ed8aa2aa2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 31 Aug 2014 00:23:06 +0200 Subject: [PATCH 078/202] ConnectionError not caught properly --- couchpotato/core/notifications/xbmc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/notifications/xbmc.py b/couchpotato/core/notifications/xbmc.py index bf5310e..1eef709 100644 --- a/couchpotato/core/notifications/xbmc.py +++ b/couchpotato/core/notifications/xbmc.py @@ -7,8 +7,8 @@ import urllib from couchpotato.core.helpers.variable import splitString, getTitle from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification -import requests -from requests.packages.urllib3.exceptions import MaxRetryError, ConnectionError +from requests.exceptions import ConnectionError, Timeout +from requests.packages.urllib3.exceptions import MaxRetryError log = CPLog(__name__) @@ -172,7 +172,7 @@ class XBMC(Notification): # manually fake expected response array return [{'result': 'Error'}] - except (MaxRetryError, requests.exceptions.Timeout, ConnectionError): + except (MaxRetryError, Timeout, ConnectionError): log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off') return [{'result': 'Error'}] except: @@ -208,7 +208,7 @@ class XBMC(Notification): log.debug('Returned from request %s: %s', (host, response)) return response - except (MaxRetryError, requests.exceptions.Timeout, ConnectionError): + except (MaxRetryError, Timeout, ConnectionError): log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off') return [] except: From c5a0d521d14ff28261343abdc58c9aac42625182 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 31 Aug 2014 01:11:40 +0200 Subject: [PATCH 079/202] Simplify coming soon / late listing --- couchpotato/core/plugins/dashboard.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/couchpotato/core/plugins/dashboard.py b/couchpotato/core/plugins/dashboard.py index d8ea4c9..d4af7ad 100644 --- a/couchpotato/core/plugins/dashboard.py +++ b/couchpotato/core/plugins/dashboard.py @@ -1,4 +1,3 @@ -from datetime import date import random as rndm import time @@ -48,7 +47,6 @@ class Dashboard(Plugin): active_ids = [x['_id'] for x in fireEvent('media.with_status', 'active', with_doc = False, single = True)] medias = [] - now_year = date.today().year if len(active_ids) > 0: @@ -70,22 +68,25 @@ class Dashboard(Plugin): # Theater quality if pp.get('theater') and fireEvent('movie.searcher.could_be_released', True, eta, media['info']['year'], single = True): - coming_soon = True + coming_soon = 'theater' elif pp.get('dvd') and fireEvent('movie.searcher.could_be_released', False, eta, media['info']['year'], single = True): - coming_soon = True + coming_soon = 'dvd' if coming_soon: # Don't list older movies - if ((not late and (media['info']['year'] >= now_year - 1) and (not eta.get('dvd') and not eta.get('theater') or eta.get('dvd') and eta.get('dvd') > (now - 2419200))) or - (late and (media['info']['year'] < now_year - 1 or (eta.get('dvd', 0) > 0 or eta.get('theater')) and eta.get('dvd') < (now - 2419200)))): + eta_date = eta.get(coming_soon) + eta_3month_passed = eta_date < (now - 7862400) # Release was more than 3 months ago + + if (not late and not eta_3month_passed) or \ + (late and eta_3month_passed): add = True # Check if it doesn't have any releases if late: media['releases'] = fireEvent('release.for_media', media['_id'], single = True) - + for release in media.get('releases'): if release.get('status') in ['snatched', 'available', 'seeding', 'downloaded']: add = False From 0b14fe545435cdc40bbcb22d8eb9657d3d581696 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 31 Aug 2014 15:02:35 +0200 Subject: [PATCH 080/202] Look at all releases for restatus "done" --- couchpotato/core/media/_base/media/main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 1544c68..11747c8 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -492,12 +492,13 @@ class MediaPlugin(MediaBase): done_releases = [release for release in media_releases if release.get('status') == 'done'] if done_releases: - # Only look at latest added release - release = sorted(done_releases, key = itemgetter('last_edit'), reverse = True)[0] # Check if we are finished with the media - if fireEvent('quality.isfinish', {'identifier': release['quality'], 'is_3d': release.get('is_3d', False)}, profile, timedelta(seconds = time.time() - release['last_edit']).days, single = True): - m['status'] = 'done' + for release in done_releases: + if fireEvent('quality.isfinish', {'identifier': release['quality'], 'is_3d': release.get('is_3d', False)}, profile, timedelta(seconds = time.time() - release['last_edit']).days, single = True): + m['status'] = 'done' + break + elif previous_status == 'done': m['status'] = 'done' From 286f14a6d28345c3c06ad089d3516b15f71fbfb4 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 31 Aug 2014 15:03:10 +0200 Subject: [PATCH 081/202] Return titles included in video headers --- couchpotato/core/plugins/scanner.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/scanner.py b/couchpotato/core/plugins/scanner.py index 3d39b29..8e835b5 100644 --- a/couchpotato/core/plugins/scanner.py +++ b/couchpotato/core/plugins/scanner.py @@ -11,7 +11,6 @@ from couchpotato.core.helpers.variable import getExt, getImdb, tryInt, \ splitString, getIdentifier from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin -from enzyme.exceptions import NoParserError, ParseError from guessit import guess_movie_info from subliminal.videos import Video import enzyme @@ -457,6 +456,7 @@ class Scanner(Plugin): meta = self.getMeta(cur_file) try: + data['titles'] = meta.get('titles', []) data['video'] = meta.get('video', self.getCodec(cur_file, self.codecs['video'])) data['audio'] = meta.get('audio', self.getCodec(cur_file, self.codecs['audio'])) data['audio_channels'] = meta.get('audio_channels', 2.0) @@ -527,16 +527,33 @@ class Scanner(Plugin): try: ac = self.audio_codec_map.get(p.audio[0].codec) except: pass + # Find title in video headers + titles = [] + + try: + if p.title and self.findYear(p.title): + titles.append(ss(p.title)) + except: + log.error('Failed getting title from meta: %s', traceback.format_exc()) + + for video in p.video: + try: + if video.title and self.findYear(video.title): + titles.append(ss(video.title)) + except: + log.error('Failed getting title from meta: %s', traceback.format_exc()) + return { + 'titles': list(set(titles)), 'video': vc, 'audio': ac, 'resolution_width': tryInt(p.video[0].width), 'resolution_height': tryInt(p.video[0].height), 'audio_channels': p.audio[0].channels, } - except ParseError: + except enzyme.exceptions.ParseError: log.debug('Failed to parse meta for %s', filename) - except NoParserError: + except enzyme.exceptions.NoParserError: log.debug('No parser found for %s', filename) except: log.debug('Failed parsing %s', filename) From 08c381cf0d394b2d57ba974b34e27b8053de3ecf Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 31 Aug 2014 15:03:51 +0200 Subject: [PATCH 082/202] Use video metadata titles for scoring --- couchpotato/core/plugins/quality/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 20e6b07..353aac1 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -207,6 +207,10 @@ class QualityPlugin(Plugin): '3d': {} } + # Use metadata titles as extra check + if extra and extra.get('titles'): + files.extend(extra.get('titles')) + for cur_file in files: words = re.split('\W+', cur_file.lower()) name_year = fireEvent('scanner.name_year', cur_file, file_name = cur_file, single = True) From 83b4c17969f77b82aa7d84052f4fe28fb0d34e19 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 31 Aug 2014 16:26:25 +0200 Subject: [PATCH 083/202] Better quality guessing based on size --- couchpotato/core/plugins/quality/main.py | 58 +++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 353aac1..10364a8 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -1,3 +1,4 @@ +from math import fabs, ceil import traceback import re @@ -6,7 +7,7 @@ from couchpotato import get_db from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import toUnicode, ss -from couchpotato.core.helpers.variable import mergeDicts, getExt, tryInt, splitString +from couchpotato.core.helpers.variable import mergeDicts, getExt, tryInt, splitString, tryFloat from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.quality.index import QualityIndex @@ -22,17 +23,17 @@ class QualityPlugin(Plugin): } qualities = [ - {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25', ('br', 'disk')], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']}, - {'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts'], 'tags': ['m2ts', 'x264', 'h264']}, - {'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']}, - {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip')], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, - {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']}, - {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, - {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]}, - {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p'], 'ext':[]}, - {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p'], 'ext':[]}, - {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p'], 'ext':[]}, - {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': ['720p'], 'ext':[]} + {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'median_size': 40000, 'label': 'BR-Disk', 'alternative': ['bd25', ('br', 'disk')], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']}, + {'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'median_size': 10000, 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts'], 'tags': ['m2ts', 'x264', 'h264']}, + {'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'median_size': 5500, 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']}, + {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'median_size': 2000, 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip')], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, + {'identifier': 'dvdr', 'size': (3000, 10000), 'median_size': 4500, 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']}, + {'identifier': 'dvdrip', 'size': (600, 2400), 'median_size': 1500, 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, + {'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]}, + {'identifier': 'r5', 'size': (600, 1000), 'median_size': 700, 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p'], 'ext':[]}, + {'identifier': 'tc', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p'], 'ext':[]}, + {'identifier': 'ts', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p'], 'ext':[]}, + {'identifier': 'cam', 'size': (600, 1000), 'median_size': 700, 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': ['720p'], 'ext':[]} ] pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr'] threed_tags = { @@ -330,7 +331,7 @@ class QualityPlugin(Plugin): # Check width resolution, range 20 if quality.get('width') and (quality.get('width') - 20) <= extra.get('resolution_width', 0) <= (quality.get('width') + 20): log.debug('Found %s via resolution_width: %s == %s', (quality['identifier'], quality.get('width'), extra.get('resolution_width', 0))) - score += 5 + score += 10 # Check height resolution, range 20 if quality.get('height') and (quality.get('height') - 20) <= extra.get('resolution_height', 0) <= (quality.get('height') + 20): @@ -350,9 +351,22 @@ class QualityPlugin(Plugin): if size: - if tryInt(quality['size_min']) <= tryInt(size) <= tryInt(quality['size_max']): - log.debug('Found %s via release size: %s MB < %s MB < %s MB', (quality['identifier'], quality['size_min'], size, quality['size_max'])) - score += 5 + size = tryFloat(size) + size_min = tryFloat(quality['size_min']) + size_max = tryFloat(quality['size_max']) + + if size_min <= size <= size_max: + log.debug('Found %s via release size: %s MB < %s MB < %s MB', (quality['identifier'], size_min, size, size_max)) + + proc_range = size_max - size_min + size_diff = size - size_min + size_proc = (size_diff / proc_range) + + median_diff = quality['median_size'] - size_min + media_proc = (median_diff / proc_range) + + max_points = 10 + score += ceil(max_points - (fabs(size_proc - media_proc) * max_points)) else: score -= 5 @@ -449,10 +463,12 @@ class QualityPlugin(Plugin): 'Movie Monuments 2013 BrRip 1080p': {'size': 1800, 'quality': 'brrip'}, 'Movie Monuments 2013 BrRip 720p': {'size': 1300, 'quality': 'brrip'}, 'The.Movie.2014.3D.1080p.BluRay.AVC.DTS-HD.MA.5.1-GroupName': {'size': 30000, 'quality': 'bd50', 'is_3d': True}, - '/home/namehou/Movie Monuments (2013)/Movie Monuments.mkv': {'size': 4500, 'quality': '1080p', 'is_3d': False}, - '/home/namehou/Movie Monuments (2013)/Movie Monuments Full-OU.mkv': {'size': 4500, 'quality': '1080p', 'is_3d': True}, + '/home/namehou/Movie Monuments (2012)/Movie Monuments.mkv': {'size': 5500, 'quality': '720p', 'is_3d': False}, + '/home/namehou/Movie Monuments (2012)/Movie Monuments Full-OU.mkv': {'size': 5500, 'quality': '720p', 'is_3d': True}, + '/home/namehou/Movie Monuments (2013)/Movie Monuments.mkv': {'size': 10000, 'quality': '1080p', 'is_3d': False}, + '/home/namehou/Movie Monuments (2013)/Movie Monuments Full-OU.mkv': {'size': 10000, 'quality': '1080p', 'is_3d': True}, '/volume1/Public/3D/Moviename/Moviename (2009).3D.SBS.ts': {'size': 7500, 'quality': '1080p', 'is_3d': True}, - '/volume1/Public/Moviename/Moviename (2009).ts': {'size': 5500, 'quality': '1080p'}, + '/volume1/Public/Moviename/Moviename (2009).ts': {'size': 7500, 'quality': '1080p'}, '/movies/BluRay HDDVD H.264 MKV 720p EngSub/QuiQui le fou (criterion collection #123, 1915)/QuiQui le fou (1915) 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p'}, 'C:\\movies\QuiQui le fou (collection #123, 1915)\QuiQui le fou (1915) 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p'}, 'C:\\movies\QuiQui le fou (collection #123, 1915)\QuiQui le fou (1915) half-sbs 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p', 'is_3d': True}, @@ -465,6 +481,10 @@ class QualityPlugin(Plugin): # 'Movie.Name.2014.1080p.HDrip.x264.aac-ReleaseGroup': {'size': 7500, 'quality': 'brrip'}, 'Movie.Name.2014.HDCam.Chinese.Subs-ReleaseGroup': {'size': 15000, 'quality': 'cam'}, 'Movie Name 2014 HQ DVDRip X264 AC3 (bla)': {'size': 0, 'quality': 'dvdrip'}, + 'Movie Name1 (2012).mkv': {'size': 4500, 'quality': '720p'}, + 'Movie Name (2013).mkv': {'size': 8500, 'quality': '1080p'}, + 'Movie Name (2014).mkv': {'size': 4500, 'quality': '720p', 'extra': {'titles': ['Movie Name 2014 720p Bluray']}}, + 'Movie Name (2015).mkv': {'size': 500, 'quality': '1080p', 'extra': {'resolution_width': 1920}}, } correct = 0 From 487ddf1c25d98e09dc9ce808a2015984f2cf0329 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 31 Aug 2014 22:19:09 +0200 Subject: [PATCH 084/202] Don't check message in dev mode --- couchpotato/core/notifications/core/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/core/main.py b/couchpotato/core/notifications/core/main.py index 5190218..fdc837a 100644 --- a/couchpotato/core/notifications/core/main.py +++ b/couchpotato/core/notifications/core/main.py @@ -66,7 +66,9 @@ class CoreNotifier(Notification): fireEvent('schedule.interval', 'core.clean_messages', self.cleanMessages, seconds = 15, single = True) addEvent('app.load', self.clean) - addEvent('app.load', self.checkMessages) + + if not Env.get('dev'): + addEvent('app.load', self.checkMessages) self.messages = [] self.listeners = [] From 089609d5d269fd941b91c7b8c9bdefc05e8ea13d Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 31 Aug 2014 22:19:41 +0200 Subject: [PATCH 085/202] Give higher penalty for allowed identifiers --- couchpotato/core/plugins/quality/main.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 10364a8..6035001 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -224,7 +224,7 @@ class QualityPlugin(Plugin): contains_score = self.containsTagScore(quality, words, cur_file) threedscore = self.contains3D(quality, threed_words, cur_file) if quality.get('allow_3d') else (0, None) - self.calcScore(score, quality, contains_score, threedscore) + self.calcScore(score, quality, contains_score, threedscore, penalty = contains_score) size_scores = [] for quality in qualities: @@ -236,11 +236,11 @@ class QualityPlugin(Plugin): if size_score > 0: size_scores.append(quality) - self.calcScore(score, quality, size_score + loose_score, penalty = False) + self.calcScore(score, quality, size_score + loose_score) # Add additional size score if only 1 size validated if len(size_scores) == 1: - self.calcScore(score, size_scores[0], 8, penalty = False) + self.calcScore(score, size_scores[0], 8) del size_scores # Return nothing if all scores are <= 0 @@ -271,11 +271,11 @@ class QualityPlugin(Plugin): words = words[:-1] points = { - 'identifier': 10, - 'label': 10, - 'alternative': 10, - 'tags': 9, - 'ext': 3, + 'identifier': 20, + 'label': 20, + 'alternative': 20, + 'tags': 11, + 'ext': 5, } # Check alt and tags @@ -363,16 +363,16 @@ class QualityPlugin(Plugin): size_proc = (size_diff / proc_range) median_diff = quality['median_size'] - size_min - media_proc = (median_diff / proc_range) + median_proc = (median_diff / proc_range) - max_points = 10 - score += ceil(max_points - (fabs(size_proc - media_proc) * max_points)) + max_points = 8 + score += ceil(max_points - (fabs(size_proc - median_proc) * max_points)) else: score -= 5 return score - def calcScore(self, score, quality, add_score, threedscore = (0, None), penalty = True): + def calcScore(self, score, quality, add_score, threedscore = (0, None), penalty = 0): score[quality['identifier']]['score'] += add_score @@ -391,7 +391,7 @@ class QualityPlugin(Plugin): if penalty and add_score != 0: for allow in quality.get('allow', []): - score[allow]['score'] -= 40 if self.cached_order[allow] < self.cached_order[quality['identifier']] else 5 + score[allow]['score'] -= ((penalty * 2) if self.cached_order[allow] < self.cached_order[quality['identifier']] else penalty) * 2 # Give panelty for all other qualities for q in self.qualities: @@ -485,6 +485,7 @@ class QualityPlugin(Plugin): 'Movie Name (2013).mkv': {'size': 8500, 'quality': '1080p'}, 'Movie Name (2014).mkv': {'size': 4500, 'quality': '720p', 'extra': {'titles': ['Movie Name 2014 720p Bluray']}}, 'Movie Name (2015).mkv': {'size': 500, 'quality': '1080p', 'extra': {'resolution_width': 1920}}, + 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, } correct = 0 From 9dbb477dd8cacf6144562a6be2e3985c293fdbbd Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 31 Aug 2014 23:11:28 +0200 Subject: [PATCH 086/202] Use latest found release if files are the same --- couchpotato/core/plugins/manage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/manage.py b/couchpotato/core/plugins/manage.py index d389e22..0f01774 100755 --- a/couchpotato/core/plugins/manage.py +++ b/couchpotato/core/plugins/manage.py @@ -165,7 +165,7 @@ class Manage(Plugin): already_used = used_files.get(release_file) if already_used: - release_id = release['_id'] if already_used.get('last_edit', 0) < release.get('last_edit', 0) else already_used['_id'] + release_id = release['_id'] if already_used.get('last_edit', 0) > release.get('last_edit', 0) else already_used['_id'] if release_id not in deleted_releases: fireEvent('release.delete', release_id, single = True) deleted_releases.append(release_id) From 356322c5b10d25d012f1a9647c59fb0e58bb99af Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Sep 2014 13:40:14 +0200 Subject: [PATCH 087/202] Use named argument for with_status calls --- couchpotato/core/media/_base/media/main.py | 1 + couchpotato/core/media/movie/searcher.py | 2 +- couchpotato/core/media/movie/suggestion/main.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 11747c8..e032dca 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -193,6 +193,7 @@ class MediaPlugin(MediaBase): except: pass + log.debug('No media found with identifiers: %s', identifiers) return False def list(self, types = None, status = None, release_status = None, status_or = False, limit_offset = None, with_tags = None, starts_with = None, search = None): diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index a8d6fe5..d7ca27c 100755 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -74,7 +74,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): self.in_progress = True fireEvent('notify.frontend', type = 'movie.searcher.started', data = True, message = 'Full search started') - medias = [x['_id'] for x in fireEvent('media.with_status', 'active', 'movie', single = True)] + medias = [x['_id'] for x in fireEvent('media.with_status', 'active', types = 'movie', with_doc = False, single = True)] random.shuffle(medias) total = len(medias) diff --git a/couchpotato/core/media/movie/suggestion/main.py b/couchpotato/core/media/movie/suggestion/main.py index 3df67ab..47171aa 100755 --- a/couchpotato/core/media/movie/suggestion/main.py +++ b/couchpotato/core/media/movie/suggestion/main.py @@ -27,7 +27,7 @@ class Suggestion(Plugin): else: if not movies or len(movies) == 0: - active_movies = fireEvent('media.with_status', ['active', 'done'], 'movie', single = True) + active_movies = fireEvent('media.with_status', ['active', 'done'], types = 'movie', single = True) movies = [getIdentifier(x) for x in active_movies] if not ignored or len(ignored) == 0: From c97bd38c83ea131389f0482d51a15049e916f593 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Sep 2014 14:23:26 +0200 Subject: [PATCH 088/202] Import cleanup --- couchpotato/core/media/_base/media/main.py | 8 ++++---- couchpotato/core/media/movie/_base/main.py | 1 - couchpotato/core/media/movie/charts/main.py | 1 - couchpotato/core/media/movie/searcher.py | 2 +- couchpotato/core/plugins/manage.py | 1 + 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index e032dca..4a97688 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -1,5 +1,4 @@ from datetime import timedelta -from operator import itemgetter import time import traceback from string import ascii_lowercase @@ -313,7 +312,8 @@ class MediaPlugin(MediaBase): def addSingleListView(self): for media_type in fireEvent('media.types', merge = True): - tempList = lambda media_type = media_type, *args, **kwargs : self.listView(type = media_type, **kwargs) + tempList = lambda *args, **kwargs : self.listView(type = media_type, **kwargs) + print tempList addApiView('%s.list' % media_type, tempList, docs = { 'desc': 'List media', 'params': { @@ -395,7 +395,7 @@ class MediaPlugin(MediaBase): def addSingleCharView(self): for media_type in fireEvent('media.types', merge = True): - tempChar = lambda media_type = media_type, *args, **kwargs : self.charView(type = media_type, **kwargs) + tempChar = lambda *args, **kwargs : self.charView(type = media_type, **kwargs) addApiView('%s.available_chars' % media_type, tempChar) def delete(self, media_id, delete_from = None): @@ -464,7 +464,7 @@ class MediaPlugin(MediaBase): def addSingleDeleteView(self): for media_type in fireEvent('media.types', merge = True): - tempDelete = lambda media_type = media_type, *args, **kwargs : self.deleteView(type = media_type, **kwargs) + tempDelete = lambda *args, **kwargs : self.deleteView(type = media_type, **kwargs) addApiView('%s.delete' % media_type, tempDelete, docs = { 'desc': 'Delete a ' + media_type + ' from the wanted list', 'params': { diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 8a04d0b..0dc39e5 100755 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -1,4 +1,3 @@ -import os import traceback import time diff --git a/couchpotato/core/media/movie/charts/main.py b/couchpotato/core/media/movie/charts/main.py index 28d5d68..9ab57dd 100644 --- a/couchpotato/core/media/movie/charts/main.py +++ b/couchpotato/core/media/movie/charts/main.py @@ -1,6 +1,5 @@ import time -from couchpotato import tryInt from couchpotato.core.logger import CPLog from couchpotato.api import addApiView from couchpotato.core.event import addEvent,fireEvent diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index d7ca27c..97181ae 100755 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -277,7 +277,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): # Contains lower quality string contains_other = fireEvent('searcher.contains_other_quality', nzb, movie_year = media['info']['year'], preferred_quality = preferred_quality, single = True) - if contains_other != False: + if contains_other and isinstance(contains_other, dict): log.info2('Wrong: %s, looking for %s, found %s', (nzb['name'], quality['label'], [x for x in contains_other] if contains_other else 'no quality')) return False diff --git a/couchpotato/core/plugins/manage.py b/couchpotato/core/plugins/manage.py index 0f01774..132f7ec 100755 --- a/couchpotato/core/plugins/manage.py +++ b/couchpotato/core/plugins/manage.py @@ -190,6 +190,7 @@ class Manage(Plugin): delete_me = {} + # noinspection PyTypeChecker for folder in self.in_progress: if self.in_progress[folder]['to_go'] <= 0: delete_me[folder] = True From 6e455e62d5eb5f57451111f56849afab12bdfd36 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Sep 2014 14:49:46 +0200 Subject: [PATCH 089/202] Remove debug print --- couchpotato/core/media/_base/media/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 4a97688..d5e54d7 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -313,7 +313,6 @@ class MediaPlugin(MediaBase): for media_type in fireEvent('media.types', merge = True): tempList = lambda *args, **kwargs : self.listView(type = media_type, **kwargs) - print tempList addApiView('%s.list' % media_type, tempList, docs = { 'desc': 'List media', 'params': { From bad26026ae7cd8bf9545fbaf78237b62777a0fee Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Sep 2014 14:50:42 +0200 Subject: [PATCH 090/202] Clean done after week, not 3 days --- couchpotato/core/plugins/release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index e4bfdc2..614b315 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -65,7 +65,7 @@ class Release(Plugin): log.debug('Removing releases from dashboard') now = time.time() - week = 262080 + week = 604800 db = get_db() From 3b34196901ebc32f5559edd67265b7d619619c7f Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Sep 2014 14:57:59 +0200 Subject: [PATCH 091/202] Also untag active movies --- couchpotato/core/plugins/release/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 614b315..31c3e61 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -95,7 +95,7 @@ class Release(Plugin): del media_exist # get movies last_edit more than a week ago - medias = fireEvent('media.with_status', 'done', single = True) + medias = fireEvent('media.with_status', ['done','active'], single = True) for media in medias: if media.get('last_edit', 0) > (now - week): @@ -111,7 +111,8 @@ class Release(Plugin): elif rel['status'] in ['snatched', 'downloaded']: self.updateStatus(rel['_id'], status = 'ignored') - fireEvent('media.untag', media.get('_id'), 'recent', single = True) + if 'recent' in media.get('tags', []): + fireEvent('media.untag', media.get('_id'), 'recent', single = True) def add(self, group, update_info = True, update_id = None): From 9d21dd9196eec5500c3b8749c1c4a29b3bc17068 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Sep 2014 22:13:49 +0200 Subject: [PATCH 092/202] Newznab download status code not caught properly --- couchpotato/core/media/_base/providers/nzb/newznab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/nzb/newznab.py b/couchpotato/core/media/_base/providers/nzb/newznab.py index 81622ad..87ecc75 100644 --- a/couchpotato/core/media/_base/providers/nzb/newznab.py +++ b/couchpotato/core/media/_base/providers/nzb/newznab.py @@ -187,7 +187,7 @@ class Base(NZBProvider, RSS): self.limits_reached[host] = False return data except HTTPError as e: - if e.code == 503: + if e.response.status_code == 503: response = e.read().lower() if 'maximum api' in response or 'download limit' in response: if not self.limits_reached.get(host): From 5a2df624621a78c31d12d49afdbdfc5783905700 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Sep 2014 22:51:55 +0200 Subject: [PATCH 093/202] Add quality test --- couchpotato/core/plugins/quality/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 6035001..baeca67 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -478,7 +478,7 @@ class QualityPlugin(Plugin): 'Moviename.2014.720p.R6.WEB-DL.x264.AC3-xyz': {'size': 750, 'quality': 'r5'}, 'Movie name 2014 New Source 720p HDCAM x264 AC3 xyz': {'size': 750, 'quality': 'cam'}, 'Movie.Name.2014.720p.HD.TS.AC3.x264': {'size': 750, 'quality': 'ts'}, - # 'Movie.Name.2014.1080p.HDrip.x264.aac-ReleaseGroup': {'size': 7500, 'quality': 'brrip'}, + 'Movie.Name.2014.1080p.HDrip.x264.aac-ReleaseGroup': {'size': 7000, 'quality': 'brrip'}, 'Movie.Name.2014.HDCam.Chinese.Subs-ReleaseGroup': {'size': 15000, 'quality': 'cam'}, 'Movie Name 2014 HQ DVDRip X264 AC3 (bla)': {'size': 0, 'quality': 'dvdrip'}, 'Movie Name1 (2012).mkv': {'size': 4500, 'quality': '720p'}, From 8d55b0c92a30c567a317ba56d5ad96984d8658be Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 2 Sep 2014 21:11:54 +0200 Subject: [PATCH 094/202] Assume non-3d if quality guess fails --- couchpotato/core/media/_base/searcher/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index 4e8dae2..5845c0d 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -129,7 +129,11 @@ class Searcher(SearcherBase): # Try guessing via quality tags guess = fireEvent('quality.guess', [nzb.get('name')], single = True) - return threed == guess.get('is_3d') + if guess: + return threed == guess.get('is_3d') + # If no quality guess, assume not 3d + else: + return threed == False def correctYear(self, haystack, year, year_range): From 50a150f57042b5b3ec31d13ca13c627db40ceeed Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 3 Sep 2014 16:45:44 +0200 Subject: [PATCH 095/202] Update library: httplib2 --- libs/httplib2/__init__.py | 33 +- libs/httplib2/cacerts.txt | 2362 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 1931 insertions(+), 464 deletions(-) diff --git a/libs/httplib2/__init__.py b/libs/httplib2/__init__.py index 9780d4e..d1212b5 100644 --- a/libs/httplib2/__init__.py +++ b/libs/httplib2/__init__.py @@ -22,7 +22,7 @@ __contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", "Sam Ruby", "Louis Nyffenegger"] __license__ = "MIT" -__version__ = "0.8" +__version__ = "0.9" import re import sys @@ -1082,7 +1082,9 @@ try: def _new_fixed_fetch(validate_certificate): def fixed_fetch(url, payload=None, method="GET", headers={}, allow_truncated=False, follow_redirects=True, - deadline=5): + deadline=None): + if deadline is None: + deadline = socket.getdefaulttimeout() or 5 return fetch(url, payload=payload, method=method, headers=headers, allow_truncated=allow_truncated, follow_redirects=follow_redirects, deadline=deadline, @@ -1246,7 +1248,10 @@ class Http(object): self.authorizations = [] def _conn_request(self, conn, request_uri, method, body, headers): - for i in range(RETRIES): + i = 0 + seen_bad_status_line = False + while i < RETRIES: + i += 1 try: if hasattr(conn, 'sock') and conn.sock is None: conn.connect() @@ -1284,6 +1289,19 @@ class Http(object): continue try: response = conn.getresponse() + except httplib.BadStatusLine: + # If we get a BadStatusLine on the first try then that means + # the connection just went stale, so retry regardless of the + # number of RETRIES set. + if not seen_bad_status_line and i == 1: + i = 0 + seen_bad_status_line = True + conn.close() + conn.connect() + continue + else: + conn.close() + raise except (socket.error, httplib.HTTPException): if i < RETRIES-1: conn.close() @@ -1364,7 +1382,10 @@ class Http(object): if response.status in [302, 303]: redirect_method = "GET" body = None - (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1) + (response, content) = self.request( + location, method=redirect_method, + body=body, headers=headers, + redirections=redirections - 1) response.previous = old_response else: raise RedirectLimit("Redirected more times than rediection_limit allows.", response, content) @@ -1506,7 +1527,9 @@ class Http(object): # Should cached permanent redirects be counted in our redirection count? For now, yes. if redirections <= 0: raise RedirectLimit("Redirected more times than rediection_limit allows.", {}, "") - (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1) + (response, new_content) = self.request( + info['-x-permanent-redirect-url'], method='GET', + headers=headers, redirections=redirections - 1) response.previous = Response(info) response.previous.fromcache = True else: diff --git a/libs/httplib2/cacerts.txt b/libs/httplib2/cacerts.txt index d8a0027..70990f1 100644 --- a/libs/httplib2/cacerts.txt +++ b/libs/httplib2/cacerts.txt @@ -1,135 +1,33 @@ -# Certifcate Authority certificates for validating SSL connections. -# -# This file contains PEM format certificates generated from -# http://mxr.mozilla.org/seamonkey/source/security/nss/lib/ckfw/builtins/certdata.txt -# -# ***** BEGIN LICENSE BLOCK ***** -# Version: MPL 1.1/GPL 2.0/LGPL 2.1 -# -# The contents of this file are subject to the Mozilla Public License Version -# 1.1 (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.mozilla.org/MPL/ -# -# Software distributed under the License is distributed on an "AS IS" basis, -# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -# for the specific language governing rights and limitations under the -# License. -# -# The Original Code is the Netscape security libraries. -# -# The Initial Developer of the Original Code is -# Netscape Communications Corporation. -# Portions created by the Initial Developer are Copyright (C) 1994-2000 -# the Initial Developer. All Rights Reserved. -# -# Contributor(s): -# -# Alternatively, the contents of this file may be used under the terms of -# either the GNU General Public License Version 2 or later (the "GPL"), or -# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), -# in which case the provisions of the GPL or the LGPL are applicable instead -# of those above. If you wish to allow use of your version of this file only -# under the terms of either the GPL or the LGPL, and not to allow others to -# use your version of this file under the terms of the MPL, indicate your -# decision by deleting the provisions above and replace them with the notice -# and other provisions required by the GPL or the LGPL. If you do not delete -# the provisions above, a recipient may use your version of this file under -# the terms of any one of the MPL, the GPL or the LGPL. -# -# ***** END LICENSE BLOCK ***** - -Verisign/RSA Secure Server CA -============================= - ------BEGIN CERTIFICATE----- -MIICNDCCAaECEAKtZn5ORf5eV288mBle3cAwDQYJKoZIhvcNAQECBQAwXzELMAkG -A1UEBhMCVVMxIDAeBgNVBAoTF1JTQSBEYXRhIFNlY3VyaXR5LCBJbmMuMS4wLAYD -VQQLEyVTZWN1cmUgU2VydmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk0 -MTEwOTAwMDAwMFoXDTEwMDEwNzIzNTk1OVowXzELMAkGA1UEBhMCVVMxIDAeBgNV -BAoTF1JTQSBEYXRhIFNlY3VyaXR5LCBJbmMuMS4wLAYDVQQLEyVTZWN1cmUgU2Vy -dmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGbMA0GCSqGSIb3DQEBAQUAA4GJ -ADCBhQJ+AJLOesGugz5aqomDV6wlAXYMra6OLDfO6zV4ZFQD5YRAUcm/jwjiioII -0haGN1XpsSECrXZogZoFokvJSyVmIlZsiAeP94FZbYQHZXATcXY+m3dM41CJVphI -uR2nKRoTLkoRWZweFdVJVCxzOmmCsZc5nG1wZ0jl3S3WyB57AgMBAAEwDQYJKoZI -hvcNAQECBQADfgBl3X7hsuyw4jrg7HFGmhkRuNPHoLQDQCYCPgmc4RKz0Vr2N6W3 -YQO2WxZpO8ZECAyIUwxrl0nHPjXcbLm7qt9cuzovk2C2qUtN8iD3zV9/ZHuO3ABc -1/p3yjkWWW8O6tO1g39NTUJWdrTJXwT4OPjr0l91X817/OWOgHz8UA== ------END CERTIFICATE----- - -Thawte Personal Basic CA -======================== - ------BEGIN CERTIFICATE----- -MIIDITCCAoqgAwIBAgIBADANBgkqhkiG9w0BAQQFADCByzELMAkGA1UEBhMCWkEx -FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD -VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT -ZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFBlcnNvbmFsIEJhc2lj -IENBMSgwJgYJKoZIhvcNAQkBFhlwZXJzb25hbC1iYXNpY0B0aGF3dGUuY29tMB4X -DTk2MDEwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgcsxCzAJBgNVBAYTAlpBMRUw -EwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEaMBgGA1UE -ChMRVGhhd3RlIENvbnN1bHRpbmcxKDAmBgNVBAsTH0NlcnRpZmljYXRpb24gU2Vy -dmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQZXJzb25hbCBCYXNpYyBD -QTEoMCYGCSqGSIb3DQEJARYZcGVyc29uYWwtYmFzaWNAdGhhd3RlLmNvbTCBnzAN -BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvLyTU23AUE+CFeZIlDWmWr5vQvoPR+53 -dXLdjUmbllegeNTKP1GzaQuRdhciB5dqxFGTS+CN7zeVoQxN2jSQHReJl+A1OFdK -wPQIcOk8RHtQfmGakOMj04gRRif1CwcOu93RfyAKiLlWCy4cgNrx454p7xS9CkT7 -G1sY0b8jkyECAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQQF -AAOBgQAt4plrsD16iddZopQBHyvdEktTwq1/qqcAXJFAVyVKOKqEcLnZgA+le1z7 -c8a914phXAPjLSeoF+CEhULcXpvGt7Jtu3Sv5D/Lp7ew4F2+eIMllNLbgQ95B21P -9DkVWlIBe94y1k049hJcBlDfBVu9FEuh3ym6O0GN92NWod8isQ== ------END CERTIFICATE----- - -Thawte Personal Premium CA -========================== - ------BEGIN CERTIFICATE----- -MIIDKTCCApKgAwIBAgIBADANBgkqhkiG9w0BAQQFADCBzzELMAkGA1UEBhMCWkEx -FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD -VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT -ZXJ2aWNlcyBEaXZpc2lvbjEjMCEGA1UEAxMaVGhhd3RlIFBlcnNvbmFsIFByZW1p -dW0gQ0ExKjAoBgkqhkiG9w0BCQEWG3BlcnNvbmFsLXByZW1pdW1AdGhhd3RlLmNv -bTAeFw05NjAxMDEwMDAwMDBaFw0yMDEyMzEyMzU5NTlaMIHPMQswCQYDVQQGEwJa -QTEVMBMGA1UECBMMV2VzdGVybiBDYXBlMRIwEAYDVQQHEwlDYXBlIFRvd24xGjAY -BgNVBAoTEVRoYXd0ZSBDb25zdWx0aW5nMSgwJgYDVQQLEx9DZXJ0aWZpY2F0aW9u -IFNlcnZpY2VzIERpdmlzaW9uMSMwIQYDVQQDExpUaGF3dGUgUGVyc29uYWwgUHJl -bWl1bSBDQTEqMCgGCSqGSIb3DQEJARYbcGVyc29uYWwtcHJlbWl1bUB0aGF3dGUu -Y29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDJZtn4B0TPuYwu8KHvE0Vs -Bd/eJxZRNkERbGw77f4QfRKe5ZtCmv5gMcNmt3M6SK5O0DI3lIi1DbbZ8/JE2dWI -Et12TfIa/G8jHnrx2JhFTgcQ7xZC0EN1bUre4qrJMf8fAHB8Zs8QJQi6+u4A6UYD -ZicRFTuqW/KY3TZCstqIdQIDAQABoxMwETAPBgNVHRMBAf8EBTADAQH/MA0GCSqG -SIb3DQEBBAUAA4GBAGk2ifc0KjNyL2071CKyuG+axTZmDhs8obF1Wub9NdP4qPIH -b4Vnjt4rueIXsDqg8A6iAJrf8xQVbrvIhVqYgPn/vnQdPfP+MCXRNzRn+qVxeTBh -KXLA4CxM+1bkOqhv5TJZUtt1KFBZDPgLGeSs2a+WjS9Q2wfD6h+rM+D1KzGJ ------END CERTIFICATE----- - -Thawte Personal Freemail CA -=========================== - ------BEGIN CERTIFICATE----- -MIIDLTCCApagAwIBAgIBADANBgkqhkiG9w0BAQQFADCB0TELMAkGA1UEBhMCWkEx -FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD -VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT -ZXJ2aWNlcyBEaXZpc2lvbjEkMCIGA1UEAxMbVGhhd3RlIFBlcnNvbmFsIEZyZWVt -YWlsIENBMSswKQYJKoZIhvcNAQkBFhxwZXJzb25hbC1mcmVlbWFpbEB0aGF3dGUu -Y29tMB4XDTk2MDEwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgdExCzAJBgNVBAYT -AlpBMRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEa -MBgGA1UEChMRVGhhd3RlIENvbnN1bHRpbmcxKDAmBgNVBAsTH0NlcnRpZmljYXRp -b24gU2VydmljZXMgRGl2aXNpb24xJDAiBgNVBAMTG1RoYXd0ZSBQZXJzb25hbCBG -cmVlbWFpbCBDQTErMCkGCSqGSIb3DQEJARYccGVyc29uYWwtZnJlZW1haWxAdGhh -d3RlLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA1GnX1LCUZFtx6UfY -DFG26nKRsIRefS0Nj3sS34UldSh0OkIsYyeflXtL734Zhx2G6qPduc6WZBrCFG5E -rHzmj+hND3EfQDimAKOHePb5lIZererAXnbr2RSjXW56fAylS1V/Bhkpf56aJtVq -uzgkCGqYx7Hao5iR/Xnb5VrEHLkCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zAN -BgkqhkiG9w0BAQQFAAOBgQDH7JJ+Tvj1lqVnYiqk8E0RYNBvjWBYYawmu1I1XAjP -MPuoSpaKH2JCI4wXD/S6ZJwXrEcp352YXtJsYHFcoqzceePnbgBHH7UNKOgCneSa -/RP0ptl8sfjcXyMmCZGAc9AUG95DqYMl8uacLxXK/qarigd1iwzdUYRr5PjRznei -gQ== ------END CERTIFICATE----- - -Thawte Server CA -================ +# Issuer: CN=GTE CyberTrust Global Root O=GTE Corporation OU=GTE CyberTrust Solutions, Inc. +# Subject: CN=GTE CyberTrust Global Root O=GTE Corporation OU=GTE CyberTrust Solutions, Inc. +# Label: "GTE CyberTrust Global Root" +# Serial: 421 +# MD5 Fingerprint: ca:3d:d3:68:f1:03:5c:d0:32:fa:b8:2b:59:e8:5a:db +# SHA1 Fingerprint: 97:81:79:50:d8:1c:96:70:cc:34:d8:09:cf:79:44:31:36:7e:f4:74 +# SHA256 Fingerprint: a5:31:25:18:8d:21:10:aa:96:4b:02:c7:b7:c6:da:32:03:17:08:94:e5:fb:71:ff:fb:66:67:d5:e6:81:0a:36 +-----BEGIN CERTIFICATE----- +MIICWjCCAcMCAgGlMA0GCSqGSIb3DQEBBAUAMHUxCzAJBgNVBAYTAlVTMRgwFgYD +VQQKEw9HVEUgQ29ycG9yYXRpb24xJzAlBgNVBAsTHkdURSBDeWJlclRydXN0IFNv +bHV0aW9ucywgSW5jLjEjMCEGA1UEAxMaR1RFIEN5YmVyVHJ1c3QgR2xvYmFsIFJv +b3QwHhcNOTgwODEzMDAyOTAwWhcNMTgwODEzMjM1OTAwWjB1MQswCQYDVQQGEwJV +UzEYMBYGA1UEChMPR1RFIENvcnBvcmF0aW9uMScwJQYDVQQLEx5HVEUgQ3liZXJU +cnVzdCBTb2x1dGlvbnMsIEluYy4xIzAhBgNVBAMTGkdURSBDeWJlclRydXN0IEds +b2JhbCBSb290MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCVD6C28FCc6HrH +iM3dFw4usJTQGz0O9pTAipTHBsiQl8i4ZBp6fmw8U+E3KHNgf7KXUwefU/ltWJTS +r41tiGeA5u2ylc9yMcqlHHK6XALnZELn+aks1joNrI1CqiQBOeacPwGFVw1Yh0X4 +04Wqk2kmhXBIgD8SFcd5tB8FLztimQIDAQABMA0GCSqGSIb3DQEBBAUAA4GBAG3r +GwnpXtlR22ciYaQqPEh346B8pt5zohQDhT37qw4wxYMWM4ETCJ57NE7fQMh017l9 +3PR2VX2bY1QY6fDq81yx2YtCHrnAlU66+tXifPVoYb+O7AWXX1uw16OFNMQkpw0P +lZPvy5TYnh+dXIVtx6quTx8itc2VrbqnzPmrC3p/ +-----END CERTIFICATE----- +# Issuer: CN=Thawte Server CA O=Thawte Consulting cc OU=Certification Services Division +# Subject: CN=Thawte Server CA O=Thawte Consulting cc OU=Certification Services Division +# Label: "Thawte Server CA" +# Serial: 1 +# MD5 Fingerprint: c5:70:c4:a2:ed:53:78:0c:c8:10:53:81:64:cb:d0:1d +# SHA1 Fingerprint: 23:e5:94:94:51:95:f2:41:48:03:b4:d5:64:d2:a3:a3:f5:d8:8b:8c +# SHA256 Fingerprint: b4:41:0b:73:e2:e6:ea:ca:47:fb:c4:2f:8f:a4:01:8a:f4:38:1d:c5:4c:fa:a8:44:50:46:1e:ed:09:45:4d:e9 -----BEGIN CERTIFICATE----- MIIDEzCCAnygAwIBAgIBATANBgkqhkiG9w0BAQQFADCBxDELMAkGA1UEBhMCWkEx FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD @@ -150,9 +48,13 @@ QNuozDJ0uW8NxuOzRAvZim+aKZuZGCg70eNAKJpaPNW15yAbi8qkq43pUdniTCxZ qdq5snUb9kLy78fyGPmJvKP/iiMucEc= -----END CERTIFICATE----- -Thawte Premium Server CA -======================== - +# Issuer: CN=Thawte Premium Server CA O=Thawte Consulting cc OU=Certification Services Division +# Subject: CN=Thawte Premium Server CA O=Thawte Consulting cc OU=Certification Services Division +# Label: "Thawte Premium Server CA" +# Serial: 1 +# MD5 Fingerprint: 06:9f:69:79:16:66:90:02:1b:8c:8c:a2:c3:07:6f:3a +# SHA1 Fingerprint: 62:7f:8d:78:27:65:63:99:d2:7d:7f:90:44:c9:fe:b3:f3:3e:fa:9a +# SHA256 Fingerprint: ab:70:36:36:5c:71:54:aa:29:c2:c2:9f:5d:41:91:16:3b:16:2a:22:25:01:13:57:d5:6d:07:ff:a7:bc:1f:72 -----BEGIN CERTIFICATE----- MIIDJzCCApCgAwIBAgIBATANBgkqhkiG9w0BAQQFADCBzjELMAkGA1UEBhMCWkEx FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD @@ -173,9 +75,13 @@ hfzJATj/Tb7yFkJD57taRvvBxhEf8UqwKEbJw8RCfbz6q1lu1bdRiBHjpIUZa4JM pAwSremkrj/xw0llmozFyD4lt5SZu5IycQfwhl7tUCemDaYj+bvLpgcUQg== -----END CERTIFICATE----- -Equifax Secure CA -================= - +# Issuer: O=Equifax OU=Equifax Secure Certificate Authority +# Subject: O=Equifax OU=Equifax Secure Certificate Authority +# Label: "Equifax Secure CA" +# Serial: 903804111 +# MD5 Fingerprint: 67:cb:9d:c0:13:24:8a:82:9b:b2:17:1e:d1:1b:ec:d4 +# SHA1 Fingerprint: d2:32:09:ad:23:d3:14:23:21:74:e4:0d:7f:9d:62:13:97:86:63:3a +# SHA256 Fingerprint: 08:29:7a:40:47:db:a2:36:80:c7:31:db:6e:31:76:53:ca:78:48:e1:be:bd:3a:0b:01:79:a7:07:f9:2c:f1:78 -----BEGIN CERTIFICATE----- MIIDIDCCAomgAwIBAgIENd70zzANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJV UzEQMA4GA1UEChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2Vy @@ -196,46 +102,13 @@ A4GBAFjOKer89961zgK5F7WF0bnj4JXMJTENAKaSbn+2kmOeUJXRmm/kEd5jhW6Y 1voqZiegDfqnc1zqcPGUIWVEX/r87yloqaKHee9570+sB3c4 -----END CERTIFICATE----- -Verisign Class 1 Public Primary Certification Authority -======================================================= - ------BEGIN CERTIFICATE----- -MIICPTCCAaYCEQDNun9W8N/kvFT+IqyzcqpVMA0GCSqGSIb3DQEBAgUAMF8xCzAJ -BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE3MDUGA1UECxMuQ2xh -c3MgMSBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw05 -NjAxMjkwMDAwMDBaFw0yODA4MDEyMzU5NTlaMF8xCzAJBgNVBAYTAlVTMRcwFQYD -VQQKEw5WZXJpU2lnbiwgSW5jLjE3MDUGA1UECxMuQ2xhc3MgMSBQdWJsaWMgUHJp -bWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCBnzANBgkqhkiG9w0BAQEFAAOB -jQAwgYkCgYEA5Rm/baNWYS2ZSHH2Z965jeu3noaACpEO+jglr0aIguVzqKCbJF0N -H8xlbgyw0FaEGIeaBpsQoXPftFg5a27B9hXVqKg/qhIGjTGsf7A01480Z4gJzRQR -4k5FVmkfeAKA2txHkSm7NsljXMXg1y2He6G3MrB7MLoqLzGq7qNn2tsCAwEAATAN -BgkqhkiG9w0BAQIFAAOBgQBMP7iLxmjf7kMzDl3ppssHhE16M/+SG/Q2rdiVIjZo -EWx8QszznC7EBz8UsA9P/5CSdvnivErpj82ggAr3xSnxgiJduLHdgSOjeyUVRjB5 -FvjqBUuUfx3CHMjjt/QQQDwTw18fU+hI5Ia0e6E1sHslurjTjqs/OJ0ANACY89Fx -lA== ------END CERTIFICATE----- - -Verisign Class 2 Public Primary Certification Authority -======================================================= - ------BEGIN CERTIFICATE----- -MIICPDCCAaUCEC0b/EoXjaOR6+f/9YtFvgswDQYJKoZIhvcNAQECBQAwXzELMAkG -A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz -cyAyIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2 -MDEyOTAwMDAwMFoXDTI4MDgwMTIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV -BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAyIFB1YmxpYyBQcmlt -YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN -ADCBiQKBgQC2WoujDWojg4BrzzmH9CETMwZMJaLtVRKXxaeAufqDwSCg+i8VDXyh -YGt+eSz6Bg86rvYbb7HS/y8oUl+DfUvEerf4Zh+AVPy3wo5ZShRXRtGak75BkQO7 -FYCTXOvnzAhsPz6zSvz/S2wj1VCCJkQZjiPDceoZJEcEnnW/yKYAHwIDAQABMA0G -CSqGSIb3DQEBAgUAA4GBAIobK/o5wXTXXtgZZKJYSi034DNHD6zt96rbHuSLBlxg -J8pFUs4W7z8GZOeUaHxgMxURaa+dYo2jA1Rrpr7l7gUYYAS/QoD90KioHgE796Nc -r6Pc5iaAIzy4RHT3Cq5Ji2F4zCS/iIqnDupzGUH9TQPwiNHleI2lKk/2lw0Xd8rY ------END CERTIFICATE----- - -Verisign Class 3 Public Primary Certification Authority -======================================================= - +# Issuer: O=VeriSign, Inc. OU=Class 3 Public Primary Certification Authority +# Subject: O=VeriSign, Inc. OU=Class 3 Public Primary Certification Authority +# Label: "Verisign Class 3 Public Primary Certification Authority" +# Serial: 149843929435818692848040365716851702463 +# MD5 Fingerprint: 10:fc:63:5d:f6:26:3e:0d:f3:25:be:5f:79:cd:67:67 +# SHA1 Fingerprint: 74:2c:31:92:e6:07:e4:24:eb:45:49:54:2b:e1:bb:c5:3e:61:74:e2 +# SHA256 Fingerprint: e7:68:56:34:ef:ac:f6:9a:ce:93:9a:6b:25:5b:7b:4f:ab:ef:42:93:5b:50:a2:65:ac:b5:cb:60:27:e4:4e:70 -----BEGIN CERTIFICATE----- MIICPDCCAaUCEHC65B0Q2Sk0tjjKewPMur8wDQYJKoZIhvcNAQECBQAwXzELMAkG A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz @@ -251,55 +124,13 @@ lbwdj2wsqFHMc9ikwFPwTtYmwHYBV4GSXiHx0bH/59AhWM1pF+NEHJwZRDmJXNyc AA9WjQKZ7aKQRUzkuxCkPfAyAw7xzvjoyVGM5mKf5p/AfbdynMk2OmufTqj/ZA1k -----END CERTIFICATE----- -Verisign Class 1 Public Primary Certification Authority - G2 -============================================================ - ------BEGIN CERTIFICATE----- -MIIDAjCCAmsCEEzH6qqYPnHTkxD4PTqJkZIwDQYJKoZIhvcNAQEFBQAwgcExCzAJ -BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh -c3MgMSBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy -MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp -emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X -DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw -FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgMSBQdWJsaWMg -UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo -YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5 -MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB -AQUAA4GNADCBiQKBgQCq0Lq+Fi24g9TK0g+8djHKlNgdk4xWArzZbxpvUjZudVYK -VdPfQ4chEWWKfo+9Id5rMj8bhDSVBZ1BNeuS65bdqlk/AVNtmU/t5eIqWpDBucSm -Fc/IReumXY6cPvBkJHalzasab7bYe1FhbqZ/h8jit+U03EGI6glAvnOSPWvndQID -AQABMA0GCSqGSIb3DQEBBQUAA4GBAKlPww3HZ74sy9mozS11534Vnjty637rXC0J -h9ZrbWB85a7FkCMMXErQr7Fd88e2CtvgFZMN3QO8x3aKtd1Pw5sTdbgBwObJW2ul -uIncrKTdcu1OofdPvAbT6shkdHvClUGcZXNY8ZCaPGqxmMnEh7zPRW1F4m4iP/68 -DzFc6PLZ ------END CERTIFICATE----- - -Verisign Class 2 Public Primary Certification Authority - G2 -============================================================ - ------BEGIN CERTIFICATE----- -MIIDAzCCAmwCEQC5L2DMiJ+hekYJuFtwbIqvMA0GCSqGSIb3DQEBBQUAMIHBMQsw -CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xPDA6BgNVBAsTM0Ns -YXNzIDIgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBH -MjE6MDgGA1UECxMxKGMpIDE5OTggVmVyaVNpZ24sIEluYy4gLSBGb3IgYXV0aG9y -aXplZCB1c2Ugb25seTEfMB0GA1UECxMWVmVyaVNpZ24gVHJ1c3QgTmV0d29yazAe -Fw05ODA1MTgwMDAwMDBaFw0yODA4MDEyMzU5NTlaMIHBMQswCQYDVQQGEwJVUzEX -MBUGA1UEChMOVmVyaVNpZ24sIEluYy4xPDA6BgNVBAsTM0NsYXNzIDIgUHVibGlj -IFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBHMjE6MDgGA1UECxMx -KGMpIDE5OTggVmVyaVNpZ24sIEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25s -eTEfMB0GA1UECxMWVmVyaVNpZ24gVHJ1c3QgTmV0d29yazCBnzANBgkqhkiG9w0B -AQEFAAOBjQAwgYkCgYEAp4gBIXQs5xoD8JjhlzwPIQjxnNuX6Zr8wgQGE75fUsjM -HiwSViy4AWkszJkfrbCWrnkE8hM5wXuYuggs6MKEEyyqaekJ9MepAqRCwiNPStjw -DqL7MWzJ5m+ZJwf15vRMeJ5t60aG+rmGyVTyssSv1EYcWskVMP8NbPUtDm3Of3cC -AwEAATANBgkqhkiG9w0BAQUFAAOBgQByLvl/0fFx+8Se9sVeUYpAmLho+Jscg9ji -nb3/7aHmZuovCfTK1+qlK5X2JGCGTUQug6XELaDTrnhpb3LabK4I8GOSN+a7xDAX -rXfMSTWqz9iP0b63GJZHc2pUIjRkLbYWm1lbtFFZOrMLFPQS32eg9K0yZF6xRnIn -jBJ7xUS0rg== ------END CERTIFICATE----- - -Verisign Class 3 Public Primary Certification Authority - G2 -============================================================ - +# Issuer: O=VeriSign, Inc. OU=Class 3 Public Primary Certification Authority - G2/(c) 1998 VeriSign, Inc. - For authorized use only/VeriSign Trust Network +# Subject: O=VeriSign, Inc. OU=Class 3 Public Primary Certification Authority - G2/(c) 1998 VeriSign, Inc. - For authorized use only/VeriSign Trust Network +# Label: "Verisign Class 3 Public Primary Certification Authority - G2" +# Serial: 167285380242319648451154478808036881606 +# MD5 Fingerprint: a2:33:9b:4c:74:78:73:d4:6c:e7:c1:f3:8d:cb:5c:e9 +# SHA1 Fingerprint: 85:37:1c:a6:e5:50:14:3d:ce:28:03:47:1b:de:3a:09:e8:f8:77:0f +# SHA256 Fingerprint: 83:ce:3c:12:29:68:8a:59:3d:48:5f:81:97:3c:0f:91:95:43:1e:da:37:cc:5e:36:43:0e:79:c7:a8:88:63:8b -----BEGIN CERTIFICATE----- MIIDAjCCAmsCEH3Z/gfPqB63EHln+6eJNMYwDQYJKoZIhvcNAQEFBQAwgcExCzAJ BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh @@ -320,88 +151,150 @@ F6YM40AIOw7n60RzKprxaZLvcRTDOaxxp5EJb+RxBrO6WVcmeQD2+A2iMzAo1KpY oJ2daZH9 -----END CERTIFICATE----- -Verisign Class 4 Public Primary Certification Authority - G2 -============================================================ +# Issuer: CN=GlobalSign Root CA O=GlobalSign nv-sa OU=Root CA +# Subject: CN=GlobalSign Root CA O=GlobalSign nv-sa OU=Root CA +# Label: "GlobalSign Root CA" +# Serial: 4835703278459707669005204 +# MD5 Fingerprint: 3e:45:52:15:09:51:92:e1:b7:5d:37:9f:b1:87:29:8a +# SHA1 Fingerprint: b1:bc:96:8b:d4:f4:9d:62:2a:a8:9a:81:f2:15:01:52:a4:1d:82:9c +# SHA256 Fingerprint: eb:d4:10:40:e4:bb:3e:c7:42:c9:e3:81:d3:1e:f2:a4:1a:48:b6:68:5c:96:e7:ce:f3:c1:df:6c:d4:33:1c:99 +-----BEGIN CERTIFICATE----- +MIIDdTCCAl2gAwIBAgILBAAAAAABFUtaw5QwDQYJKoZIhvcNAQEFBQAwVzELMAkG +A1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jv +b3QgQ0ExGzAZBgNVBAMTEkdsb2JhbFNpZ24gUm9vdCBDQTAeFw05ODA5MDExMjAw +MDBaFw0yODAxMjgxMjAwMDBaMFcxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9i +YWxTaWduIG52LXNhMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJHbG9iYWxT +aWduIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDuaZ +jc6j40+Kfvvxi4Mla+pIH/EqsLmVEQS98GPR4mdmzxzdzxtIK+6NiY6arymAZavp +xy0Sy6scTHAHoT0KMM0VjU/43dSMUBUc71DuxC73/OlS8pF94G3VNTCOXkNz8kHp +1Wrjsok6Vjk4bwY8iGlbKk3Fp1S4bInMm/k8yuX9ifUSPJJ4ltbcdG6TRGHRjcdG +snUOhugZitVtbNV4FpWi6cgKOOvyJBNPc1STE4U6G7weNLWLBYy5d4ux2x8gkasJ +U26Qzns3dLlwR5EiUWMWea6xrkEmCMgZK9FGqkjWZCrXgzT/LCrBbBlDSgeF59N8 +9iFo7+ryUp9/k5DPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E +BTADAQH/MB0GA1UdDgQWBBRge2YaRQ2XyolQL30EzTSo//z9SzANBgkqhkiG9w0B +AQUFAAOCAQEA1nPnfE920I2/7LqivjTFKDK1fPxsnCwrvQmeU79rXqoRSLblCKOz +yj1hTdNGCbM+w6DjY1Ub8rrvrTnhQ7k4o+YviiY776BQVvnGCv04zcQLcFGUl5gE +38NflNUVyRRBnMRddWQVDf9VMOyGj/8N7yy5Y0b2qvzfvGn9LhJIZJrglfCm7ymP +AbEVtQwdpf5pLGkkeB6zpxxxYu7KyJesF12KwvhHhm4qxFYxldBniYUr+WymXUad +DKqC5JlR3XC321Y9YeRq4VzW9v493kHMB65jUr9TU/Qr6cf9tveCX4XSQRjbgbME +HMUfpIBvFSDJ3gyICh3WZlXi/EjJKSZp4A== +-----END CERTIFICATE----- +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R2 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R2 +# Label: "GlobalSign Root CA - R2" +# Serial: 4835703278459682885658125 +# MD5 Fingerprint: 94:14:77:7e:3e:5e:fd:8f:30:bd:41:b0:cf:e7:d0:30 +# SHA1 Fingerprint: 75:e0:ab:b6:13:85:12:27:1c:04:f8:5f:dd:de:38:e4:b7:24:2e:fe +# SHA256 Fingerprint: ca:42:dd:41:74:5f:d0:b8:1e:b9:02:36:2c:f9:d8:bf:71:9d:a1:bd:1b:1e:fc:94:6f:5b:4c:99:f4:2c:1b:9e -----BEGIN CERTIFICATE----- -MIIDAjCCAmsCEDKIjprS9esTR/h/xCA3JfgwDQYJKoZIhvcNAQEFBQAwgcExCzAJ -BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh -c3MgNCBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy -MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp -emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X -DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw -FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgNCBQdWJsaWMg -UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo -YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5 -MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB -AQUAA4GNADCBiQKBgQC68OTP+cSuhVS5B1f5j8V/aBH4xBewRNzjMHPVKmIquNDM -HO0oW369atyzkSTKQWI8/AIBvxwWMZQFl3Zuoq29YRdsTjCG8FE3KlDHqGKB3FtK -qsGgtG7rL+VXxbErQHDbWk2hjh+9Ax/YA9SPTJlxvOKCzFjomDqG04Y48wApHwID -AQABMA0GCSqGSIb3DQEBBQUAA4GBAIWMEsGnuVAVess+rLhDityq3RS6iYF+ATwj -cSGIL4LcY/oCRaxFWdcqWERbt5+BO5JoPeI3JPV7bI92NZYJqFmduc4jq3TWg/0y -cyfYaT5DdPauxYma51N86Xv2S/PBZYPejYqcPIiNOVn8qj8ijaHBZlCBckztImRP -T8qAkbYp +MIIDujCCAqKgAwIBAgILBAAAAAABD4Ym5g0wDQYJKoZIhvcNAQEFBQAwTDEgMB4G +A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjIxEzARBgNVBAoTCkdsb2JhbFNp +Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDYxMjE1MDgwMDAwWhcNMjExMjE1 +MDgwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEG +A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAKbPJA6+Lm8omUVCxKs+IVSbC9N/hHD6ErPL +v4dfxn+G07IwXNb9rfF73OX4YJYJkhD10FPe+3t+c4isUoh7SqbKSaZeqKeMWhG8 +eoLrvozps6yWJQeXSpkqBy+0Hne/ig+1AnwblrjFuTosvNYSuetZfeLQBoZfXklq +tTleiDTsvHgMCJiEbKjNS7SgfQx5TfC4LcshytVsW33hoCmEofnTlEnLJGKRILzd +C9XZzPnqJworc5HGnRusyMvo4KD0L5CLTfuwNhv2GXqF4G3yYROIXJ/gkwpRl4pa +zq+r1feqCapgvdzZX99yqWATXgAByUr6P6TqBwMhAo6CygPCm48CAwEAAaOBnDCB +mTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUm+IH +V2ccHsBqBt5ZtJot39wZhi4wNgYDVR0fBC8wLTAroCmgJ4YlaHR0cDovL2NybC5n +bG9iYWxzaWduLm5ldC9yb290LXIyLmNybDAfBgNVHSMEGDAWgBSb4gdXZxwewGoG +3lm0mi3f3BmGLjANBgkqhkiG9w0BAQUFAAOCAQEAmYFThxxol4aR7OBKuEQLq4Gs +J0/WwbgcQ3izDJr86iw8bmEbTUsp9Z8FHSbBuOmDAGJFtqkIk7mpM0sYmsL4h4hO +291xNBrBVNpGP+DTKqttVCL1OmLNIG+6KYnX3ZHu01yiPqFbQfXf5WRDLenVOavS +ot+3i9DAgBkcRcAtjOj4LaR0VknFBbVPFd5uRHg5h6h+u/N5GJG79G+dwfCMNYxd +AfvDbbnvRG15RjF+Cv6pgsH/76tuIMRQyV+dTZsXjAzlAcmgQWpzU/qlULRuJQ/7 +TBj0/VLZjmmx6BEP3ojY+x1J96relc8geMJgEtslQIxq/H5COEBkEveegeGTLg== +-----END CERTIFICATE----- + +# Issuer: CN=http://www.valicert.com/ O=ValiCert, Inc. OU=ValiCert Class 1 Policy Validation Authority +# Subject: CN=http://www.valicert.com/ O=ValiCert, Inc. OU=ValiCert Class 1 Policy Validation Authority +# Label: "ValiCert Class 1 VA" +# Serial: 1 +# MD5 Fingerprint: 65:58:ab:15:ad:57:6c:1e:a8:a7:b5:69:ac:bf:ff:eb +# SHA1 Fingerprint: e5:df:74:3c:b6:01:c4:9b:98:43:dc:ab:8c:e8:6a:81:10:9f:e4:8e +# SHA256 Fingerprint: f4:c1:49:55:1a:30:13:a3:5b:c7:bf:fe:17:a7:f3:44:9b:c1:ab:5b:5a:0a:e7:4b:06:c2:3b:90:00:4c:01:04 +-----BEGIN CERTIFICATE----- +MIIC5zCCAlACAQEwDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0 +IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAz +BgNVBAsTLFZhbGlDZXJ0IENsYXNzIDEgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9y +aXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG +9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTk5MDYyNTIyMjM0OFoXDTE5MDYy +NTIyMjM0OFowgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0IFZhbGlkYXRpb24gTmV0d29y +azEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENs +YXNzIDEgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9yaXR5MSEwHwYDVQQDExhodHRw +Oi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG9w0BCQEWEWluZm9AdmFsaWNl +cnQuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDYWYJ6ibiWuqYvaG9Y +LqdUHAZu9OqNSLwxlBfw8068srg1knaw0KWlAdcAAxIiGQj4/xEjm84H9b9pGib+ +TunRf50sQB1ZaG6m+FiwnRqP0z/x3BkGgagO4DrdyFNFCQbmD3DD+kCmDuJWBQ8Y +TfwggtFzVXSNdnKgHZ0dwN0/cQIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAFBoPUn0 +LBwGlN+VYH+Wexf+T3GtZMjdd9LvWVXoP+iOBSoh8gfStadS/pyxtuJbdxdA6nLW +I8sogTLDAHkY7FkXicnGah5xyf23dKUlRWnFSKsZ4UWKJWsZ7uW7EvV/96aNUcPw +nXS3qT6gpf+2SQMT2iLM7XGCK5nPOrf1LXLI -----END CERTIFICATE----- -Verisign Class 1 Public Primary Certification Authority - G3 -============================================================ +# Issuer: CN=http://www.valicert.com/ O=ValiCert, Inc. OU=ValiCert Class 2 Policy Validation Authority +# Subject: CN=http://www.valicert.com/ O=ValiCert, Inc. OU=ValiCert Class 2 Policy Validation Authority +# Label: "ValiCert Class 2 VA" +# Serial: 1 +# MD5 Fingerprint: a9:23:75:9b:ba:49:36:6e:31:c2:db:f2:e7:66:ba:87 +# SHA1 Fingerprint: 31:7a:2a:d0:7f:2b:33:5e:f5:a1:c3:4e:4b:57:e8:b7:d8:f1:fc:a6 +# SHA256 Fingerprint: 58:d0:17:27:9c:d4:dc:63:ab:dd:b1:96:a6:c9:90:6c:30:c4:e0:87:83:ea:e8:c1:60:99:54:d6:93:55:59:6b +-----BEGIN CERTIFICATE----- +MIIC5zCCAlACAQEwDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0 +IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAz +BgNVBAsTLFZhbGlDZXJ0IENsYXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9y +aXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG +9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTk5MDYyNjAwMTk1NFoXDTE5MDYy +NjAwMTk1NFowgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0IFZhbGlkYXRpb24gTmV0d29y +azEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENs +YXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9yaXR5MSEwHwYDVQQDExhodHRw +Oi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG9w0BCQEWEWluZm9AdmFsaWNl +cnQuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDOOnHK5avIWZJV16vY +dA757tn2VUdZZUcOBVXc65g2PFxTXdMwzzjsvUGJ7SVCCSRrCl6zfN1SLUzm1NZ9 +WlmpZdRJEy0kTRxQb7XBhVQ7/nHk01xC+YDgkRoKWzk2Z/M/VXwbP7RfZHM047QS +v4dk+NoS/zcnwbNDu+97bi5p9wIDAQABMA0GCSqGSIb3DQEBBQUAA4GBADt/UG9v +UJSZSWI4OB9L+KXIPqeCgfYrx+jFzug6EILLGACOTb2oWH+heQC1u+mNr0HZDzTu +IYEZoDJJKPTEjlbVUjP9UNV+mWwD5MlM/Mtsq2azSiGM5bUMMj4QssxsodyamEwC +W/POuZ6lcg5Ktz885hZo+L7tdEy8W9ViH0Pd +-----END CERTIFICATE----- +# Issuer: CN=http://www.valicert.com/ O=ValiCert, Inc. OU=ValiCert Class 3 Policy Validation Authority +# Subject: CN=http://www.valicert.com/ O=ValiCert, Inc. OU=ValiCert Class 3 Policy Validation Authority +# Label: "RSA Root Certificate 1" +# Serial: 1 +# MD5 Fingerprint: a2:6f:53:b7:ee:40:db:4a:68:e7:fa:18:d9:10:4b:72 +# SHA1 Fingerprint: 69:bd:8c:f4:9c:d3:00:fb:59:2e:17:93:ca:55:6a:f3:ec:aa:35:fb +# SHA256 Fingerprint: bc:23:f9:8a:31:3c:b9:2d:e3:bb:fc:3a:5a:9f:44:61:ac:39:49:4c:4a:e1:5a:9e:9d:f1:31:e9:9b:73:01:9a -----BEGIN CERTIFICATE----- -MIIEGjCCAwICEQCLW3VWhFSFCwDPrzhIzrGkMA0GCSqGSIb3DQEBBQUAMIHKMQsw -CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl -cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu -LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT -aWduIENsYXNzIDEgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp -dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD -VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT -aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ -bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu -IENsYXNzIDEgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg -LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN2E1Lm0+afY8wR4 -nN493GwTFtl63SRRZsDHJlkNrAYIwpTRMx/wgzUfbhvI3qpuFU5UJ+/EbRrsC+MO -8ESlV8dAWB6jRx9x7GD2bZTIGDnt/kIYVt/kTEkQeE4BdjVjEjbdZrwBBDajVWjV -ojYJrKshJlQGrT/KFOCsyq0GHZXi+J3x4GD/wn91K0zM2v6HmSHquv4+VNfSWXjb -PG7PoBMAGrgnoeS+Z5bKoMWznN3JdZ7rMJpfo83ZrngZPyPpXNspva1VyBtUjGP2 -6KbqxzcSXKMpHgLZ2x87tNcPVkeBFQRKr4Mn0cVYiMHd9qqnoxjaaKptEVHhv2Vr -n5Z20T0CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAq2aN17O6x5q25lXQBfGfMY1a -qtmqRiYPce2lrVNWYgFHKkTp/j90CxObufRNG7LRX7K20ohcs5/Ny9Sn2WCVhDr4 -wTcdYcrnsMXlkdpUpqwxga6X3s0IrLjAl4B/bnKk52kTlWUfxJM8/XmPBNQ+T+r3 -ns7NZ3xPZQL/kYVUc8f/NveGLezQXk//EZ9yBta4GvFMDSZl4kSAHsef493oCtrs -pSCAaWihT37ha88HQfqDjrw43bAuEbFrskLMmrz5SCJ5ShkPshw+IHTZasO+8ih4 -E1Z5T21Q6huwtVexN2ZYI/PcD98Kh8TvhgXVOBRgmaNL3gaWcSzy27YfpO8/7g== ------END CERTIFICATE----- - -Verisign Class 2 Public Primary Certification Authority - G3 -============================================================ - ------BEGIN CERTIFICATE----- -MIIEGTCCAwECEGFwy0mMX5hFKeewptlQW3owDQYJKoZIhvcNAQEFBQAwgcoxCzAJ -BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjEfMB0GA1UECxMWVmVy -aVNpZ24gVHJ1c3QgTmV0d29yazE6MDgGA1UECxMxKGMpIDE5OTkgVmVyaVNpZ24s -IEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTFFMEMGA1UEAxM8VmVyaVNp -Z24gQ2xhc3MgMiBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0 -eSAtIEczMB4XDTk5MTAwMTAwMDAwMFoXDTM2MDcxNjIzNTk1OVowgcoxCzAJBgNV -BAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjEfMB0GA1UECxMWVmVyaVNp -Z24gVHJ1c3QgTmV0d29yazE6MDgGA1UECxMxKGMpIDE5OTkgVmVyaVNpZ24sIElu -Yy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTFFMEMGA1UEAxM8VmVyaVNpZ24g -Q2xhc3MgMiBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAt -IEczMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArwoNwtUs22e5LeWU -J92lvuCwTY+zYVY81nzD9M0+hsuiiOLh2KRpxbXiv8GmR1BeRjmL1Za6tW8UvxDO -JxOeBUebMXoT2B/Z0wI3i60sR/COgQanDTAM6/c8DyAd3HJG7qUCyFvDyVZpTMUY -wZF7C9UTAJu878NIPkZgIIUq1ZC2zYugzDLdt/1AVbJQHFauzI13TccgTacxdu9o -koqQHgiBVrKtaaNS0MscxCM9H5n+TOgWY47GCI72MfbS+uV23bUckqNJzc0BzWjN -qWm6o+sdDZykIKbBoMXRRkwXbdKsZj+WjOCE1Db/IlnF+RFgqF8EffIa9iVCYQ/E -Srg+iQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQA0JhU8wI1NQ0kdvekhktdmnLfe -xbjQ5F1fdiLAJvmEOjr5jLX77GDx6M4EsMjdpwOPMPOY36TmpDHf0xwLRtxyID+u -7gU8pDM/CzmscHhzS5kr3zDCVLCoO1Wh/hYozUK9dG6A2ydEp85EXdQbkJgNHkKU -sQAsBNB0owIFImNjzYO1+8FtYmtpdf1dcEG59b98377BMnMiIYtYgXsVkXq642RI -sH/7NiXaldDxJBQX3RiAa0YjOVT1jmIJBB2UkKab5iXiQkWquJCtvgiPqQtCGJTP -cjnhsUPgKM+351psE2tJs//jGHyJizNdrDPXp/naOlXJWBD5qu9ats9LS98q ------END CERTIFICATE----- - -Verisign Class 3 Public Primary Certification Authority - G3 -============================================================ +MIIC5zCCAlACAQEwDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0 +IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAz +BgNVBAsTLFZhbGlDZXJ0IENsYXNzIDMgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9y +aXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG +9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTk5MDYyNjAwMjIzM1oXDTE5MDYy +NjAwMjIzM1owgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0IFZhbGlkYXRpb24gTmV0d29y +azEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENs +YXNzIDMgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9yaXR5MSEwHwYDVQQDExhodHRw +Oi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG9w0BCQEWEWluZm9AdmFsaWNl +cnQuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDjmFGWHOjVsQaBalfD +cnWTq8+epvzzFlLWLU2fNUSoLgRNB0mKOCn1dzfnt6td3zZxFJmP3MKS8edgkpfs +2Ejcv8ECIMYkpChMMFp2bbFc893enhBxoYjHW5tBbcqwuI4V7q0zK89HBFx1cQqY +JJgpp0lZpd34t0NiYfPT4tBVPwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAFa7AliE +Zwgs3x/be0kz9dNnnfS0ChCzycUs4pJqcXgn8nCDQtM+z6lU9PHYkhaM0QTLS6vJ +n0WuPIqpsHEzXcjFV9+vqDWzf4mH6eglkrh/hXqu1rweN1gqZ8mRzyqBPu3GOd/A +PhmcGcwTTYJBtYze4D1gCCAPRX5ron+jjBXu +-----END CERTIFICATE----- +# Issuer: CN=VeriSign Class 3 Public Primary Certification Authority - G3 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 1999 VeriSign, Inc. - For authorized use only +# Subject: CN=VeriSign Class 3 Public Primary Certification Authority - G3 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 1999 VeriSign, Inc. - For authorized use only +# Label: "Verisign Class 3 Public Primary Certification Authority - G3" +# Serial: 206684696279472310254277870180966723415 +# MD5 Fingerprint: cd:68:b6:a7:c7:c4:ce:75:e0:1d:4f:57:44:61:92:09 +# SHA1 Fingerprint: 13:2d:0d:45:53:4b:69:97:cd:b2:d5:c3:39:e2:55:76:60:9b:5c:c6 +# SHA256 Fingerprint: eb:04:cf:5e:b1:f3:9a:fa:76:2f:2b:b1:20:f2:96:cb:a5:20:c1:b9:7d:b1:58:95:65:b8:1c:b9:a1:7b:72:44 -----BEGIN CERTIFICATE----- MIIEGjCCAwICEQCbfgZJoz5iudXukEhxKe9XMA0GCSqGSIb3DQEBBQUAMIHKMQsw CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl @@ -427,9 +320,13 @@ F4ErWjfJXir0xuKhXFSbplQAz/DxwceYMBo7Nhbbo27q/a2ywtrvAkcTisDxszGt TxzhT5yvDwyd93gN2PQ1VoDat20Xj50egWTh/sVFuq1ruQp6Tk9LhO5L8X3dEQ== -----END CERTIFICATE----- -Verisign Class 4 Public Primary Certification Authority - G3 -============================================================ - +# Issuer: CN=VeriSign Class 4 Public Primary Certification Authority - G3 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 1999 VeriSign, Inc. - For authorized use only +# Subject: CN=VeriSign Class 4 Public Primary Certification Authority - G3 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 1999 VeriSign, Inc. - For authorized use only +# Label: "Verisign Class 4 Public Primary Certification Authority - G3" +# Serial: 314531972711909413743075096039378935511 +# MD5 Fingerprint: db:c8:f2:27:2e:b1:ea:6a:29:23:5d:fe:56:3e:33:df +# SHA1 Fingerprint: c8:ec:8c:87:92:69:cb:4b:ab:39:e9:8d:7e:57:67:f3:14:95:73:9d +# SHA256 Fingerprint: e3:89:36:0d:0f:db:ae:b3:d2:50:58:4b:47:30:31:4e:22:2f:39:c1:56:a0:20:14:4e:8d:96:05:61:79:15:06 -----BEGIN CERTIFICATE----- MIIEGjCCAwICEQDsoKeLbnVqAc/EfMwvlF7XMA0GCSqGSIb3DQEBBQUAMIHKMQsw CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl @@ -455,9 +352,112 @@ fjaF3H48ZwC15DtS4KjrXRX5xm3wrR0OhbepmnMUWluPQSjA1egtTaRezarZ7c7c bLvSHgCwIe34QWKCudiyxLtGUPMxxY8BqHTr9Xgn2uf3ZkPznoM+IKrDNWCRzg== -----END CERTIFICATE----- -Equifax Secure Global eBusiness CA -================================== +# Issuer: CN=Entrust.net Secure Server Certification Authority O=Entrust.net OU=www.entrust.net/CPS incorp. by ref. (limits liab.)/(c) 1999 Entrust.net Limited +# Subject: CN=Entrust.net Secure Server Certification Authority O=Entrust.net OU=www.entrust.net/CPS incorp. by ref. (limits liab.)/(c) 1999 Entrust.net Limited +# Label: "Entrust.net Secure Server CA" +# Serial: 927650371 +# MD5 Fingerprint: df:f2:80:73:cc:f1:e6:61:73:fc:f5:42:e9:c5:7c:ee +# SHA1 Fingerprint: 99:a6:9b:e6:1a:fe:88:6b:4d:2b:82:00:7c:b8:54:fc:31:7e:15:39 +# SHA256 Fingerprint: 62:f2:40:27:8c:56:4c:4d:d8:bf:7d:9d:4f:6f:36:6e:a8:94:d2:2f:5f:34:d9:89:a9:83:ac:ec:2f:ff:ed:50 +-----BEGIN CERTIFICATE----- +MIIE2DCCBEGgAwIBAgIEN0rSQzANBgkqhkiG9w0BAQUFADCBwzELMAkGA1UEBhMC +VVMxFDASBgNVBAoTC0VudHJ1c3QubmV0MTswOQYDVQQLEzJ3d3cuZW50cnVzdC5u +ZXQvQ1BTIGluY29ycC4gYnkgcmVmLiAobGltaXRzIGxpYWIuKTElMCMGA1UECxMc +KGMpIDE5OTkgRW50cnVzdC5uZXQgTGltaXRlZDE6MDgGA1UEAxMxRW50cnVzdC5u +ZXQgU2VjdXJlIFNlcnZlciBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw05OTA1 +MjUxNjA5NDBaFw0xOTA1MjUxNjM5NDBaMIHDMQswCQYDVQQGEwJVUzEUMBIGA1UE +ChMLRW50cnVzdC5uZXQxOzA5BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5j +b3JwLiBieSByZWYuIChsaW1pdHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBF +bnRydXN0Lm5ldCBMaW1pdGVkMTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUg +U2VydmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGdMA0GCSqGSIb3DQEBAQUA +A4GLADCBhwKBgQDNKIM0VBuJ8w+vN5Ex/68xYMmo6LIQaO2f55M28Qpku0f1BBc/ +I0dNxScZgSYMVHINiC3ZH5oSn7yzcdOAGT9HZnuMNSjSuQrfJNqc1lB5gXpa0zf3 +wkrYKZImZNHkmGw6AIr1NJtl+O3jEP/9uElY3KDegjlrgbEWGWG5VLbmQwIBA6OC +AdcwggHTMBEGCWCGSAGG+EIBAQQEAwIABzCCARkGA1UdHwSCARAwggEMMIHeoIHb +oIHYpIHVMIHSMQswCQYDVQQGEwJVUzEUMBIGA1UEChMLRW50cnVzdC5uZXQxOzA5 +BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5jb3JwLiBieSByZWYuIChsaW1p +dHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBFbnRydXN0Lm5ldCBMaW1pdGVk +MTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUgU2VydmVyIENlcnRpZmljYXRp +b24gQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMCmgJ6AlhiNodHRwOi8vd3d3LmVu +dHJ1c3QubmV0L0NSTC9uZXQxLmNybDArBgNVHRAEJDAigA8xOTk5MDUyNTE2MDk0 +MFqBDzIwMTkwNTI1MTYwOTQwWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAU8Bdi +E1U9s/8KAGv7UISX8+1i0BowHQYDVR0OBBYEFPAXYhNVPbP/CgBr+1CEl/PtYtAa +MAwGA1UdEwQFMAMBAf8wGQYJKoZIhvZ9B0EABAwwChsEVjQuMAMCBJAwDQYJKoZI +hvcNAQEFBQADgYEAkNwwAvpkdMKnCqV8IY00F6j7Rw7/JXyNEwr75Ji174z4xRAN +95K+8cPV1ZVqBLssziY2ZcgxxufuP+NXdYR6Ee9GTxj005i7qIcyunL2POI9n9cd +2cNgQ4xYDiKWL2KjLB+6rQXvqzJ4h6BUcxm1XAX5Uj5tLUUL9wqT6u0G+bI= +-----END CERTIFICATE----- +# Issuer: CN=Entrust.net Certification Authority (2048) O=Entrust.net OU=www.entrust.net/CPS_2048 incorp. by ref. (limits liab.)/(c) 1999 Entrust.net Limited +# Subject: CN=Entrust.net Certification Authority (2048) O=Entrust.net OU=www.entrust.net/CPS_2048 incorp. by ref. (limits liab.)/(c) 1999 Entrust.net Limited +# Label: "Entrust.net Premium 2048 Secure Server CA" +# Serial: 946059622 +# MD5 Fingerprint: ba:21:ea:20:d6:dd:db:8f:c1:57:8b:40:ad:a1:fc:fc +# SHA1 Fingerprint: 80:1d:62:d0:7b:44:9d:5c:5c:03:5c:98:ea:61:fa:44:3c:2a:58:fe +# SHA256 Fingerprint: d1:c3:39:ea:27:84:eb:87:0f:93:4f:c5:63:4e:4a:a9:ad:55:05:01:64:01:f2:64:65:d3:7a:57:46:63:35:9f +-----BEGIN CERTIFICATE----- +MIIEXDCCA0SgAwIBAgIEOGO5ZjANBgkqhkiG9w0BAQUFADCBtDEUMBIGA1UEChML +RW50cnVzdC5uZXQxQDA+BgNVBAsUN3d3dy5lbnRydXN0Lm5ldC9DUFNfMjA0OCBp +bmNvcnAuIGJ5IHJlZi4gKGxpbWl0cyBsaWFiLikxJTAjBgNVBAsTHChjKSAxOTk5 +IEVudHJ1c3QubmV0IExpbWl0ZWQxMzAxBgNVBAMTKkVudHJ1c3QubmV0IENlcnRp +ZmljYXRpb24gQXV0aG9yaXR5ICgyMDQ4KTAeFw05OTEyMjQxNzUwNTFaFw0xOTEy +MjQxODIwNTFaMIG0MRQwEgYDVQQKEwtFbnRydXN0Lm5ldDFAMD4GA1UECxQ3d3d3 +LmVudHJ1c3QubmV0L0NQU18yMDQ4IGluY29ycC4gYnkgcmVmLiAobGltaXRzIGxp +YWIuKTElMCMGA1UECxMcKGMpIDE5OTkgRW50cnVzdC5uZXQgTGltaXRlZDEzMDEG +A1UEAxMqRW50cnVzdC5uZXQgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgKDIwNDgp +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArU1LqRKGsuqjIAcVFmQq +K0vRvwtKTY7tgHalZ7d4QMBzQshowNtTK91euHaYNZOLGp18EzoOH1u3Hs/lJBQe +sYGpjX24zGtLA/ECDNyrpUAkAH90lKGdCCmziAv1h3edVc3kw37XamSrhRSGlVuX +MlBvPci6Zgzj/L24ScF2iUkZ/cCovYmjZy/Gn7xxGWC4LeksyZB2ZnuU4q941mVT +XTzWnLLPKQP5L6RQstRIzgUyVYr9smRMDuSYB3Xbf9+5CFVghTAp+XtIpGmG4zU/ +HoZdenoVve8AjhUiVBcAkCaTvA5JaJG/+EfTnZVCwQ5N328mz8MYIWJmQ3DW1cAH +4QIDAQABo3QwcjARBglghkgBhvhCAQEEBAMCAAcwHwYDVR0jBBgwFoAUVeSB0RGA +vtiJuQijMfmhJAkWuXAwHQYDVR0OBBYEFFXkgdERgL7YibkIozH5oSQJFrlwMB0G +CSqGSIb2fQdBAAQQMA4bCFY1LjA6NC4wAwIEkDANBgkqhkiG9w0BAQUFAAOCAQEA +WUesIYSKF8mciVMeuoCFGsY8Tj6xnLZ8xpJdGGQC49MGCBFhfGPjK50xA3B20qMo +oPS7mmNz7W3lKtvtFKkrxjYR0CvrB4ul2p5cGZ1WEvVUKcgF7bISKo30Axv/55IQ +h7A6tcOdBTcSo8f0FbnVpDkWm1M6I5HxqIKiaohowXkCIryqptau37AUX7iH0N18 +f3v/rxzP5tsHrV7bhZ3QKw0z2wTR5klAEyt2+z7pnIkPFc4YsIV4IU9rTw76NmfN +B/L/CNDi3tm/Kq+4h4YhPATKt5Rof8886ZjXOP/swNlQ8C5LWK5Gb9Auw2DaclVy +vUxFnmG6v4SBkgPR0ml8xQ== +-----END CERTIFICATE----- + +# Issuer: CN=Baltimore CyberTrust Root O=Baltimore OU=CyberTrust +# Subject: CN=Baltimore CyberTrust Root O=Baltimore OU=CyberTrust +# Label: "Baltimore CyberTrust Root" +# Serial: 33554617 +# MD5 Fingerprint: ac:b6:94:a5:9c:17:e0:d7:91:52:9b:b1:97:06:a6:e4 +# SHA1 Fingerprint: d4:de:20:d0:5e:66:fc:53:fe:1a:50:88:2c:78:db:28:52:ca:e4:74 +# SHA256 Fingerprint: 16:af:57:a9:f6:76:b0:ab:12:60:95:aa:5e:ba:de:f2:2a:b3:11:19:d6:44:ac:95:cd:4b:93:db:f3:f2:6a:eb +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIEAgAAuTANBgkqhkiG9w0BAQUFADBaMQswCQYDVQQGEwJJ +RTESMBAGA1UEChMJQmFsdGltb3JlMRMwEQYDVQQLEwpDeWJlclRydXN0MSIwIAYD +VQQDExlCYWx0aW1vcmUgQ3liZXJUcnVzdCBSb290MB4XDTAwMDUxMjE4NDYwMFoX +DTI1MDUxMjIzNTkwMFowWjELMAkGA1UEBhMCSUUxEjAQBgNVBAoTCUJhbHRpbW9y +ZTETMBEGA1UECxMKQ3liZXJUcnVzdDEiMCAGA1UEAxMZQmFsdGltb3JlIEN5YmVy +VHJ1c3QgUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKMEuyKr +mD1X6CZymrV51Cni4eiVgLGw41uOKymaZN+hXe2wCQVt2yguzmKiYv60iNoS6zjr +IZ3AQSsBUnuId9Mcj8e6uYi1agnnc+gRQKfRzMpijS3ljwumUNKoUMMo6vWrJYeK +mpYcqWe4PwzV9/lSEy/CG9VwcPCPwBLKBsua4dnKM3p31vjsufFoREJIE9LAwqSu +XmD+tqYF/LTdB1kC1FkYmGP1pWPgkAx9XbIGevOF6uvUA65ehD5f/xXtabz5OTZy +dc93Uk3zyZAsuT3lySNTPx8kmCFcB5kpvcY67Oduhjprl3RjM71oGDHweI12v/ye +jl0qhqdNkNwnGjkCAwEAAaNFMEMwHQYDVR0OBBYEFOWdWTCCR1jMrPoIVDaGezq1 +BE3wMBIGA1UdEwEB/wQIMAYBAf8CAQMwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3 +DQEBBQUAA4IBAQCFDF2O5G9RaEIFoN27TyclhAO992T9Ldcw46QQF+vaKSm2eT92 +9hkTI7gQCvlYpNRhcL0EYWoSihfVCr3FvDB81ukMJY2GQE/szKN+OMY3EU/t3Wgx +jkzSswF07r51XgdIGn9w/xZchMB5hbgF/X++ZRGjD8ACtPhSNzkE1akxehi/oCr0 +Epn3o0WC4zxe9Z2etciefC7IpJ5OCBRLbf1wbWsaY71k5h+3zvDyny67G7fyUIhz +ksLi4xaNmjICq44Y3ekQEe5+NauQrz4wlHrQMz2nZQ/1/I6eYs9HRCwBXbsdtTLS +R9I4LtD+gdwyah617jzV/OeBHRnDJELqYzmp +-----END CERTIFICATE----- + +# Issuer: CN=Equifax Secure Global eBusiness CA-1 O=Equifax Secure Inc. +# Subject: CN=Equifax Secure Global eBusiness CA-1 O=Equifax Secure Inc. +# Label: "Equifax Secure Global eBusiness CA" +# Serial: 1 +# MD5 Fingerprint: 8f:5d:77:06:27:c4:98:3c:5b:93:78:e7:d7:7d:9b:cc +# SHA1 Fingerprint: 7e:78:4a:10:1c:82:65:cc:2d:e1:f1:6d:47:b4:40:ca:d9:0a:19:45 +# SHA256 Fingerprint: 5f:0b:62:ea:b5:e3:53:ea:65:21:65:16:58:fb:b6:53:59:f4:43:28:0a:4a:fb:d1:04:d7:7d:10:f9:f0:4c:07 -----BEGIN CERTIFICATE----- MIICkDCCAfmgAwIBAgIBATANBgkqhkiG9w0BAQQFADBaMQswCQYDVQQGEwJVUzEc MBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5jLjEtMCsGA1UEAxMkRXF1aWZheCBT @@ -475,9 +475,13 @@ Z70Br83gcfxaz2TE4JaY0KNA4gGK7ycH8WUBikQtBmV1UsCGECAhX2xrD2yuCRyv 8qIYNMR1pHMc8Y3c7635s3a0kr/clRAevsvIO1qEYBlWlKlV -----END CERTIFICATE----- -Equifax Secure eBusiness CA 1 -============================= - +# Issuer: CN=Equifax Secure eBusiness CA-1 O=Equifax Secure Inc. +# Subject: CN=Equifax Secure eBusiness CA-1 O=Equifax Secure Inc. +# Label: "Equifax Secure eBusiness CA 1" +# Serial: 4 +# MD5 Fingerprint: 64:9c:ef:2e:44:fc:c6:8f:52:07:d0:51:73:8f:cb:3d +# SHA1 Fingerprint: da:40:18:8b:91:89:a3:ed:ee:ae:da:97:fe:2f:9d:f5:b7:d1:8a:41 +# SHA256 Fingerprint: cf:56:ff:46:a4:a1:86:10:9d:d9:65:84:b5:ee:b5:8a:51:0c:42:75:b0:e5:f9:4f:40:bb:ae:86:5e:19:f6:73 -----BEGIN CERTIFICATE----- MIICgjCCAeugAwIBAgIBBDANBgkqhkiG9w0BAQQFADBTMQswCQYDVQQGEwJVUzEc MBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5jLjEmMCQGA1UEAxMdRXF1aWZheCBT @@ -495,9 +499,13 @@ WB5Hh1Q+WKG1tfgq73HnvMP2sUlG4tega+VWeponmHxGYhTnyfxuAxJ5gDgdSIKN /Bf+KpYrtWKmpj29f5JZzVoqgrI3eQ== -----END CERTIFICATE----- -Equifax Secure eBusiness CA 2 -============================= - +# Issuer: O=Equifax Secure OU=Equifax Secure eBusiness CA-2 +# Subject: O=Equifax Secure OU=Equifax Secure eBusiness CA-2 +# Label: "Equifax Secure eBusiness CA 2" +# Serial: 930140085 +# MD5 Fingerprint: aa:bf:bf:64:97:da:98:1d:6f:c6:08:3a:95:70:33:ca +# SHA1 Fingerprint: 39:4f:f6:85:0b:06:be:52:e5:18:56:cc:10:e1:80:e8:82:b3:85:cc +# SHA256 Fingerprint: 2f:27:4e:48:ab:a4:ac:7b:76:59:33:10:17:75:50:6d:c3:0e:e3:8e:f6:ac:d5:c0:49:32:cf:e0:41:23:42:20 -----BEGIN CERTIFICATE----- MIIDIDCCAomgAwIBAgIEN3DPtTANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJV UzEXMBUGA1UEChMORXF1aWZheCBTZWN1cmUxJjAkBgNVBAsTHUVxdWlmYXggU2Vj @@ -518,30 +526,820 @@ A4GBAAyGgq3oThr1jokn4jVYPSm0B482UJW/bsGe68SQsoWou7dC4A8HOd/7npCy E4qUoSek1nDFbZS1yX2doNLGCEnZZpum0/QL3MUmV+GRMOrN -----END CERTIFICATE----- -Thawte Time Stamping CA -======================= +# Issuer: CN=AddTrust Class 1 CA Root O=AddTrust AB OU=AddTrust TTP Network +# Subject: CN=AddTrust Class 1 CA Root O=AddTrust AB OU=AddTrust TTP Network +# Label: "AddTrust Low-Value Services Root" +# Serial: 1 +# MD5 Fingerprint: 1e:42:95:02:33:92:6b:b9:5f:c0:7f:da:d6:b2:4b:fc +# SHA1 Fingerprint: cc:ab:0e:a0:4c:23:01:d6:69:7b:dd:37:9f:cd:12:eb:24:e3:94:9d +# SHA256 Fingerprint: 8c:72:09:27:9a:c0:4e:27:5e:16:d0:7f:d3:b7:75:e8:01:54:b5:96:80:46:e3:1f:52:dd:25:76:63:24:e9:a7 +-----BEGIN CERTIFICATE----- +MIIEGDCCAwCgAwIBAgIBATANBgkqhkiG9w0BAQUFADBlMQswCQYDVQQGEwJTRTEU +MBIGA1UEChMLQWRkVHJ1c3QgQUIxHTAbBgNVBAsTFEFkZFRydXN0IFRUUCBOZXR3 +b3JrMSEwHwYDVQQDExhBZGRUcnVzdCBDbGFzcyAxIENBIFJvb3QwHhcNMDAwNTMw +MTAzODMxWhcNMjAwNTMwMTAzODMxWjBlMQswCQYDVQQGEwJTRTEUMBIGA1UEChML +QWRkVHJ1c3QgQUIxHTAbBgNVBAsTFEFkZFRydXN0IFRUUCBOZXR3b3JrMSEwHwYD +VQQDExhBZGRUcnVzdCBDbGFzcyAxIENBIFJvb3QwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQCWltQhSWDia+hBBwzexODcEyPNwTXH+9ZOEQpnXvUGW2ul +CDtbKRY654eyNAbFvAWlA3yCyykQruGIgb3WntP+LVbBFc7jJp0VLhD7Bo8wBN6n +tGO0/7Gcrjyvd7ZWxbWroulpOj0OM3kyP3CCkplhbY0wCI9xP6ZIVxn4JdxLZlyl +dI+Yrsj5wAYi56xz36Uu+1LcsRVlIPo1Zmne3yzxbrww2ywkEtvrNTVokMsAsJch +PXQhI2U0K7t4WaPW4XY5mqRJjox0r26kmqPZm9I4XJuiGMx1I4S+6+JNM3GOGvDC ++Mcdoq0Dlyz4zyXG9rgkMbFjXZJ/Y/AlyVMuH79NAgMBAAGjgdIwgc8wHQYDVR0O +BBYEFJWxtPCUtr3H2tERCSG+wa9J/RB7MAsGA1UdDwQEAwIBBjAPBgNVHRMBAf8E +BTADAQH/MIGPBgNVHSMEgYcwgYSAFJWxtPCUtr3H2tERCSG+wa9J/RB7oWmkZzBl +MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxHTAbBgNVBAsTFEFk +ZFRydXN0IFRUUCBOZXR3b3JrMSEwHwYDVQQDExhBZGRUcnVzdCBDbGFzcyAxIENB +IFJvb3SCAQEwDQYJKoZIhvcNAQEFBQADggEBACxtZBsfzQ3duQH6lmM0MkhHma6X +7f1yFqZzR1r0693p9db7RcwpiURdv0Y5PejuvE1Uhh4dbOMXJ0PhiVYrqW9yTkkz +43J8KiOavD7/KCrto/8cI7pDVwlnTUtiBi34/2ydYB7YHEt9tTEv2dB8Xfjea4MY +eDdXL+gzB2ffHsdrKpV2ro9Xo/D0UrSpUwjP4E/TelOL/bscVjby/rK25Xa71SJl +pz/+0WatC7xrmYbvP33zGDLKe8bjq2RGlfgmadlVg3sslgf/WSxEo8bl6ancoWOA +WiFeIc9TVPC6b4nbqKqVz4vjccweGyBECMB6tkD9xOQ14R0WHNC8K47Wcdk= +-----END CERTIFICATE----- + +# Issuer: CN=AddTrust External CA Root O=AddTrust AB OU=AddTrust External TTP Network +# Subject: CN=AddTrust External CA Root O=AddTrust AB OU=AddTrust External TTP Network +# Label: "AddTrust External Root" +# Serial: 1 +# MD5 Fingerprint: 1d:35:54:04:85:78:b0:3f:42:42:4d:bf:20:73:0a:3f +# SHA1 Fingerprint: 02:fa:f3:e2:91:43:54:68:60:78:57:69:4d:f5:e4:5b:68:85:18:68 +# SHA256 Fingerprint: 68:7f:a4:51:38:22:78:ff:f0:c8:b1:1f:8d:43:d5:76:67:1c:6e:b2:bc:ea:b4:13:fb:83:d9:65:d0:6d:2f:f2 +-----BEGIN CERTIFICATE----- +MIIENjCCAx6gAwIBAgIBATANBgkqhkiG9w0BAQUFADBvMQswCQYDVQQGEwJTRTEU +MBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0IEV4dGVybmFs +IFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBFeHRlcm5hbCBDQSBSb290 +MB4XDTAwMDUzMDEwNDgzOFoXDTIwMDUzMDEwNDgzOFowbzELMAkGA1UEBhMCU0Ux +FDASBgNVBAoTC0FkZFRydXN0IEFCMSYwJAYDVQQLEx1BZGRUcnVzdCBFeHRlcm5h +bCBUVFAgTmV0d29yazEiMCAGA1UEAxMZQWRkVHJ1c3QgRXh0ZXJuYWwgQ0EgUm9v +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALf3GjPm8gAELTngTlvt +H7xsD821+iO2zt6bETOXpClMfZOfvUq8k+0DGuOPz+VtUFrWlymUWoCwSXrbLpX9 +uMq/NzgtHj6RQa1wVsfwTz/oMp50ysiQVOnGXw94nZpAPA6sYapeFI+eh6FqUNzX +mk6vBbOmcZSccbNQYArHE504B4YCqOmoaSYYkKtMsE8jqzpPhNjfzp/haW+710LX +a0Tkx63ubUFfclpxCDezeWWkWaCUN/cALw3CknLa0Dhy2xSoRcRdKn23tNbE7qzN +E0S3ySvdQwAl+mG5aWpYIxG3pzOPVnVZ9c0p10a3CitlttNCbxWyuHv77+ldU9U0 +WicCAwEAAaOB3DCB2TAdBgNVHQ4EFgQUrb2YejS0Jvf6xCZU7wO94CTLVBowCwYD +VR0PBAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wgZkGA1UdIwSBkTCBjoAUrb2YejS0 +Jvf6xCZU7wO94CTLVBqhc6RxMG8xCzAJBgNVBAYTAlNFMRQwEgYDVQQKEwtBZGRU +cnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0ZXJuYWwgVFRQIE5ldHdvcmsx +IjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENBIFJvb3SCAQEwDQYJKoZIhvcN +AQEFBQADggEBALCb4IUlwtYj4g+WBpKdQZic2YR5gdkeWxQHIzZlj7DYd7usQWxH +YINRsPkyPef89iYTx4AWpb9a/IfPeHmJIZriTAcKhjW88t5RxNKWt9x+Tu5w/Rw5 +6wwCURQtjr0W4MHfRnXnJK3s9EK0hZNwEGe6nQY1ShjTK3rMUUKhemPR5ruhxSvC +Nr4TDea9Y355e6cJDUCrat2PisP29owaQgVR1EX1n6diIWgVIEM8med8vSTYqZEX +c4g/VhsxOBi0cQ+azcgOno4uG+GMmIPLHzHxREzGBHNJdmAPx/i9F4BrLunMTA5a +mnkPIAou1Z5jJh5VkpTYghdae9C8x49OhgQ= +-----END CERTIFICATE----- + +# Issuer: CN=AddTrust Public CA Root O=AddTrust AB OU=AddTrust TTP Network +# Subject: CN=AddTrust Public CA Root O=AddTrust AB OU=AddTrust TTP Network +# Label: "AddTrust Public Services Root" +# Serial: 1 +# MD5 Fingerprint: c1:62:3e:23:c5:82:73:9c:03:59:4b:2b:e9:77:49:7f +# SHA1 Fingerprint: 2a:b6:28:48:5e:78:fb:f3:ad:9e:79:10:dd:6b:df:99:72:2c:96:e5 +# SHA256 Fingerprint: 07:91:ca:07:49:b2:07:82:aa:d3:c7:d7:bd:0c:df:c9:48:58:35:84:3e:b2:d7:99:60:09:ce:43:ab:6c:69:27 +-----BEGIN CERTIFICATE----- +MIIEFTCCAv2gAwIBAgIBATANBgkqhkiG9w0BAQUFADBkMQswCQYDVQQGEwJTRTEU +MBIGA1UEChMLQWRkVHJ1c3QgQUIxHTAbBgNVBAsTFEFkZFRydXN0IFRUUCBOZXR3 +b3JrMSAwHgYDVQQDExdBZGRUcnVzdCBQdWJsaWMgQ0EgUm9vdDAeFw0wMDA1MzAx +MDQxNTBaFw0yMDA1MzAxMDQxNTBaMGQxCzAJBgNVBAYTAlNFMRQwEgYDVQQKEwtB +ZGRUcnVzdCBBQjEdMBsGA1UECxMUQWRkVHJ1c3QgVFRQIE5ldHdvcmsxIDAeBgNV +BAMTF0FkZFRydXN0IFB1YmxpYyBDQSBSb290MIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEA6Rowj4OIFMEg2Dybjxt+A3S72mnTRqX4jsIMEZBRpS9mVEBV +6tsfSlbunyNu9DnLoblv8n75XYcmYZ4c+OLspoH4IcUkzBEMP9smcnrHAZcHF/nX +GCwwfQ56HmIexkvA/X1id9NEHif2P0tEs7c42TkfYNVRknMDtABp4/MUTu7R3AnP +dzRGULD4EfL+OHn3Bzn+UZKXC1sIXzSGAa2Il+tmzV7R/9x98oTaunet3IAIx6eH +1lWfl2royBFkuucZKT8Rs3iQhCBSWxHveNCD9tVIkNAwHM+A+WD+eeSI8t0A65RF +62WUaUC6wNW0uLp9BBGo6zEFlpROWCGOn9Bg/QIDAQABo4HRMIHOMB0GA1UdDgQW +BBSBPjfYkrAfd59ctKtzquf2NGAv+jALBgNVHQ8EBAMCAQYwDwYDVR0TAQH/BAUw +AwEB/zCBjgYDVR0jBIGGMIGDgBSBPjfYkrAfd59ctKtzquf2NGAv+qFopGYwZDEL +MAkGA1UEBhMCU0UxFDASBgNVBAoTC0FkZFRydXN0IEFCMR0wGwYDVQQLExRBZGRU +cnVzdCBUVFAgTmV0d29yazEgMB4GA1UEAxMXQWRkVHJ1c3QgUHVibGljIENBIFJv +b3SCAQEwDQYJKoZIhvcNAQEFBQADggEBAAP3FUr4JNojVhaTdt02KLmuG7jD8WS6 +IBh4lSknVwW8fCr0uVFV2ocC3g8WFzH4qnkuCRO7r7IgGRLlk/lL+YPoRNWyQSW/ +iHVv/xD8SlTQX/D67zZzfRs2RcYhbbQVuE7PnFylPVoAjgbjPGsye/Kf8Lb93/Ao +GEjwxrzQvzSAlsJKsW2Ox5BF3i9nrEUEo3rcVZLJR2bYGozH7ZxOmuASu7VqTITh +4SINhwBk/ox9Yjllpu9CtoAlEmEBqCQTcAARJl/6NVDFSMwGR+gn2HCNX2TmoUQm +XiLsks3/QppEIW1cxeMiHV9HEufOX1362KqxMy3ZdvJOOjMMK7MtkAY= +-----END CERTIFICATE----- + +# Issuer: CN=AddTrust Qualified CA Root O=AddTrust AB OU=AddTrust TTP Network +# Subject: CN=AddTrust Qualified CA Root O=AddTrust AB OU=AddTrust TTP Network +# Label: "AddTrust Qualified Certificates Root" +# Serial: 1 +# MD5 Fingerprint: 27:ec:39:47:cd:da:5a:af:e2:9a:01:65:21:a9:4c:bb +# SHA1 Fingerprint: 4d:23:78:ec:91:95:39:b5:00:7f:75:8f:03:3b:21:1e:c5:4d:8b:cf +# SHA256 Fingerprint: 80:95:21:08:05:db:4b:bc:35:5e:44:28:d8:fd:6e:c2:cd:e3:ab:5f:b9:7a:99:42:98:8e:b8:f4:dc:d0:60:16 +-----BEGIN CERTIFICATE----- +MIIEHjCCAwagAwIBAgIBATANBgkqhkiG9w0BAQUFADBnMQswCQYDVQQGEwJTRTEU +MBIGA1UEChMLQWRkVHJ1c3QgQUIxHTAbBgNVBAsTFEFkZFRydXN0IFRUUCBOZXR3 +b3JrMSMwIQYDVQQDExpBZGRUcnVzdCBRdWFsaWZpZWQgQ0EgUm9vdDAeFw0wMDA1 +MzAxMDQ0NTBaFw0yMDA1MzAxMDQ0NTBaMGcxCzAJBgNVBAYTAlNFMRQwEgYDVQQK +EwtBZGRUcnVzdCBBQjEdMBsGA1UECxMUQWRkVHJ1c3QgVFRQIE5ldHdvcmsxIzAh +BgNVBAMTGkFkZFRydXN0IFF1YWxpZmllZCBDQSBSb290MIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEA5B6a/twJWoekn0e+EV+vhDTbYjx5eLfpMLXsDBwq +xBb/4Oxx64r1EW7tTw2R0hIYLUkVAcKkIhPHEWT/IhKauY5cLwjPcWqzZwFZ8V1G +87B4pfYOQnrjfxvM0PC3KP0q6p6zsLkEqv32x7SxuCqg+1jxGaBvcCV+PmlKfw8i +2O+tCBGaKZnhqkRFmhJePp1tUvznoD1oL/BLcHwTOK28FSXx1s6rosAx1i+f4P8U +WfyEk9mHfExUE+uf0S0R+Bg6Ot4l2ffTQO2kBhLEO+GRwVY18BTcZTYJbqukB8c1 +0cIDMzZbdSZtQvESa0NvS3GU+jQd7RNuyoB/mC9suWXY6QIDAQABo4HUMIHRMB0G +A1UdDgQWBBQ5lYtii1zJ1IC6WA+XPxUIQ8yYpzALBgNVHQ8EBAMCAQYwDwYDVR0T +AQH/BAUwAwEB/zCBkQYDVR0jBIGJMIGGgBQ5lYtii1zJ1IC6WA+XPxUIQ8yYp6Fr +pGkwZzELMAkGA1UEBhMCU0UxFDASBgNVBAoTC0FkZFRydXN0IEFCMR0wGwYDVQQL +ExRBZGRUcnVzdCBUVFAgTmV0d29yazEjMCEGA1UEAxMaQWRkVHJ1c3QgUXVhbGlm +aWVkIENBIFJvb3SCAQEwDQYJKoZIhvcNAQEFBQADggEBABmrder4i2VhlRO6aQTv +hsoToMeqT2QbPxj2qC0sVY8FtzDqQmodwCVRLae/DLPt7wh/bDxGGuoYQ992zPlm +hpwsaPXpF/gxsxjE1kh9I0xowX67ARRvxdlu3rsEQmr49lx95dr6h+sNNVJn0J6X +dgWTP5XHAeZpVTh/EGGZyeNfpso+gmNIquIISD6q8rKFYqa0p9m9N5xotS1WfbC3 +P6CxB9bpT9zeRXEwMn8bLgn5v1Kh7sKAPgZcLlVAwRv1cEWw3F369nJad9Jjzc9Y +iQBCYz95OdBEsIJuQRno3eDBiFrRHnGTHyQwdOUeqN48Jzd/g66ed8/wMLH/S5no +xqE= +-----END CERTIFICATE----- + +# Issuer: CN=Entrust Root Certification Authority O=Entrust, Inc. OU=www.entrust.net/CPS is incorporated by reference/(c) 2006 Entrust, Inc. +# Subject: CN=Entrust Root Certification Authority O=Entrust, Inc. OU=www.entrust.net/CPS is incorporated by reference/(c) 2006 Entrust, Inc. +# Label: "Entrust Root Certification Authority" +# Serial: 1164660820 +# MD5 Fingerprint: d6:a5:c3:ed:5d:dd:3e:00:c1:3d:87:92:1f:1d:3f:e4 +# SHA1 Fingerprint: b3:1e:b1:b7:40:e3:6c:84:02:da:dc:37:d4:4d:f5:d4:67:49:52:f9 +# SHA256 Fingerprint: 73:c1:76:43:4f:1b:c6:d5:ad:f4:5b:0e:76:e7:27:28:7c:8d:e5:76:16:c1:e6:e6:14:1a:2b:2c:bc:7d:8e:4c +-----BEGIN CERTIFICATE----- +MIIEkTCCA3mgAwIBAgIERWtQVDANBgkqhkiG9w0BAQUFADCBsDELMAkGA1UEBhMC +VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xOTA3BgNVBAsTMHd3dy5lbnRydXN0 +Lm5ldC9DUFMgaXMgaW5jb3Jwb3JhdGVkIGJ5IHJlZmVyZW5jZTEfMB0GA1UECxMW +KGMpIDIwMDYgRW50cnVzdCwgSW5jLjEtMCsGA1UEAxMkRW50cnVzdCBSb290IENl +cnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA2MTEyNzIwMjM0MloXDTI2MTEyNzIw +NTM0MlowgbAxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMTkw +NwYDVQQLEzB3d3cuZW50cnVzdC5uZXQvQ1BTIGlzIGluY29ycG9yYXRlZCBieSBy +ZWZlcmVuY2UxHzAdBgNVBAsTFihjKSAyMDA2IEVudHJ1c3QsIEluYy4xLTArBgNV +BAMTJEVudHJ1c3QgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBALaVtkNC+sZtKm9I35RMOVcF7sN5EUFo +Nu3s/poBj6E4KPz3EEZmLk0eGrEaTsbRwJWIsMn/MYszA9u3g3s+IIRe7bJWKKf4 +4LlAcTfFy0cOlypowCKVYhXbR9n10Cv/gkvJrT7eTNuQgFA/CYqEAOwwCj0Yzfv9 +KlmaI5UXLEWeH25DeW0MXJj+SKfFI0dcXv1u5x609mhF0YaDW6KKjbHjKYD+JXGI +rb68j6xSlkuqUY3kEzEZ6E5Nn9uss2rVvDlUccp6en+Q3X0dgNmBu1kmwhH+5pPi +94DkZfs0Nw4pgHBNrziGLp5/V6+eF67rHMsoIV+2HNjnogQi+dPa2MsCAwEAAaOB +sDCBrTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zArBgNVHRAEJDAi +gA8yMDA2MTEyNzIwMjM0MlqBDzIwMjYxMTI3MjA1MzQyWjAfBgNVHSMEGDAWgBRo +kORnpKZTgMeGZqTx90tD+4S9bTAdBgNVHQ4EFgQUaJDkZ6SmU4DHhmak8fdLQ/uE +vW0wHQYJKoZIhvZ9B0EABBAwDhsIVjcuMTo0LjADAgSQMA0GCSqGSIb3DQEBBQUA +A4IBAQCT1DCw1wMgKtD5Y+iRDAUgqV8ZyntyTtSx29CW+1RaGSwMCPeyvIWonX9t +O1KzKtvn1ISMY/YPyyYBkVBs9F8U4pN0wBOeMDpQ47RgxRzwIkSNcUesyBrJ6Zua +AGAT/3B+XxFNSRuzFVJ7yVTav52Vr2ua2J7p8eRDjeIRRDq/r72DQnNSi6q7pynP +9WQcCk3RvKqsnyrQ/39/2n3qse0wJcGE2jTSW3iDVuycNsMm4hH2Z0kdkquM++v/ +eu6FSqdQgPCnXEqULl8FmTxSQeDNtGPPAUO6nIPcj2A781q0tHuu2guQOHXvgR1m +0vdXcDazv/wor3ElhVsT/h5/WrQ8 +-----END CERTIFICATE----- + +# Issuer: CN=GeoTrust Global CA O=GeoTrust Inc. +# Subject: CN=GeoTrust Global CA O=GeoTrust Inc. +# Label: "GeoTrust Global CA" +# Serial: 144470 +# MD5 Fingerprint: f7:75:ab:29:fb:51:4e:b7:77:5e:ff:05:3c:99:8e:f5 +# SHA1 Fingerprint: de:28:f4:a4:ff:e5:b9:2f:a3:c5:03:d1:a3:49:a7:f9:96:2a:82:12 +# SHA256 Fingerprint: ff:85:6a:2d:25:1d:cd:88:d3:66:56:f4:50:12:67:98:cf:ab:aa:de:40:79:9c:72:2d:e4:d2:b5:db:36:a7:3a +-----BEGIN CERTIFICATE----- +MIIDVDCCAjygAwIBAgIDAjRWMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT +MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i +YWwgQ0EwHhcNMDIwNTIxMDQwMDAwWhcNMjIwNTIxMDQwMDAwWjBCMQswCQYDVQQG +EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMSR2VvVHJ1c3Qg +R2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2swYYzD9 +9BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9mOSm9BXiLnTjoBbdq +fnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIuT8rxh0PBFpVXLVDv +iS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6cJmTM386DGXHKTubU +1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmRCw7+OC7RHQWa9k0+ +bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5aszPeE4uwc2hGKceeoW +MPRfwCvocWvk+QIDAQABo1MwUTAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTA +ephojYn7qwVkDBF9qn1luMrMTjAfBgNVHSMEGDAWgBTAephojYn7qwVkDBF9qn1l +uMrMTjANBgkqhkiG9w0BAQUFAAOCAQEANeMpauUvXVSOKVCUn5kaFOSPeCpilKIn +Z57QzxpeR+nBsqTP3UEaBU6bS+5Kb1VSsyShNwrrZHYqLizz/Tt1kL/6cdjHPTfS +tQWVYrmm3ok9Nns4d0iXrKYgjy6myQzCsplFAMfOEVEiIuCl6rYVSAlk6l5PdPcF +PseKUgzbFbS9bZvlxrFUaKnjaZC2mqUPuLk/IH2uSrW4nOQdtqvmlKXBx4Ot2/Un +hw4EbNX/3aBd7YdStysVAq45pmp06drE57xNNB6pXE0zX5IJL4hmXXeXxx12E6nV +5fEWCRE11azbJHFwLJhWC9kXtNHjUStedejV0NxPNO3CBWaAocvmMw== +-----END CERTIFICATE----- + +# Issuer: CN=GeoTrust Global CA 2 O=GeoTrust Inc. +# Subject: CN=GeoTrust Global CA 2 O=GeoTrust Inc. +# Label: "GeoTrust Global CA 2" +# Serial: 1 +# MD5 Fingerprint: 0e:40:a7:6c:de:03:5d:8f:d1:0f:e4:d1:8d:f9:6c:a9 +# SHA1 Fingerprint: a9:e9:78:08:14:37:58:88:f2:05:19:b0:6d:2b:0d:2b:60:16:90:7d +# SHA256 Fingerprint: ca:2d:82:a0:86:77:07:2f:8a:b6:76:4f:f0:35:67:6c:fe:3e:5e:32:5e:01:21:72:df:3f:92:09:6d:b7:9b:85 +-----BEGIN CERTIFICATE----- +MIIDZjCCAk6gAwIBAgIBATANBgkqhkiG9w0BAQUFADBEMQswCQYDVQQGEwJVUzEW +MBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEdMBsGA1UEAxMUR2VvVHJ1c3QgR2xvYmFs +IENBIDIwHhcNMDQwMzA0MDUwMDAwWhcNMTkwMzA0MDUwMDAwWjBEMQswCQYDVQQG +EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEdMBsGA1UEAxMUR2VvVHJ1c3Qg +R2xvYmFsIENBIDIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDvPE1A +PRDfO1MA4Wf+lGAVPoWI8YkNkMgoI5kF6CsgncbzYEbYwbLVjDHZ3CB5JIG/NTL8 +Y2nbsSpr7iFY8gjpeMtvy/wWUsiRxP89c96xPqfCfWbB9X5SJBri1WeR0IIQ13hL +TytCOb1kLUCgsBDTOEhGiKEMuzozKmKY+wCdE1l/bztyqu6mD4b5BWHqZ38MN5aL +5mkWRxHCJ1kDs6ZgwiFAVvqgx306E+PsV8ez1q6diYD3Aecs9pYrEw15LNnA5IZ7 +S4wMcoKK+xfNAGw6EzywhIdLFnopsk/bHdQL82Y3vdj2V7teJHq4PIu5+pIaGoSe +2HSPqht/XvT+RSIhAgMBAAGjYzBhMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE +FHE4NvICMVNHK266ZUapEBVYIAUJMB8GA1UdIwQYMBaAFHE4NvICMVNHK266ZUap +EBVYIAUJMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQUFAAOCAQEAA/e1K6td +EPx7srJerJsOflN4WT5CBP51o62sgU7XAotexC3IUnbHLB/8gTKY0UvGkpMzNTEv +/NgdRN3ggX+d6YvhZJFiCzkIjKx0nVnZellSlxG5FntvRdOW2TF9AjYPnDtuzywN +A0ZF66D0f0hExghAzN4bcLUprbqLOzRldRtxIR0sFAqwlpW41uryZfspuk/qkZN0 +abby/+Ea0AzRdoXLiiW9l14sbxWZJue2Kf8i7MkCx1YAzUm5s2x7UwQa4qjJqhIF +I8LO57sEAszAR6LkxCkvW0VXiVHuPOtSCP8HNR6fNWpHSlaY0VqFH4z1Ir+rzoPz +4iIprn2DQKi6bA== +-----END CERTIFICATE----- + +# Issuer: CN=GeoTrust Universal CA O=GeoTrust Inc. +# Subject: CN=GeoTrust Universal CA O=GeoTrust Inc. +# Label: "GeoTrust Universal CA" +# Serial: 1 +# MD5 Fingerprint: 92:65:58:8b:a2:1a:31:72:73:68:5c:b4:a5:7a:07:48 +# SHA1 Fingerprint: e6:21:f3:35:43:79:05:9a:4b:68:30:9d:8a:2f:74:22:15:87:ec:79 +# SHA256 Fingerprint: a0:45:9b:9f:63:b2:25:59:f5:fa:5d:4c:6d:b3:f9:f7:2f:f1:93:42:03:35:78:f0:73:bf:1d:1b:46:cb:b9:12 +-----BEGIN CERTIFICATE----- +MIIFaDCCA1CgAwIBAgIBATANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJVUzEW +MBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEeMBwGA1UEAxMVR2VvVHJ1c3QgVW5pdmVy +c2FsIENBMB4XDTA0MDMwNDA1MDAwMFoXDTI5MDMwNDA1MDAwMFowRTELMAkGA1UE +BhMCVVMxFjAUBgNVBAoTDUdlb1RydXN0IEluYy4xHjAcBgNVBAMTFUdlb1RydXN0 +IFVuaXZlcnNhbCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKYV +VaCjxuAfjJ0hUNfBvitbtaSeodlyWL0AG0y/YckUHUWCq8YdgNY96xCcOq9tJPi8 +cQGeBvV8Xx7BDlXKg5pZMK4ZyzBIle0iN430SppyZj6tlcDgFgDgEB8rMQ7XlFTT +QjOgNB0eRXbdT8oYN+yFFXoZCPzVx5zw8qkuEKmS5j1YPakWaDwvdSEYfyh3peFh +F7em6fgemdtzbvQKoiFs7tqqhZJmr/Z6a4LauiIINQ/PQvE1+mrufislzDoR5G2v +c7J2Ha3QsnhnGqQ5HFELZ1aD/ThdDc7d8Lsrlh/eezJS/R27tQahsiFepdaVaH/w +mZ7cRQg+59IJDTWU3YBOU5fXtQlEIGQWFwMCTFMNaN7VqnJNk22CDtucvc+081xd +VHppCZbW2xHBjXWotM85yM48vCR85mLK4b19p71XZQvk/iXttmkQ3CgaRr0BHdCX +teGYO8A3ZNY9lO4L4fUorgtWv3GLIylBjobFS1J72HGrH4oVpjuDWtdYAVHGTEHZ +f9hBZ3KiKN9gg6meyHv8U3NyWfWTehd2Ds735VzZC1U0oqpbtWpU5xPKV+yXbfRe +Bi9Fi1jUIxaS5BZuKGNZMN9QAZxjiRqf2xeUgnA3wySemkfWWspOqGmJch+RbNt+ +nhutxx9z3SxPGWX9f5NAEC7S8O08ni4oPmkmM8V7AgMBAAGjYzBhMA8GA1UdEwEB +/wQFMAMBAf8wHQYDVR0OBBYEFNq7LqqwDLiIJlF0XG0D08DYj3rWMB8GA1UdIwQY +MBaAFNq7LqqwDLiIJlF0XG0D08DYj3rWMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG +9w0BAQUFAAOCAgEAMXjmx7XfuJRAyXHEqDXsRh3ChfMoWIawC/yOsjmPRFWrZIRc +aanQmjg8+uUfNeVE44B5lGiku8SfPeE0zTBGi1QrlaXv9z+ZhP015s8xxtxqv6fX +IwjhmF7DWgh2qaavdy+3YL1ERmrvl/9zlcGO6JP7/TG37FcREUWbMPEaiDnBTzyn +ANXH/KttgCJwpQzgXQQpAvvLoJHRfNbDflDVnVi+QTjruXU8FdmbyUqDWcDaU/0z +uzYYm4UPFd3uLax2k7nZAY1IEKj79TiG8dsKxr2EoyNB3tZ3b4XUhRxQ4K5RirqN +Pnbiucon8l+f725ZDQbYKxek0nxru18UGkiPGkzns0ccjkxFKyDuSN/n3QmOGKja +QI2SJhFTYXNd673nxE0pN2HrrDktZy4W1vUAg4WhzH92xH3kt0tm7wNFYGm2DFKW +koRepqO1pD4r2czYG0eq8kTaT/kD6PAUyz/zg97QwVTjt+gKN02LIFkDMBmhLMi9 +ER/frslKxfMnZmaGrGiR/9nmUxwPi1xpZQomyB40w11Re9epnAahNt3ViZS82eQt +DF4JbAiXfKM9fJP/P6EUp8+1Xevb2xzEdt+Iub1FBZUbrvxGakyvSOPOrg/Sfuvm +bJxPgWp6ZKy7PtXny3YuxadIwVyQD8vIP/rmMuGNG2+k5o7Y+SlIis5z/iw= +-----END CERTIFICATE----- + +# Issuer: CN=GeoTrust Universal CA 2 O=GeoTrust Inc. +# Subject: CN=GeoTrust Universal CA 2 O=GeoTrust Inc. +# Label: "GeoTrust Universal CA 2" +# Serial: 1 +# MD5 Fingerprint: 34:fc:b8:d0:36:db:9e:14:b3:c2:f2:db:8f:e4:94:c7 +# SHA1 Fingerprint: 37:9a:19:7b:41:85:45:35:0c:a6:03:69:f3:3c:2e:af:47:4f:20:79 +# SHA256 Fingerprint: a0:23:4f:3b:c8:52:7c:a5:62:8e:ec:81:ad:5d:69:89:5d:a5:68:0d:c9:1d:1c:b8:47:7f:33:f8:78:b9:5b:0b +-----BEGIN CERTIFICATE----- +MIIFbDCCA1SgAwIBAgIBATANBgkqhkiG9w0BAQUFADBHMQswCQYDVQQGEwJVUzEW +MBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEgMB4GA1UEAxMXR2VvVHJ1c3QgVW5pdmVy +c2FsIENBIDIwHhcNMDQwMzA0MDUwMDAwWhcNMjkwMzA0MDUwMDAwWjBHMQswCQYD +VQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEgMB4GA1UEAxMXR2VvVHJ1 +c3QgVW5pdmVyc2FsIENBIDIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC +AQCzVFLByT7y2dyxUxpZKeexw0Uo5dfR7cXFS6GqdHtXr0om/Nj1XqduGdt0DE81 +WzILAePb63p3NeqqWuDW6KFXlPCQo3RWlEQwAx5cTiuFJnSCegx2oG9NzkEtoBUG +FF+3Qs17j1hhNNwqCPkuwwGmIkQcTAeC5lvO0Ep8BNMZcyfwqph/Lq9O64ceJHdq +XbboW0W63MOhBW9Wjo8QJqVJwy7XQYci4E+GymC16qFjwAGXEHm9ADwSbSsVsaxL +se4YuU6W3Nx2/zu+z18DwPw76L5GG//aQMJS9/7jOvdqdzXQ2o3rXhhqMcceujwb +KNZrVMaqW9eiLBsZzKIC9ptZvTdrhrVtgrrY6slWvKk2WP0+GfPtDCapkzj4T8Fd +IgbQl+rhrcZV4IErKIM6+vR7IVEAvlI4zs1meaj0gVbi0IMJR1FbUGrP20gaXT73 +y/Zl92zxlfgCOzJWgjl6W70viRu/obTo/3+NjN8D8WBOWBFM66M/ECuDmgFz2ZRt +hAAnZqzwcEAJQpKtT5MNYQlRJNiS1QuUYbKHsu3/mjX/hVTK7URDrBs8FmtISgoc +QIgfksILAAX/8sgCSqSqqcyZlpwvWOB94b67B9xfBHJcMTTD7F8t4D1kkCLm0ey4 +Lt1ZrtmhN79UNdxzMk+MBB4zsslG8dhcyFVQyWi9qLo2CQIDAQABo2MwYTAPBgNV +HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR281Xh+qQ2+/CfXGJx7Tz0RzgQKzAfBgNV +HSMEGDAWgBR281Xh+qQ2+/CfXGJx7Tz0RzgQKzAOBgNVHQ8BAf8EBAMCAYYwDQYJ +KoZIhvcNAQEFBQADggIBAGbBxiPz2eAubl/oz66wsCVNK/g7WJtAJDday6sWSf+z +dXkzoS9tcBc0kf5nfo/sm+VegqlVHy/c1FEHEv6sFj4sNcZj/NwQ6w2jqtB8zNHQ +L1EuxBRa3ugZ4T7GzKQp5y6EqgYweHZUcyiYWTjgAA1i00J9IZ+uPTqM1fp3DRgr +Fg5fNuH8KrUwJM/gYwx7WBr+mbpCErGR9Hxo4sjoryzqyX6uuyo9DRXcNJW2GHSo +ag/HtPQTxORb7QrSpJdMKu0vbBKJPfEncKpqA1Ihn0CoZ1Dy81of398j9tx4TuaY +T1U6U+Pv8vSfx3zYWK8pIpe44L2RLrB27FcRz+8pRPPphXpgY+RdM4kX2TGq2tbz +GDVyz4crL2MjhF2EjD9XoIj8mZEoJmmZ1I+XRL6O1UixpCgp8RW04eWe3fiPpm8m +1wk8OhwRDqZsN/etRIcsKMfYdIKz0G9KV7s1KSegi+ghp4dkNl3M2Basx7InQJJV +OCiNUW7dFGdTbHFcJoRNdVq2fmBWqU2t+5sel/MN2dKXVHfaPRK34B7vCAas+YWH +6aLcr34YEoP9VhdBLtUpgn2Z9DH2canPLAEnpQW5qrJITirvn5NSUZU8UnOOVkwX +QMAJKOSLakhT2+zNVVXxxvjpoixMptEmX36vWkzaH6byHCx+rgIW0lbQL1dTR+iS +-----END CERTIFICATE----- + +# Issuer: CN=America Online Root Certification Authority 1 O=America Online Inc. +# Subject: CN=America Online Root Certification Authority 1 O=America Online Inc. +# Label: "America Online Root Certification Authority 1" +# Serial: 1 +# MD5 Fingerprint: 14:f1:08:ad:9d:fa:64:e2:89:e7:1c:cf:a8:ad:7d:5e +# SHA1 Fingerprint: 39:21:c1:15:c1:5d:0e:ca:5c:cb:5b:c4:f0:7d:21:d8:05:0b:56:6a +# SHA256 Fingerprint: 77:40:73:12:c6:3a:15:3d:5b:c0:0b:4e:51:75:9c:df:da:c2:37:dc:2a:33:b6:79:46:e9:8e:9b:fa:68:0a:e3 +-----BEGIN CERTIFICATE----- +MIIDpDCCAoygAwIBAgIBATANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEc +MBoGA1UEChMTQW1lcmljYSBPbmxpbmUgSW5jLjE2MDQGA1UEAxMtQW1lcmljYSBP +bmxpbmUgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAxMB4XDTAyMDUyODA2 +MDAwMFoXDTM3MTExOTIwNDMwMFowYzELMAkGA1UEBhMCVVMxHDAaBgNVBAoTE0Ft +ZXJpY2EgT25saW5lIEluYy4xNjA0BgNVBAMTLUFtZXJpY2EgT25saW5lIFJvb3Qg +Q2VydGlmaWNhdGlvbiBBdXRob3JpdHkgMTCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAKgv6KRpBgNHw+kqmP8ZonCaxlCyfqXfaE0bfA+2l2h9LaaLl+lk +hsmj76CGv2BlnEtUiMJIxUo5vxTjWVXlGbR0yLQFOVwWpeKVBeASrlmLojNoWBym +1BW32J/X3HGrfpq/m44zDyL9Hy7nBzbvYjnF3cu6JRQj3gzGPTzOggjmZj7aUTsW +OqMFf6Dch9Wc/HKpoH145LcxVR5lu9RhsCFg7RAycsWSJR74kEoYeEfffjA3PlAb +2xzTa5qGUwew76wGePiEmf4hjUyAtgyC9mZweRrTT6PP8c9GsEsPPt2IYriMqQko +O3rHl+Ee5fSfwMCuJKDIodkP1nsmgmkyPacCAwEAAaNjMGEwDwYDVR0TAQH/BAUw +AwEB/zAdBgNVHQ4EFgQUAK3Zo/Z59m50qX8zPYEX10zPM94wHwYDVR0jBBgwFoAU +AK3Zo/Z59m50qX8zPYEX10zPM94wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB +BQUAA4IBAQB8itEfGDeC4Liwo+1WlchiYZwFos3CYiZhzRAW18y0ZTTQEYqtqKkF +Zu90821fnZmv9ov761KyBZiibyrFVL0lvV+uyIbqRizBs73B6UlwGBaXCBOMIOAb +LjpHyx7kADCVW/RFo8AasAFOq73AI25jP4BKxQft3OJvx8Fi8eNy1gTIdGcL+oir +oQHIb/AUr9KZzVGTfu0uOMe9zkZQPXLjeSWdm4grECDdpbgyn43gKd8hdIaC2y+C +MMbHNYaz+ZZfRtsMRf3zUMNvxsNIrUam4SdHCh0Om7bCd39j8uB9Gr784N/Xx6ds +sPmuujz9dLQR6FgNgLzTqIA6me11zEZ7 +-----END CERTIFICATE----- + +# Issuer: CN=America Online Root Certification Authority 2 O=America Online Inc. +# Subject: CN=America Online Root Certification Authority 2 O=America Online Inc. +# Label: "America Online Root Certification Authority 2" +# Serial: 1 +# MD5 Fingerprint: d6:ed:3c:ca:e2:66:0f:af:10:43:0d:77:9b:04:09:bf +# SHA1 Fingerprint: 85:b5:ff:67:9b:0c:79:96:1f:c8:6e:44:22:00:46:13:db:17:92:84 +# SHA256 Fingerprint: 7d:3b:46:5a:60:14:e5:26:c0:af:fc:ee:21:27:d2:31:17:27:ad:81:1c:26:84:2d:00:6a:f3:73:06:cc:80:bd +-----BEGIN CERTIFICATE----- +MIIFpDCCA4ygAwIBAgIBATANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEc +MBoGA1UEChMTQW1lcmljYSBPbmxpbmUgSW5jLjE2MDQGA1UEAxMtQW1lcmljYSBP +bmxpbmUgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAyMB4XDTAyMDUyODA2 +MDAwMFoXDTM3MDkyOTE0MDgwMFowYzELMAkGA1UEBhMCVVMxHDAaBgNVBAoTE0Ft +ZXJpY2EgT25saW5lIEluYy4xNjA0BgNVBAMTLUFtZXJpY2EgT25saW5lIFJvb3Qg +Q2VydGlmaWNhdGlvbiBBdXRob3JpdHkgMjCCAiIwDQYJKoZIhvcNAQEBBQADggIP +ADCCAgoCggIBAMxBRR3pPU0Q9oyxQcngXssNt79Hc9PwVU3dxgz6sWYFas14tNwC +206B89enfHG8dWOgXeMHDEjsJcQDIPT/DjsS/5uN4cbVG7RtIuOx238hZK+GvFci +KtZHgVdEglZTvYYUAQv8f3SkWq7xuhG1m1hagLQ3eAkzfDJHA1zEpYNI9FdWboE2 +JxhP7JsowtS013wMPgwr38oE18aO6lhOqKSlGBxsRZijQdEt0sdtjRnxrXm3gT+9 +BoInLRBYBbV4Bbkv2wxrkJB+FFk4u5QkE+XRnRTf04JNRvCAOVIyD+OEsnpD8l7e +Xz8d3eOyG6ChKiMDbi4BFYdcpnV1x5dhvt6G3NRI270qv0pV2uh9UPu0gBe4lL8B +PeraunzgWGcXuVjgiIZGZ2ydEEdYMtA1fHkqkKJaEBEjNa0vzORKW6fIJ/KD3l67 +Xnfn6KVuY8INXWHQjNJsWiEOyiijzirplcdIz5ZvHZIlyMbGwcEMBawmxNJ10uEq +Z8A9W6Wa6897GqidFEXlD6CaZd4vKL3Ob5Rmg0gp2OpljK+T2WSfVVcmv2/LNzGZ +o2C7HK2JNDJiuEMhBnIMoVxtRsX6Kc8w3onccVvdtjc+31D1uAclJuW8tf48ArO3 ++L5DwYcRlJ4jbBeKuIonDFRH8KmzwICMoCfrHRnjB453cMor9H124HhnAgMBAAGj +YzBhMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFE1FwWg4u3OpaaEg5+31IqEj +FNeeMB8GA1UdIwQYMBaAFE1FwWg4u3OpaaEg5+31IqEjFNeeMA4GA1UdDwEB/wQE +AwIBhjANBgkqhkiG9w0BAQUFAAOCAgEAZ2sGuV9FOypLM7PmG2tZTiLMubekJcmn +xPBUlgtk87FYT15R/LKXeydlwuXK5w0MJXti4/qftIe3RUavg6WXSIylvfEWK5t2 +LHo1YGwRgJfMqZJS5ivmae2p+DYtLHe/YUjRYwu5W1LtGLBDQiKmsXeu3mnFzccc +obGlHBD7GL4acN3Bkku+KVqdPzW+5X1R+FXgJXUjhx5c3LqdsKyzadsXg8n33gy8 +CNyRnqjQ1xU3c6U1uPx+xURABsPr+CKAXEfOAuMRn0T//ZoyzH1kUQ7rVyZ2OuMe +IjzCpjbdGe+n/BLzJsBZMYVMnNjP36TMzCmT/5RtdlwTCJfy7aULTd3oyWgOZtMA +DjMSW7yV5TKQqLPGbIOtd+6Lfn6xqavT4fG2wLHqiMDn05DpKJKUe2h7lyoKZy2F +AjgQ5ANh1NolNscIWC2hp1GvMApJ9aZphwctREZ2jirlmjvXGKL8nDgQzMY70rUX +Om/9riW99XJZZLF0KjhfGEzfz3EEWjbUvy+ZnOjZurGV5gJLIaFb1cFPj65pbVPb +AZO1XB4Y3WRayhgoPmMEEf0cjQAPuDffZ4qdZqkCapH/E8ovXYO8h5Ns3CRRFgQl +Zvqz2cK6Kb6aSDiCmfS/O0oxGfm/jiEzFMpPVF/7zvuPcX/9XhmgD0uRuMRUvAaw +RY8mkaKO/qk= +-----END CERTIFICATE----- + +# Issuer: CN=AAA Certificate Services O=Comodo CA Limited +# Subject: CN=AAA Certificate Services O=Comodo CA Limited +# Label: "Comodo AAA Services root" +# Serial: 1 +# MD5 Fingerprint: 49:79:04:b0:eb:87:19:ac:47:b0:bc:11:51:9b:74:d0 +# SHA1 Fingerprint: d1:eb:23:a4:6d:17:d6:8f:d9:25:64:c2:f1:f1:60:17:64:d8:e3:49 +# SHA256 Fingerprint: d7:a7:a0:fb:5d:7e:27:31:d7:71:e9:48:4e:bc:de:f7:1d:5f:0c:3e:0a:29:48:78:2b:c8:3e:e0:ea:69:9e:f4 +-----BEGIN CERTIFICATE----- +MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb +MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow +GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj +YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM +GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua +BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe +3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4 +YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR +rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm +ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU +oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF +MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v +QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t +b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF +AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q +GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz +Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2 +G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi +l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3 +smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg== +-----END CERTIFICATE----- + +# Issuer: CN=Secure Certificate Services O=Comodo CA Limited +# Subject: CN=Secure Certificate Services O=Comodo CA Limited +# Label: "Comodo Secure Services root" +# Serial: 1 +# MD5 Fingerprint: d3:d9:bd:ae:9f:ac:67:24:b3:c8:1b:52:e1:b9:a9:bd +# SHA1 Fingerprint: 4a:65:d5:f4:1d:ef:39:b8:b8:90:4a:4a:d3:64:81:33:cf:c7:a1:d1 +# SHA256 Fingerprint: bd:81:ce:3b:4f:65:91:d1:1a:67:b5:fc:7a:47:fd:ef:25:52:1b:f9:aa:4e:18:b9:e3:df:2e:34:a7:80:3b:e8 +-----BEGIN CERTIFICATE----- +MIIEPzCCAyegAwIBAgIBATANBgkqhkiG9w0BAQUFADB+MQswCQYDVQQGEwJHQjEb +MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow +GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEkMCIGA1UEAwwbU2VjdXJlIENlcnRp +ZmljYXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVow +fjELMAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxJDAiBgNV +BAMMG1NlY3VyZSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAMBxM4KK0HDrc4eCQNUd5MvJDkKQ+d40uaG6EfQlhfPM +cm3ye5drswfxdySRXyWP9nQ95IDC+DwN879A6vfIUtFyb+/Iq0G4bi4XKpVpDM3S +HpR7LZQdqnXXs5jLrLxkU0C8j6ysNstcrbvd4JQX7NFc0L/vpZXJkMWwrPsbQ996 +CF23uPJAGysnnlDOXmWCiIxe004MeuoIkbY2qitC++rCoznl2yY4rYsK7hljxxwk +3wN42ubqwUcaCwtGCd0C/N7Lh1/XMGNooa7cMqG6vv5Eq2i2pRcV/b3Vp6ea5EQz +6YiO/O1R65NxTq0B50SOqy3LqP4BSUjwwN3HaNiS/j0CAwEAAaOBxzCBxDAdBgNV +HQ4EFgQUPNiTiMLAggnMAZkGkyDpnnAJY08wDgYDVR0PAQH/BAQDAgEGMA8GA1Ud +EwEB/wQFMAMBAf8wgYEGA1UdHwR6MHgwO6A5oDeGNWh0dHA6Ly9jcmwuY29tb2Rv +Y2EuY29tL1NlY3VyZUNlcnRpZmljYXRlU2VydmljZXMuY3JsMDmgN6A1hjNodHRw +Oi8vY3JsLmNvbW9kby5uZXQvU2VjdXJlQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmww +DQYJKoZIhvcNAQEFBQADggEBAIcBbSMdflsXfcFhMs+P5/OKlFlm4J4oqF7Tt/Q0 +5qo5spcWxYJvMqTpjOev/e/C6LlLqqP05tqNZSH7uoDrJiiFGv45jN5bBAS0VPmj +Z55B+glSzAVIqMk/IQQezkhr/IXownuvf7fM+F86/TXGDe+X3EyrEeFryzHRbPtI +gKvcnDe4IRRLDXE97IMzbtFuMhbsmMcWi1mmNKsFVy2T96oTy9IT4rcuO81rUBcJ +aD61JlfutuC23bkpgHl9j6PwpCikFcSF9CfUa7/lXORlAnZUtOM3ZiTTGWHIUhDl +izeauan5Hb/qmZJhlv8BzaFfDbxxvA6sCx1HRR3B7Hzs/Sk= +-----END CERTIFICATE----- + +# Issuer: CN=Trusted Certificate Services O=Comodo CA Limited +# Subject: CN=Trusted Certificate Services O=Comodo CA Limited +# Label: "Comodo Trusted Services root" +# Serial: 1 +# MD5 Fingerprint: 91:1b:3f:6e:cd:9e:ab:ee:07:fe:1f:71:d2:b3:61:27 +# SHA1 Fingerprint: e1:9f:e3:0e:8b:84:60:9e:80:9b:17:0d:72:a8:c5:ba:6e:14:09:bd +# SHA256 Fingerprint: 3f:06:e5:56:81:d4:96:f5:be:16:9e:b5:38:9f:9f:2b:8f:f6:1e:17:08:df:68:81:72:48:49:cd:5d:27:cb:69 +-----BEGIN CERTIFICATE----- +MIIEQzCCAyugAwIBAgIBATANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJHQjEb +MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow +GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDElMCMGA1UEAwwcVHJ1c3RlZCBDZXJ0 +aWZpY2F0ZSBTZXJ2aWNlczAeFw0wNDAxMDEwMDAwMDBaFw0yODEyMzEyMzU5NTla +MH8xCzAJBgNVBAYTAkdCMRswGQYDVQQIDBJHcmVhdGVyIE1hbmNoZXN0ZXIxEDAO +BgNVBAcMB1NhbGZvcmQxGjAYBgNVBAoMEUNvbW9kbyBDQSBMaW1pdGVkMSUwIwYD +VQQDDBxUcnVzdGVkIENlcnRpZmljYXRlIFNlcnZpY2VzMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEA33FvNlhTWvI2VFeAxHQIIO0Yfyod5jWaHiWsnOWW +fnJSoBVC21ndZHoa0Lh73TkVvFVIxO06AOoxEbrycXQaZ7jPM8yoMa+j49d/vzMt +TGo87IvDktJTdyR0nAducPy9C1t2ul/y/9c3S0pgePfw+spwtOpZqqPOSC+pw7IL +fhdyFgymBwwbOM/JYrc/oJOlh0Hyt3BAd9i+FHzjqMB6juljatEPmsbS9Is6FARW +1O24zG71++IsWL1/T2sr92AkWCTOJu80kTrV44HQsvAEAtdbtz6SrGsSivnkBbA7 +kUlcsutT6vifR4buv5XAwAaf0lteERv0xwQ1KdJVXOTt6wIDAQABo4HJMIHGMB0G +A1UdDgQWBBTFe1i97doladL3WRaoszLAeydb9DAOBgNVHQ8BAf8EBAMCAQYwDwYD +VR0TAQH/BAUwAwEB/zCBgwYDVR0fBHwwejA8oDqgOIY2aHR0cDovL2NybC5jb21v +ZG9jYS5jb20vVHJ1c3RlZENlcnRpZmljYXRlU2VydmljZXMuY3JsMDqgOKA2hjRo +dHRwOi8vY3JsLmNvbW9kby5uZXQvVHJ1c3RlZENlcnRpZmljYXRlU2VydmljZXMu +Y3JsMA0GCSqGSIb3DQEBBQUAA4IBAQDIk4E7ibSvuIQSTI3S8NtwuleGFTQQuS9/ +HrCoiWChisJ3DFBKmwCL2Iv0QeLQg4pKHBQGsKNoBXAxMKdTmw7pSqBYaWcOrp32 +pSxBvzwGa+RZzG0Q8ZZvH9/0BAKkn0U+yNj6NkZEUD+Cl5EfKNsYEYwq5GWDVxIS +jBc/lDb+XbDABHcTuPQV1T84zJQ6VdCsmPW6AF/ghhmBeC8owH7TzEIK9a5QoNE+ +xqFx7D+gIIxmOom0jtTYsU0lR+4viMi14QVFwL4Ucd56/Y57fU0IlqUSc/Atyjcn +dBInTMu2l+nZrghtWjlA3QVHdWpaIbOjGM9O9y5Xt5hwXsjEeLBi +-----END CERTIFICATE----- +# Issuer: CN=UTN - DATACorp SGC O=The USERTRUST Network OU=http://www.usertrust.com +# Subject: CN=UTN - DATACorp SGC O=The USERTRUST Network OU=http://www.usertrust.com +# Label: "UTN DATACorp SGC Root CA" +# Serial: 91374294542884689855167577680241077609 +# MD5 Fingerprint: b3:a5:3e:77:21:6d:ac:4a:c0:c9:fb:d5:41:3d:ca:06 +# SHA1 Fingerprint: 58:11:9f:0e:12:82:87:ea:50:fd:d9:87:45:6f:4f:78:dc:fa:d6:d4 +# SHA256 Fingerprint: 85:fb:2f:91:dd:12:27:5a:01:45:b6:36:53:4f:84:02:4a:d6:8b:69:b8:ee:88:68:4f:f7:11:37:58:05:b3:48 -----BEGIN CERTIFICATE----- -MIICoTCCAgqgAwIBAgIBADANBgkqhkiG9w0BAQQFADCBizELMAkGA1UEBhMCWkEx -FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTEUMBIGA1UEBxMLRHVyYmFudmlsbGUxDzAN -BgNVBAoTBlRoYXd0ZTEdMBsGA1UECxMUVGhhd3RlIENlcnRpZmljYXRpb24xHzAd -BgNVBAMTFlRoYXd0ZSBUaW1lc3RhbXBpbmcgQ0EwHhcNOTcwMTAxMDAwMDAwWhcN -MjAxMjMxMjM1OTU5WjCBizELMAkGA1UEBhMCWkExFTATBgNVBAgTDFdlc3Rlcm4g -Q2FwZTEUMBIGA1UEBxMLRHVyYmFudmlsbGUxDzANBgNVBAoTBlRoYXd0ZTEdMBsG -A1UECxMUVGhhd3RlIENlcnRpZmljYXRpb24xHzAdBgNVBAMTFlRoYXd0ZSBUaW1l -c3RhbXBpbmcgQ0EwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYrWHhhRYZT -6jR7UZztsOYuGA7+4F+oJ9O0yeB8WU4WDnNUYMF/9p8u6TqFJBU820cEY8OexJQa -Wt9MevPZQx08EHp5JduQ/vBR5zDWQQD9nyjfeb6Uu522FOMjhdepQeBMpHmwKxqL -8vg7ij5FrHGSALSQQZj7X+36ty6K+Ig3AgMBAAGjEzARMA8GA1UdEwEB/wQFMAMB -Af8wDQYJKoZIhvcNAQEEBQADgYEAZ9viwuaHPUCDhjc1fR/OmsMMZiCouqoEiYbC -9RAIDb/LogWK0E02PvTX72nGXuSwlG9KuefeW4i2e9vjJ+V2w/A1wcu1J5szedyQ -pgCed/r8zSeUQhac0xxo7L9c3eWpexAKMnRUEzGLhQOEkbdYATAUOK8oyvyxUBkZ -CayJSdM= +MIIEXjCCA0agAwIBAgIQRL4Mi1AAIbQR0ypoBqmtaTANBgkqhkiG9w0BAQUFADCB +kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw +IFNHQzAeFw05OTA2MjQxODU3MjFaFw0xOTA2MjQxOTA2MzBaMIGTMQswCQYDVQQG +EwJVUzELMAkGA1UECBMCVVQxFzAVBgNVBAcTDlNhbHQgTGFrZSBDaXR5MR4wHAYD +VQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxITAfBgNVBAsTGGh0dHA6Ly93d3cu +dXNlcnRydXN0LmNvbTEbMBkGA1UEAxMSVVROIC0gREFUQUNvcnAgU0dDMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3+5YEKIrblXEjr8uRgnn4AgPLit6 +E5Qbvfa2gI5lBZMAHryv4g+OGQ0SR+ysraP6LnD43m77VkIVni5c7yPeIbkFdicZ +D0/Ww5y0vpQZY/KmEQrrU0icvvIpOxboGqBMpsn0GFlowHDyUwDAXlCCpVZvNvlK +4ESGoE1O1kduSUrLZ9emxAW5jh70/P/N5zbgnAVssjMiFdC04MwXwLLA9P4yPykq +lXvY8qdOD1R8oQ2AswkDwf9c3V6aPryuvEeKaq5xyh+xKrhfQgUL7EYw0XILyulW +bfXv33i+Ybqypa4ETLyorGkVl73v67SMvzX41MPRKA5cOp9wGDMgd8SirwIDAQAB +o4GrMIGoMAsGA1UdDwQEAwIBxjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRT +MtGzz3/64PGgXYVOktKeRR20TzA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vY3Js +LnVzZXJ0cnVzdC5jb20vVVROLURBVEFDb3JwU0dDLmNybDAqBgNVHSUEIzAhBggr +BgEFBQcDAQYKKwYBBAGCNwoDAwYJYIZIAYb4QgQBMA0GCSqGSIb3DQEBBQUAA4IB +AQAnNZcAiosovcYzMB4p/OL31ZjUQLtgyr+rFywJNn9Q+kHcrpY6CiM+iVnJowft +Gzet/Hy+UUla3joKVAgWRcKZsYfNjGjgaQPpxE6YsjuMFrMOoAyYUJuTqXAJyCyj +j98C5OBxOvG0I3KgqgHf35g+FFCgMSa9KOlaMCZ1+XtgHI3zzVAmbQQnmt/VDUVH +KWss5nbZqSl9Mt3JNjy9rjXxEZ4du5A/EkdOjtd+D2JzHVImOBwYSf0wdJrE5SIv +2MCN7ZF6TACPcn9d2t0bi0Vr591pl6jFVkwPDPafepE39peC4N1xaf92P2BNPM/3 +mfnGV/TJVTl4uix5yaaIK/QI -----END CERTIFICATE----- -thawte Primary Root CA -====================== +# Issuer: CN=UTN-USERFirst-Hardware O=The USERTRUST Network OU=http://www.usertrust.com +# Subject: CN=UTN-USERFirst-Hardware O=The USERTRUST Network OU=http://www.usertrust.com +# Label: "UTN USERFirst Hardware Root CA" +# Serial: 91374294542884704022267039221184531197 +# MD5 Fingerprint: 4c:56:41:e5:0d:bb:2b:e8:ca:a3:ed:18:08:ad:43:39 +# SHA1 Fingerprint: 04:83:ed:33:99:ac:36:08:05:87:22:ed:bc:5e:46:00:e3:be:f9:d7 +# SHA256 Fingerprint: 6e:a5:47:41:d0:04:66:7e:ed:1b:48:16:63:4a:a3:a7:9e:6e:4b:96:95:0f:82:79:da:fc:8d:9b:d8:81:21:37 +-----BEGIN CERTIFICATE----- +MIIEdDCCA1ygAwIBAgIQRL4Mi1AAJLQR0zYq/mUK/TANBgkqhkiG9w0BAQUFADCB +lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt +SGFyZHdhcmUwHhcNOTkwNzA5MTgxMDQyWhcNMTkwNzA5MTgxOTIyWjCBlzELMAkG +A1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0eTEe +MBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8v +d3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdh +cmUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCx98M4P7Sof885glFn +0G2f0v9Y8+efK+wNiVSZuTiZFvfgIXlIwrthdBKWHTxqctU8EGc6Oe0rE81m65UJ +M6Rsl7HoxuzBdXmcRl6Nq9Bq/bkqVRcQVLMZ8Jr28bFdtqdt++BxF2uiiPsA3/4a +MXcMmgF6sTLjKwEHOG7DpV4jvEWbe1DByTCP2+UretNb+zNAHqDVmBe8i4fDidNd +oI6yqqr2jmmIBsX6iSHzCJ1pLgkzmykNRg+MzEk0sGlRvfkGzWitZky8PqxhvQqI +DsjfPe58BEydCl5rkdbux+0ojatNh4lz0G6k0B4WixThdkQDf2Os5M1JnMWS9Ksy +oUhbAgMBAAGjgbkwgbYwCwYDVR0PBAQDAgHGMA8GA1UdEwEB/wQFMAMBAf8wHQYD +VR0OBBYEFKFyXyYbKJhDlV0HN9WFlp1L0sNFMEQGA1UdHwQ9MDswOaA3oDWGM2h0 +dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VVE4tVVNFUkZpcnN0LUhhcmR3YXJlLmNy +bDAxBgNVHSUEKjAoBggrBgEFBQcDAQYIKwYBBQUHAwUGCCsGAQUFBwMGBggrBgEF +BQcDBzANBgkqhkiG9w0BAQUFAAOCAQEARxkP3nTGmZev/K0oXnWO6y1n7k57K9cM +//bey1WiCuFMVGWTYGufEpytXoMs61quwOQt9ABjHbjAbPLPSbtNk28Gpgoiskli +CE7/yMgUsogWXecB5BKV5UU0s4tpvc+0hY91UZ59Ojg6FEgSxvunOxqNDYJAB+gE +CJChicsZUN/KHAG8HQQZexB2lzvukJDKxA4fFm517zP4029bHpbj4HR3dHuKom4t +3XbWOTCC8KucUvIqx69JXn7HaOWCgchqJ/kniCrVWFCVH/A7HFe7fRQ5YiuayZSS +KqMiDP+JJn1fIytH1xUdqWqeUQ0qUZ6B+dQ7XnASfxAynB67nfhmqA== +-----END CERTIFICATE----- + +# Issuer: CN=XRamp Global Certification Authority O=XRamp Security Services Inc OU=www.xrampsecurity.com +# Subject: CN=XRamp Global Certification Authority O=XRamp Security Services Inc OU=www.xrampsecurity.com +# Label: "XRamp Global CA Root" +# Serial: 107108908803651509692980124233745014957 +# MD5 Fingerprint: a1:0b:44:b3:ca:10:d8:00:6e:9d:0f:d8:0f:92:0a:d1 +# SHA1 Fingerprint: b8:01:86:d1:eb:9c:86:a5:41:04:cf:30:54:f3:4c:52:b7:e5:58:c6 +# SHA256 Fingerprint: ce:cd:dc:90:50:99:d8:da:df:c5:b1:d2:09:b7:37:cb:e2:c1:8c:fb:2c:10:c0:ff:0b:cf:0d:32:86:fc:1a:a2 +-----BEGIN CERTIFICATE----- +MIIEMDCCAxigAwIBAgIQUJRs7Bjq1ZxN1ZfvdY+grTANBgkqhkiG9w0BAQUFADCB +gjELMAkGA1UEBhMCVVMxHjAcBgNVBAsTFXd3dy54cmFtcHNlY3VyaXR5LmNvbTEk +MCIGA1UEChMbWFJhbXAgU2VjdXJpdHkgU2VydmljZXMgSW5jMS0wKwYDVQQDEyRY +UmFtcCBHbG9iYWwgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDQxMTAxMTcx +NDA0WhcNMzUwMTAxMDUzNzE5WjCBgjELMAkGA1UEBhMCVVMxHjAcBgNVBAsTFXd3 +dy54cmFtcHNlY3VyaXR5LmNvbTEkMCIGA1UEChMbWFJhbXAgU2VjdXJpdHkgU2Vy +dmljZXMgSW5jMS0wKwYDVQQDEyRYUmFtcCBHbG9iYWwgQ2VydGlmaWNhdGlvbiBB +dXRob3JpdHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYJB69FbS6 +38eMpSe2OAtp87ZOqCwuIR1cRN8hXX4jdP5efrRKt6atH67gBhbim1vZZ3RrXYCP +KZ2GG9mcDZhtdhAoWORlsH9KmHmf4MMxfoArtYzAQDsRhtDLooY2YKTVMIJt2W7Q +DxIEM5dfT2Fa8OT5kavnHTu86M/0ay00fOJIYRyO82FEzG+gSqmUsE3a56k0enI4 +qEHMPJQRfevIpoy3hsvKMzvZPTeL+3o+hiznc9cKV6xkmxnr9A8ECIqsAxcZZPRa +JSKNNCyy9mgdEm3Tih4U2sSPpuIjhdV6Db1q4Ons7Be7QhtnqiXtRYMh/MHJfNVi +PvryxS3T/dRlAgMBAAGjgZ8wgZwwEwYJKwYBBAGCNxQCBAYeBABDAEEwCwYDVR0P +BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFMZPoj0GY4QJnM5i5ASs +jVy16bYbMDYGA1UdHwQvMC0wK6ApoCeGJWh0dHA6Ly9jcmwueHJhbXBzZWN1cml0 +eS5jb20vWEdDQS5jcmwwEAYJKwYBBAGCNxUBBAMCAQEwDQYJKoZIhvcNAQEFBQAD +ggEBAJEVOQMBG2f7Shz5CmBbodpNl2L5JFMn14JkTpAuw0kbK5rc/Kh4ZzXxHfAR +vbdI4xD2Dd8/0sm2qlWkSLoC295ZLhVbO50WfUfXN+pfTXYSNrsf16GBBEYgoyxt +qZ4Bfj8pzgCT3/3JknOJiWSe5yvkHJEs0rnOfc5vMZnT5r7SHpDwCRR5XCOrTdLa +IR9NmXmd4c8nnxCbHIgNsIpkQTG4DmyQJKSbXHGPurt+HBvbaoAPIbzp26a3QPSy +i6mx5O+aGtA9aZnuqCij4Tyz8LIRnM98QObd50N9otg6tamN8jSZxNQQ4Qb9CYQQ +O+7ETPTsJ3xCwnR8gooJybQDJbw= +-----END CERTIFICATE----- + +# Issuer: O=The Go Daddy Group, Inc. OU=Go Daddy Class 2 Certification Authority +# Subject: O=The Go Daddy Group, Inc. OU=Go Daddy Class 2 Certification Authority +# Label: "Go Daddy Class 2 CA" +# Serial: 0 +# MD5 Fingerprint: 91:de:06:25:ab:da:fd:32:17:0c:bb:25:17:2a:84:67 +# SHA1 Fingerprint: 27:96:ba:e6:3f:18:01:e2:77:26:1b:a0:d7:77:70:02:8f:20:ee:e4 +# SHA256 Fingerprint: c3:84:6b:f2:4b:9e:93:ca:64:27:4c:0e:c6:7c:1e:cc:5e:02:4f:fc:ac:d2:d7:40:19:35:0e:81:fe:54:6a:e4 +-----BEGIN CERTIFICATE----- +MIIEADCCAuigAwIBAgIBADANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEh +MB8GA1UEChMYVGhlIEdvIERhZGR5IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBE +YWRkeSBDbGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA0MDYyOTE3 +MDYyMFoXDTM0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRo +ZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3Mg +MiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggEN +ADCCAQgCggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCA +PVYYYwhv2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6w +wdhFJ2+qN1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXi +EqITLdiOr18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMY +avx4A6lNf4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+ +YihfukEHU1jPEX44dMX4/7VpkI+EdOqXG68CAQOjgcAwgb0wHQYDVR0OBBYEFNLE +sNKR1EwRcbNhyz2h/t2oatTjMIGNBgNVHSMEgYUwgYKAFNLEsNKR1EwRcbNhyz2h +/t2oatTjoWekZTBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMYVGhlIEdvIERhZGR5 +IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQAD +ggEBADJL87LKPpH8EsahB4yOd6AzBhRckB4Y9wimPQoZ+YeAEW5p5JYXMP80kWNy +OO7MHAGjHZQopDH2esRU1/blMVgDoszOYtuURXO1v0XJJLXVggKtI3lpjbi2Tc7P +TMozI+gciKqdi0FuFskg5YmezTvacPd+mSYgFFQlq25zheabIZ0KbIIOqPjCDPoQ +HmyW74cNxA9hi63ugyuV+I6ShHI56yDqg+2DzZduCLzrTia2cyvk0/ZM/iZx4mER +dEr/VxqHD3VILs9RaRegAhJhldXRQLIQTO7ErBBDpqWeCtWVYpoNz4iCxTIM5Cuf +ReYNnyicsbkqWletNw+vHX/bvZ8= +-----END CERTIFICATE----- + +# Issuer: O=Starfield Technologies, Inc. OU=Starfield Class 2 Certification Authority +# Subject: O=Starfield Technologies, Inc. OU=Starfield Class 2 Certification Authority +# Label: "Starfield Class 2 CA" +# Serial: 0 +# MD5 Fingerprint: 32:4a:4b:bb:c8:63:69:9b:be:74:9a:c6:dd:1d:46:24 +# SHA1 Fingerprint: ad:7e:1c:28:b0:64:ef:8f:60:03:40:20:14:c3:d0:e3:37:0e:b5:8a +# SHA256 Fingerprint: 14:65:fa:20:53:97:b8:76:fa:a6:f0:a9:95:8e:55:90:e4:0f:cc:7f:aa:4f:b7:c2:c8:67:75:21:fb:5f:b6:58 +-----BEGIN CERTIFICATE----- +MIIEDzCCAvegAwIBAgIBADANBgkqhkiG9w0BAQUFADBoMQswCQYDVQQGEwJVUzEl +MCMGA1UEChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMp +U3RhcmZpZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDQw +NjI5MTczOTE2WhcNMzQwNjI5MTczOTE2WjBoMQswCQYDVQQGEwJVUzElMCMGA1UE +ChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMpU3RhcmZp +ZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggEgMA0GCSqGSIb3 +DQEBAQUAA4IBDQAwggEIAoIBAQC3Msj+6XGmBIWtDBFk385N78gDGIc/oav7PKaf +8MOh2tTYbitTkPskpD6E8J7oX+zlJ0T1KKY/e97gKvDIr1MvnsoFAZMej2YcOadN ++lq2cwQlZut3f+dZxkqZJRRU6ybH838Z1TBwj6+wRir/resp7defqgSHo9T5iaU0 +X9tDkYI22WY8sbi5gv2cOj4QyDvvBmVmepsZGD3/cVE8MC5fvj13c7JdBmzDI1aa +K4UmkhynArPkPw2vCHmCuDY96pzTNbO8acr1zJ3o/WSNF4Azbl5KXZnJHoe0nRrA +1W4TNSNe35tfPe/W93bC6j67eA0cQmdrBNj41tpvi/JEoAGrAgEDo4HFMIHCMB0G +A1UdDgQWBBS/X7fRzt0fhvRbVazc1xDCDqmI5zCBkgYDVR0jBIGKMIGHgBS/X7fR +zt0fhvRbVazc1xDCDqmI56FspGowaDELMAkGA1UEBhMCVVMxJTAjBgNVBAoTHFN0 +YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAsTKVN0YXJmaWVsZCBD +bGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQEFBQADggEBAAWdP4id0ckaVaGsafPzWdqbAYcaT1epoXkJKtv3 +L7IezMdeatiDh6GX70k1PncGQVhiv45YuApnP+yz3SFmH8lU+nLMPUxA2IGvd56D +eruix/U0F47ZEUD0/CwqTRV/p2JdLiXTAAsgGh1o+Re49L2L7ShZ3U0WixeDyLJl +xy16paq8U4Zt3VekyvggQQto8PT7dL5WXXp59fkdheMtlb71cZBDzI0fmgAKhynp +VSJYACPq4xJDKVtHCN2MQWplBqjlIapBtJUhlbl90TSrE9atvNziPTnNvT51cKEY +WQPJIrSPnNVeKtelttQKbfi3QBFGmh95DmK/D5fs4C8fF5Q= +-----END CERTIFICATE----- + +# Issuer: CN=StartCom Certification Authority O=StartCom Ltd. OU=Secure Digital Certificate Signing +# Subject: CN=StartCom Certification Authority O=StartCom Ltd. OU=Secure Digital Certificate Signing +# Label: "StartCom Certification Authority" +# Serial: 1 +# MD5 Fingerprint: 22:4d:8f:8a:fc:f7:35:c2:bb:57:34:90:7b:8b:22:16 +# SHA1 Fingerprint: 3e:2b:f7:f2:03:1b:96:f3:8c:e6:c4:d8:a8:5d:3e:2d:58:47:6a:0f +# SHA256 Fingerprint: c7:66:a9:be:f2:d4:07:1c:86:3a:31:aa:49:20:e8:13:b2:d1:98:60:8c:b7:b7:cf:e2:11:43:b8:36:df:09:ea +-----BEGIN CERTIFICATE----- +MIIHyTCCBbGgAwIBAgIBATANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQGEwJJTDEW +MBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0YWwg +Q2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwHhcNMDYwOTE3MTk0NjM2WhcNMzYwOTE3MTk0NjM2WjB9 +MQswCQYDVQQGEwJJTDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMi +U2VjdXJlIERpZ2l0YWwgQ2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3Rh +cnRDb20gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUA +A4ICDwAwggIKAoICAQDBiNsJvGxGfHiflXu1M5DycmLWwTYgIiRezul38kMKogZk +pMyONvg45iPwbm2xPN1yo4UcodM9tDMr0y+v/uqwQVlntsQGfQqedIXWeUyAN3rf +OQVSWff0G0ZDpNKFhdLDcfN1YjS6LIp/Ho/u7TTQEceWzVI9ujPW3U3eCztKS5/C +Ji/6tRYccjV3yjxd5srhJosaNnZcAdt0FCX+7bWgiA/deMotHweXMAEtcnn6RtYT +Kqi5pquDSR3l8u/d5AGOGAqPY1MWhWKpDhk6zLVmpsJrdAfkK+F2PrRt2PZE4XNi +HzvEvqBTViVsUQn3qqvKv3b9bZvzndu/PWa8DFaqr5hIlTpL36dYUNk4dalb6kMM +Av+Z6+hsTXBbKWWc3apdzK8BMewM69KN6Oqce+Zu9ydmDBpI125C4z/eIT574Q1w ++2OqqGwaVLRcJXrJosmLFqa7LH4XXgVNWG4SHQHuEhANxjJ/GP/89PrNbpHoNkm+ +Gkhpi8KWTRoSsmkXwQqQ1vp5Iki/untp+HDH+no32NgN0nZPV/+Qt+OR0t3vwmC3 +Zzrd/qqc8NSLf3Iizsafl7b4r4qgEKjZ+xjGtrVcUjyJthkqcwEKDwOzEmDyei+B +26Nu/yYwl/WL3YlXtq09s68rxbd2AvCl1iuahhQqcvbjM4xdCUsT37uMdBNSSwID +AQABo4ICUjCCAk4wDAYDVR0TBAUwAwEB/zALBgNVHQ8EBAMCAa4wHQYDVR0OBBYE +FE4L7xqkQFulF2mHMMo0aEPQQa7yMGQGA1UdHwRdMFswLKAqoCiGJmh0dHA6Ly9j +ZXJ0LnN0YXJ0Y29tLm9yZy9zZnNjYS1jcmwuY3JsMCugKaAnhiVodHRwOi8vY3Js +LnN0YXJ0Y29tLm9yZy9zZnNjYS1jcmwuY3JsMIIBXQYDVR0gBIIBVDCCAVAwggFM +BgsrBgEEAYG1NwEBATCCATswLwYIKwYBBQUHAgEWI2h0dHA6Ly9jZXJ0LnN0YXJ0 +Y29tLm9yZy9wb2xpY3kucGRmMDUGCCsGAQUFBwIBFilodHRwOi8vY2VydC5zdGFy +dGNvbS5vcmcvaW50ZXJtZWRpYXRlLnBkZjCB0AYIKwYBBQUHAgIwgcMwJxYgU3Rh +cnQgQ29tbWVyY2lhbCAoU3RhcnRDb20pIEx0ZC4wAwIBARqBl0xpbWl0ZWQgTGlh +YmlsaXR5LCByZWFkIHRoZSBzZWN0aW9uICpMZWdhbCBMaW1pdGF0aW9ucyogb2Yg +dGhlIFN0YXJ0Q29tIENlcnRpZmljYXRpb24gQXV0aG9yaXR5IFBvbGljeSBhdmFp +bGFibGUgYXQgaHR0cDovL2NlcnQuc3RhcnRjb20ub3JnL3BvbGljeS5wZGYwEQYJ +YIZIAYb4QgEBBAQDAgAHMDgGCWCGSAGG+EIBDQQrFilTdGFydENvbSBGcmVlIFNT +TCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTANBgkqhkiG9w0BAQUFAAOCAgEAFmyZ +9GYMNPXQhV59CuzaEE44HF7fpiUFS5Eyweg78T3dRAlbB0mKKctmArexmvclmAk8 +jhvh3TaHK0u7aNM5Zj2gJsfyOZEdUauCe37Vzlrk4gNXcGmXCPleWKYK34wGmkUW +FjgKXlf2Ysd6AgXmvB618p70qSmD+LIU424oh0TDkBreOKk8rENNZEXO3SipXPJz +ewT4F+irsfMuXGRuczE6Eri8sxHkfY+BUZo7jYn0TZNmezwD7dOaHZrzZVD1oNB1 +ny+v8OqCQ5j4aZyJecRDjkZy42Q2Eq/3JR44iZB3fsNrarnDy0RLrHiQi+fHLB5L +EUTINFInzQpdn4XBidUaePKVEFMy3YCEZnXZtWgo+2EuvoSoOMCZEoalHmdkrQYu +L6lwhceWD3yJZfWOQ1QOq92lgDmUYMA0yZZwLKMS9R9Ie70cfmu3nZD0Ijuu+Pwq +yvqCUqDvr0tVk+vBtfAii6w0TiYiBKGHLHVKt+V9E9e4DGTANtLJL4YSjCMJwRuC +O3NJo2pXh5Tl1njFmUNj403gdy3hZZlyaQQaRwnmDwFWJPsfvw55qVguucQJAX6V +um0ABj6y6koQOdjQK/W/7HW/lwLFCRsI3FU34oH7N4RDYiDK51ZLZer+bMEkkySh +NOsF/5oirpt9P/FlUQqmMGqz9IgcgA38corog14= +-----END CERTIFICATE----- +# Issuer: CN=DigiCert Assured ID Root CA O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Assured ID Root CA O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Assured ID Root CA" +# Serial: 17154717934120587862167794914071425081 +# MD5 Fingerprint: 87:ce:0b:7b:2a:0e:49:00:e1:58:71:9b:37:a8:93:72 +# SHA1 Fingerprint: 05:63:b8:63:0d:62:d7:5a:bb:c8:ab:1e:4b:df:b5:a8:99:b2:4d:43 +# SHA256 Fingerprint: 3e:90:99:b5:01:5e:8f:48:6c:00:bc:ea:9d:11:1e:e7:21:fa:ba:35:5a:89:bc:f1:df:69:56:1e:3d:c6:32:5c +-----BEGIN CERTIFICATE----- +MIIDtzCCAp+gAwIBAgIQDOfg5RfYRv6P5WD8G/AwOTANBgkqhkiG9w0BAQUFADBl +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv +b3QgQ0EwHhcNMDYxMTEwMDAwMDAwWhcNMzExMTEwMDAwMDAwWjBlMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl +cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgQ0EwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtDhXO5EOAXLGH87dg+XESpa7c +JpSIqvTO9SA5KFhgDPiA2qkVlTJhPLWxKISKityfCgyDF3qPkKyK53lTXDGEKvYP +mDI2dsze3Tyoou9q+yHyUmHfnyDXH+Kx2f4YZNISW1/5WBg1vEfNoTb5a3/UsDg+ +wRvDjDPZ2C8Y/igPs6eD1sNuRMBhNZYW/lmci3Zt1/GiSw0r/wty2p5g0I6QNcZ4 +VYcgoc/lbQrISXwxmDNsIumH0DJaoroTghHtORedmTpyoeb6pNnVFzF1roV9Iq4/ +AUaG9ih5yLHa5FcXxH4cDrC0kqZWs72yl+2qp/C3xag/lRbQ/6GW6whfGHdPAgMB +AAGjYzBhMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW +BBRF66Kv9JLLgjEtUYunpyGd823IDzAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYun +pyGd823IDzANBgkqhkiG9w0BAQUFAAOCAQEAog683+Lt8ONyc3pklL/3cmbYMuRC +dWKuh+vy1dneVrOfzM4UKLkNl2BcEkxY5NM9g0lFWJc1aRqoR+pWxnmrEthngYTf +fwk8lOa4JiwgvT2zKIn3X/8i4peEH+ll74fg38FnSbNd67IJKusm7Xi+fT8r87cm +NW1fiQG2SVufAQWbqz0lwcy2f8Lxb4bG+mRo64EtlOtCt/qMHt1i8b5QZ7dsvfPx +H2sMNgcWfzd8qVttevESRmCD1ycEvkvOl77DZypoEd+A5wwzZr8TDRRu838fYxAe ++o0bJW1sj6W3YQGx0qMmoRBxna3iw/nDmVG3KwcIzi7mULKn+gpFL6Lw8g== +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert Global Root CA O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Global Root CA O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Global Root CA" +# Serial: 10944719598952040374951832963794454346 +# MD5 Fingerprint: 79:e4:a9:84:0d:7d:3a:96:d7:c0:4f:e2:43:4c:89:2e +# SHA1 Fingerprint: a8:98:5d:3a:65:e5:e5:c4:b2:d7:d6:6d:40:c6:dd:2f:b1:9c:54:36 +# SHA256 Fingerprint: 43:48:a0:e9:44:4c:78:cb:26:5e:05:8d:5e:89:44:b4:d8:4f:96:62:bd:26:db:25:7f:89:34:a4:43:c7:01:61 +-----BEGIN CERTIFICATE----- +MIIDrzCCApegAwIBAgIQCDvgVpBCRrGhdWrJWZHHSjANBgkqhkiG9w0BAQUFADBh +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBD +QTAeFw0wNjExMTAwMDAwMDBaFw0zMTExMTAwMDAwMDBaMGExCzAJBgNVBAYTAlVT +MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j +b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IENBMIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4jvhEXLeqKTTo1eqUKKPC3eQyaKl7hLOllsB +CSDMAZOnTjC3U/dDxGkAV53ijSLdhwZAAIEJzs4bg7/fzTtxRuLWZscFs3YnFo97 +nh6Vfe63SKMI2tavegw5BmV/Sl0fvBf4q77uKNd0f3p4mVmFaG5cIzJLv07A6Fpt +43C/dxC//AH2hdmoRBBYMql1GNXRor5H4idq9Joz+EkIYIvUX7Q6hL+hqkpMfT7P +T19sdl6gSzeRntwi5m3OFBqOasv+zbMUZBfHWymeMr/y7vrTC0LUq7dBMtoM1O/4 +gdW7jVg/tRvoSSiicNoxBN33shbyTApOB6jtSj1etX+jkMOvJwIDAQABo2MwYTAO +BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUA95QNVbR +TLtm8KPiGxvDl7I90VUwHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUw +DQYJKoZIhvcNAQEFBQADggEBAMucN6pIExIK+t1EnE9SsPTfrgT1eXkIoyQY/Esr +hMAtudXH/vTBH1jLuG2cenTnmCmrEbXjcKChzUyImZOMkXDiqw8cvpOp/2PV5Adg +06O/nVsJ8dWO41P0jmP6P6fbtGbfYmbW0W5BjfIttep3Sp+dWOIrWcBAI+0tKIJF +PnlUkiaY4IBIqDfv8NZ5YBberOgOzW6sRBc4L0na4UU+Krk2U886UAb3LujEV0ls +YSEY1QSteDwsOoBrp+uvFRTp2InBuThs4pFsiv9kuXclVzDAGySj4dzp30d8tbQk +CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4= +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert High Assurance EV Root CA O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert High Assurance EV Root CA O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert High Assurance EV Root CA" +# Serial: 3553400076410547919724730734378100087 +# MD5 Fingerprint: d4:74:de:57:5c:39:b2:d3:9c:85:83:c5:c0:65:49:8a +# SHA1 Fingerprint: 5f:b7:ee:06:33:e2:59:db:ad:0c:4c:9a:e6:d3:8f:1a:61:c7:dc:25 +# SHA256 Fingerprint: 74:31:e5:f4:c3:c1:ce:46:90:77:4f:0b:61:e0:54:40:88:3b:a9:a0:1e:d0:0b:a6:ab:d7:80:6e:d3:b1:18:cf +-----BEGIN CERTIFICATE----- +MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBs +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j +ZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAwMFoXDTMxMTExMDAwMDAwMFowbDEL +MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3 +LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug +RVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm ++9S75S0tMqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTW +PNt0OKRKzE0lgvdKpVMSOO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEM +xChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFB +Ik5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQNAQTXKFx01p8VdteZOE3 +hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUeh10aUAsg +EsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQF +MAMBAf8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaA +FLE+w2kD+L9HAdSYJhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3Nec +nzyIZgYIVyHbIUf4KmeqvxgydkAQV8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6z +eM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFpmyPInngiK3BD41VHMWEZ71jF +hS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkKmNEVX58Svnw2 +Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe +vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep ++OkuE6N36B9K +-----END CERTIFICATE----- + +# Issuer: CN=GeoTrust Primary Certification Authority O=GeoTrust Inc. +# Subject: CN=GeoTrust Primary Certification Authority O=GeoTrust Inc. +# Label: "GeoTrust Primary Certification Authority" +# Serial: 32798226551256963324313806436981982369 +# MD5 Fingerprint: 02:26:c3:01:5e:08:30:37:43:a9:d0:7d:cf:37:e6:bf +# SHA1 Fingerprint: 32:3c:11:8e:1b:f7:b8:b6:52:54:e2:e2:10:0d:d6:02:90:37:f0:96 +# SHA256 Fingerprint: 37:d5:10:06:c5:12:ea:ab:62:64:21:f1:ec:8c:92:01:3f:c5:f8:2a:e9:8e:e5:33:eb:46:19:b8:de:b4:d0:6c +-----BEGIN CERTIFICATE----- +MIIDfDCCAmSgAwIBAgIQGKy1av1pthU6Y2yv2vrEoTANBgkqhkiG9w0BAQUFADBY +MQswCQYDVQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjExMC8GA1UEAxMo +R2VvVHJ1c3QgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wNjEx +MjcwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMFgxCzAJBgNVBAYTAlVTMRYwFAYDVQQK +Ew1HZW9UcnVzdCBJbmMuMTEwLwYDVQQDEyhHZW9UcnVzdCBQcmltYXJ5IENlcnRp +ZmljYXRpb24gQXV0aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEAvrgVe//UfH1nrYNke8hCUy3f9oQIIGHWAVlqnEQRr+92/ZV+zmEwu3qDXwK9 +AWbK7hWNb6EwnL2hhZ6UOvNWiAAxz9juapYC2e0DjPt1befquFUWBRaa9OBesYjA +ZIVcFU2Ix7e64HXprQU9nceJSOC7KMgD4TCTZF5SwFlwIjVXiIrxlQqD17wxcwE0 +7e9GceBrAqg1cmuXm2bgyxx5X9gaBGgeRwLmnWDiNpcB3841kt++Z8dtd1k7j53W +kBWUvEI0EME5+bEnPn7WinXFsq+W06Lem+SYvn3h6YGttm/81w7a4DSwDRp35+MI +mO9Y+pyEtzavwt+s0vQQBnBxNQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4G +A1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQULNVQQZcVi/CPNmFbSvtr2ZnJM5IwDQYJ +KoZIhvcNAQEFBQADggEBAFpwfyzdtzRP9YZRqSa+S7iq8XEN3GHHoOo0Hnp3DwQ1 +6CePbJC/kRYkRj5KTs4rFtULUh38H2eiAkUxT87z+gOneZ1TatnaYzr4gNfTmeGl +4b7UVXGYNTq+k+qurUKykG/g/CFNNWMziUnWm07Kx+dOCQD32sfvmWKZd7aVIl6K +oKv0uHiYyjgZmclynnjNS6yvGaBzEi38wkG6gZHaFloxt/m0cYASSJlyc1pZU8Fj +UjPtp8nSOQJw+uCxQmYpqptR7TBUIhRf2asdweSU8Pj1K/fqynhG1riR/aYNKxoU +AT6A8EKglQdebc3MS6RFjasS6LPeWuWgfOgPIh1a6Vk= +-----END CERTIFICATE----- + +# Issuer: CN=thawte Primary Root CA O=thawte, Inc. OU=Certification Services Division/(c) 2006 thawte, Inc. - For authorized use only +# Subject: CN=thawte Primary Root CA O=thawte, Inc. OU=Certification Services Division/(c) 2006 thawte, Inc. - For authorized use only +# Label: "thawte Primary Root CA" +# Serial: 69529181992039203566298953787712940909 +# MD5 Fingerprint: 8c:ca:dc:0b:22:ce:f5:be:72:ac:41:1a:11:a8:d8:12 +# SHA1 Fingerprint: 91:c6:d6:ee:3e:8a:c8:63:84:e5:48:c2:99:29:5c:75:6c:81:7b:81 +# SHA256 Fingerprint: 8d:72:2f:81:a9:c1:13:c0:79:1d:f1:36:a2:96:6d:b2:6c:95:0a:97:1d:b4:6b:41:99:f4:ea:54:b7:8b:fb:9f -----BEGIN CERTIFICATE----- MIIEIDCCAwigAwIBAgIQNE7VVyDV7exJ9C/ON9srbTANBgkqhkiG9w0BAQUFADCB qTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf @@ -568,9 +1366,13 @@ LHbTY5xZ3Y+m4Q6gLkH3LpVHz7z9M/P2C2F+fpErgUfCJzDupxBdN49cOSvkBPB7 jVaMaA== -----END CERTIFICATE----- -VeriSign Class 3 Public Primary Certification Authority - G5 -============================================================ - +# Issuer: CN=VeriSign Class 3 Public Primary Certification Authority - G5 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2006 VeriSign, Inc. - For authorized use only +# Subject: CN=VeriSign Class 3 Public Primary Certification Authority - G5 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2006 VeriSign, Inc. - For authorized use only +# Label: "VeriSign Class 3 Public Primary Certification Authority - G5" +# Serial: 33037644167568058970164719475676101450 +# MD5 Fingerprint: cb:17:e4:31:67:3e:e2:09:fe:45:57:93:f3:0a:fa:1c +# SHA1 Fingerprint: 4e:b6:d5:78:49:9b:1c:cf:5f:58:1e:ad:56:be:3d:9b:67:44:a5:e5 +# SHA256 Fingerprint: 9a:cf:ab:7e:43:c8:d8:80:d0:6b:26:2a:94:de:ee:e4:b4:65:99:89:c3:d0:ca:f1:9b:af:64:05:e4:1a:b7:df -----BEGIN CERTIFICATE----- MIIE0zCCA7ugAwIBAgIQGNrRniZ96LtKIVjNzGs7SjANBgkqhkiG9w0BAQUFADCB yjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL @@ -600,140 +1402,782 @@ WE9gyn6CagsCqiUXObXbf+eEZSqVir2G3l6BFoMtEMze/aiCKm0oHw0LxOXnGiYZ hnacRHr2lVz2XTIIM6RUthg/aFzyQkqFOFSDX9HoLPKsEdao7WNq -----END CERTIFICATE----- -Entrust.net Secure Server Certification Authority -================================================= +# Issuer: CN=COMODO Certification Authority O=COMODO CA Limited +# Subject: CN=COMODO Certification Authority O=COMODO CA Limited +# Label: "COMODO Certification Authority" +# Serial: 104350513648249232941998508985834464573 +# MD5 Fingerprint: 5c:48:dc:f7:42:72:ec:56:94:6d:1c:cc:71:35:80:75 +# SHA1 Fingerprint: 66:31:bf:9e:f7:4f:9e:b6:c9:d5:a6:0c:ba:6a:be:d1:f7:bd:ef:7b +# SHA256 Fingerprint: 0c:2c:d6:3d:f7:80:6f:a3:99:ed:e8:09:11:6b:57:5b:f8:79:89:f0:65:18:f9:80:8c:86:05:03:17:8b:af:66 +-----BEGIN CERTIFICATE----- +MIIEHTCCAwWgAwIBAgIQToEtioJl4AsC7j41AkblPTANBgkqhkiG9w0BAQUFADCB +gTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxJzAlBgNV +BAMTHkNPTU9ETyBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wNjEyMDEwMDAw +MDBaFw0yOTEyMzEyMzU5NTlaMIGBMQswCQYDVQQGEwJHQjEbMBkGA1UECBMSR3Jl +YXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHEwdTYWxmb3JkMRowGAYDVQQKExFDT01P +RE8gQ0EgTGltaXRlZDEnMCUGA1UEAxMeQ09NT0RPIENlcnRpZmljYXRpb24gQXV0 +aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0ECLi3LjkRv3 +UcEbVASY06m/weaKXTuH+7uIzg3jLz8GlvCiKVCZrts7oVewdFFxze1CkU1B/qnI +2GqGd0S7WWaXUF601CxwRM/aN5VCaTwwxHGzUvAhTaHYujl8HJ6jJJ3ygxaYqhZ8 +Q5sVW7euNJH+1GImGEaaP+vB+fGQV+useg2L23IwambV4EajcNxo2f8ESIl33rXp ++2dtQem8Ob0y2WIC8bGoPW43nOIv4tOiJovGuFVDiOEjPqXSJDlqR6sA1KGzqSX+ +DT+nHbrTUcELpNqsOO9VUCQFZUaTNE8tja3G1CEZ0o7KBWFxB3NH5YoZEr0ETc5O +nKVIrLsm9wIDAQABo4GOMIGLMB0GA1UdDgQWBBQLWOWLxkwVN6RAqTCpIb5HNlpW +/zAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zBJBgNVHR8EQjBAMD6g +PKA6hjhodHRwOi8vY3JsLmNvbW9kb2NhLmNvbS9DT01PRE9DZXJ0aWZpY2F0aW9u +QXV0aG9yaXR5LmNybDANBgkqhkiG9w0BAQUFAAOCAQEAPpiem/Yb6dc5t3iuHXIY +SdOH5EOC6z/JqvWote9VfCFSZfnVDeFs9D6Mk3ORLgLETgdxb8CPOGEIqB6BCsAv +IC9Bi5HcSEW88cbeunZrM8gALTFGTO3nnc+IlP8zwFboJIYmuNg4ON8qa90SzMc/ +RxdMosIGlgnW2/4/PEZB31jiVg88O8EckzXZOFKs7sjsLjBOlDW0JB9LeGna8gI4 +zJVSk/BwJVmcIGfE7vmLV2H0knZ9P4SNVbfo5azV8fUZVqZa+5Acr5Pr5RzUZ5dd +BA6+C4OmF4O5MBKgxTMVBbkN+8cFduPYSo38NBejxiEovjBFMR7HeL5YYTisO+IB +ZQ== +-----END CERTIFICATE----- +# Issuer: CN=Network Solutions Certificate Authority O=Network Solutions L.L.C. +# Subject: CN=Network Solutions Certificate Authority O=Network Solutions L.L.C. +# Label: "Network Solutions Certificate Authority" +# Serial: 116697915152937497490437556386812487904 +# MD5 Fingerprint: d3:f3:a6:16:c0:fa:6b:1d:59:b1:2d:96:4d:0e:11:2e +# SHA1 Fingerprint: 74:f8:a3:c3:ef:e7:b3:90:06:4b:83:90:3c:21:64:60:20:e5:df:ce +# SHA256 Fingerprint: 15:f0:ba:00:a3:ac:7a:f3:ac:88:4c:07:2b:10:11:a0:77:bd:77:c0:97:f4:01:64:b2:f8:59:8a:bd:83:86:0c -----BEGIN CERTIFICATE----- -MIIE2DCCBEGgAwIBAgIEN0rSQzANBgkqhkiG9w0BAQUFADCBwzELMAkGA1UEBhMC -VVMxFDASBgNVBAoTC0VudHJ1c3QubmV0MTswOQYDVQQLEzJ3d3cuZW50cnVzdC5u -ZXQvQ1BTIGluY29ycC4gYnkgcmVmLiAobGltaXRzIGxpYWIuKTElMCMGA1UECxMc -KGMpIDE5OTkgRW50cnVzdC5uZXQgTGltaXRlZDE6MDgGA1UEAxMxRW50cnVzdC5u -ZXQgU2VjdXJlIFNlcnZlciBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw05OTA1 -MjUxNjA5NDBaFw0xOTA1MjUxNjM5NDBaMIHDMQswCQYDVQQGEwJVUzEUMBIGA1UE -ChMLRW50cnVzdC5uZXQxOzA5BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5j -b3JwLiBieSByZWYuIChsaW1pdHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBF -bnRydXN0Lm5ldCBMaW1pdGVkMTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUg -U2VydmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGdMA0GCSqGSIb3DQEBAQUA -A4GLADCBhwKBgQDNKIM0VBuJ8w+vN5Ex/68xYMmo6LIQaO2f55M28Qpku0f1BBc/ -I0dNxScZgSYMVHINiC3ZH5oSn7yzcdOAGT9HZnuMNSjSuQrfJNqc1lB5gXpa0zf3 -wkrYKZImZNHkmGw6AIr1NJtl+O3jEP/9uElY3KDegjlrgbEWGWG5VLbmQwIBA6OC -AdcwggHTMBEGCWCGSAGG+EIBAQQEAwIABzCCARkGA1UdHwSCARAwggEMMIHeoIHb -oIHYpIHVMIHSMQswCQYDVQQGEwJVUzEUMBIGA1UEChMLRW50cnVzdC5uZXQxOzA5 -BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5jb3JwLiBieSByZWYuIChsaW1p -dHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBFbnRydXN0Lm5ldCBMaW1pdGVk -MTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUgU2VydmVyIENlcnRpZmljYXRp -b24gQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMCmgJ6AlhiNodHRwOi8vd3d3LmVu -dHJ1c3QubmV0L0NSTC9uZXQxLmNybDArBgNVHRAEJDAigA8xOTk5MDUyNTE2MDk0 -MFqBDzIwMTkwNTI1MTYwOTQwWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAU8Bdi -E1U9s/8KAGv7UISX8+1i0BowHQYDVR0OBBYEFPAXYhNVPbP/CgBr+1CEl/PtYtAa -MAwGA1UdEwQFMAMBAf8wGQYJKoZIhvZ9B0EABAwwChsEVjQuMAMCBJAwDQYJKoZI -hvcNAQEFBQADgYEAkNwwAvpkdMKnCqV8IY00F6j7Rw7/JXyNEwr75Ji174z4xRAN -95K+8cPV1ZVqBLssziY2ZcgxxufuP+NXdYR6Ee9GTxj005i7qIcyunL2POI9n9cd -2cNgQ4xYDiKWL2KjLB+6rQXvqzJ4h6BUcxm1XAX5Uj5tLUUL9wqT6u0G+bI= +MIID5jCCAs6gAwIBAgIQV8szb8JcFuZHFhfjkDFo4DANBgkqhkiG9w0BAQUFADBi +MQswCQYDVQQGEwJVUzEhMB8GA1UEChMYTmV0d29yayBTb2x1dGlvbnMgTC5MLkMu +MTAwLgYDVQQDEydOZXR3b3JrIFNvbHV0aW9ucyBDZXJ0aWZpY2F0ZSBBdXRob3Jp +dHkwHhcNMDYxMjAxMDAwMDAwWhcNMjkxMjMxMjM1OTU5WjBiMQswCQYDVQQGEwJV +UzEhMB8GA1UEChMYTmV0d29yayBTb2x1dGlvbnMgTC5MLkMuMTAwLgYDVQQDEydO +ZXR3b3JrIFNvbHV0aW9ucyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDkvH6SMG3G2I4rC7xGzuAnlt7e+foS0zwz +c7MEL7xxjOWftiJgPl9dzgn/ggwbmlFQGiaJ3dVhXRncEg8tCqJDXRfQNJIg6nPP +OCwGJgl6cvf6UDL4wpPTaaIjzkGxzOTVHzbRijr4jGPiFFlp7Q3Tf2vouAPlT2rl +mGNpSAW+Lv8ztumXWWn4Zxmuk2GWRBXTcrA/vGp97Eh/jcOrqnErU2lBUzS1sLnF +BgrEsEX1QV1uiUV7PTsmjHTC5dLRfbIR1PtYMiKagMnc/Qzpf14Dl847ABSHJ3A4 +qY5usyd2mFHgBeMhqxrVhSI8KbWaFsWAqPS7azCPL0YCorEMIuDTAgMBAAGjgZcw +gZQwHQYDVR0OBBYEFCEwyfsA106Y2oeqKtCnLrFAMadMMA4GA1UdDwEB/wQEAwIB +BjAPBgNVHRMBAf8EBTADAQH/MFIGA1UdHwRLMEkwR6BFoEOGQWh0dHA6Ly9jcmwu +bmV0c29sc3NsLmNvbS9OZXR3b3JrU29sdXRpb25zQ2VydGlmaWNhdGVBdXRob3Jp +dHkuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQC7rkvnt1frf6ott3NHhWrB5KUd5Oc8 +6fRZZXe1eltajSU24HqXLjjAV2CDmAaDn7l2em5Q4LqILPxFzBiwmZVRDuwduIj/ +h1AcgsLj4DKAv6ALR8jDMe+ZZzKATxcheQxpXN5eNK4CtSbqUN9/GGUsyfJj4akH +/nxxH2szJGoeBfcFaMBqEssuXmHLrijTfsK0ZpEmXzwuJF/LWA/rKOyvEZbz3Htv +wKeI8lN3s2Berq4o2jUsbzRF0ybh3uxbTydrFny9RAQYgrOJeRcQcT16ohZO9QHN +pGxlaKFJdlxDydi8NmdspZS11My5vWo1ViHe2MPr+8ukYEywVaCge1ey -----END CERTIFICATE----- -Go Daddy Certification Authority Root Certificate Bundle -======================================================== - ------BEGIN CERTIFICATE----- -MIIE3jCCA8agAwIBAgICAwEwDQYJKoZIhvcNAQEFBQAwYzELMAkGA1UEBhMCVVMx -ITAfBgNVBAoTGFRoZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28g -RGFkZHkgQ2xhc3MgMiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wNjExMTYw -MTU0MzdaFw0yNjExMTYwMTU0MzdaMIHKMQswCQYDVQQGEwJVUzEQMA4GA1UECBMH -QXJpem9uYTETMBEGA1UEBxMKU2NvdHRzZGFsZTEaMBgGA1UEChMRR29EYWRkeS5j -b20sIEluYy4xMzAxBgNVBAsTKmh0dHA6Ly9jZXJ0aWZpY2F0ZXMuZ29kYWRkeS5j -b20vcmVwb3NpdG9yeTEwMC4GA1UEAxMnR28gRGFkZHkgU2VjdXJlIENlcnRpZmlj -YXRpb24gQXV0aG9yaXR5MREwDwYDVQQFEwgwNzk2OTI4NzCCASIwDQYJKoZIhvcN -AQEBBQADggEPADCCAQoCggEBAMQt1RWMnCZM7DI161+4WQFapmGBWTtwY6vj3D3H -KrjJM9N55DrtPDAjhI6zMBS2sofDPZVUBJ7fmd0LJR4h3mUpfjWoqVTr9vcyOdQm -VZWt7/v+WIbXnvQAjYwqDL1CBM6nPwT27oDyqu9SoWlm2r4arV3aLGbqGmu75RpR -SgAvSMeYddi5Kcju+GZtCpyz8/x4fKL4o/K1w/O5epHBp+YlLpyo7RJlbmr2EkRT -cDCVw5wrWCs9CHRK8r5RsL+H0EwnWGu1NcWdrxcx+AuP7q2BNgWJCJjPOq8lh8BJ -6qf9Z/dFjpfMFDniNoW1fho3/Rb2cRGadDAW/hOUoz+EDU8CAwEAAaOCATIwggEu -MB0GA1UdDgQWBBT9rGEyk2xF1uLuhV+auud2mWjM5zAfBgNVHSMEGDAWgBTSxLDS -kdRMEXGzYcs9of7dqGrU4zASBgNVHRMBAf8ECDAGAQH/AgEAMDMGCCsGAQUFBwEB -BCcwJTAjBggrBgEFBQcwAYYXaHR0cDovL29jc3AuZ29kYWRkeS5jb20wRgYDVR0f -BD8wPTA7oDmgN4Y1aHR0cDovL2NlcnRpZmljYXRlcy5nb2RhZGR5LmNvbS9yZXBv -c2l0b3J5L2dkcm9vdC5jcmwwSwYDVR0gBEQwQjBABgRVHSAAMDgwNgYIKwYBBQUH -AgEWKmh0dHA6Ly9jZXJ0aWZpY2F0ZXMuZ29kYWRkeS5jb20vcmVwb3NpdG9yeTAO -BgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQEFBQADggEBANKGwOy9+aG2Z+5mC6IG -OgRQjhVyrEp0lVPLN8tESe8HkGsz2ZbwlFalEzAFPIUyIXvJxwqoJKSQ3kbTJSMU -A2fCENZvD117esyfxVgqwcSeIaha86ykRvOe5GPLL5CkKSkB2XIsKd83ASe8T+5o -0yGPwLPk9Qnt0hCqU7S+8MxZC9Y7lhyVJEnfzuz9p0iRFEUOOjZv2kWzRaJBydTX -RE4+uXR21aITVSzGh6O1mawGhId/dQb8vxRMDsxuxN89txJx9OjxUUAiKEngHUuH -qDTMBqLdElrRhjZkAzVvb3du6/KFUJheqwNTrZEjYx8WnM25sgVjOuH0aBsXBTWV -U+4= ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIE+zCCBGSgAwIBAgICAQ0wDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1Zh -bGlDZXJ0IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIElu -Yy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENsYXNzIDIgUG9saWN5IFZhbGlkYXRpb24g -QXV0aG9yaXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAe -BgkqhkiG9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTA0MDYyOTE3MDYyMFoX -DTI0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRoZSBHbyBE -YWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3MgMiBDZXJ0 -aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggENADCCAQgC -ggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCAPVYYYwhv -2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6wwdhFJ2+q -N1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXiEqITLdiO -r18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMYavx4A6lN -f4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+YihfukEH -U1jPEX44dMX4/7VpkI+EdOqXG68CAQOjggHhMIIB3TAdBgNVHQ4EFgQU0sSw0pHU -TBFxs2HLPaH+3ahq1OMwgdIGA1UdIwSByjCBx6GBwaSBvjCBuzEkMCIGA1UEBxMb -VmFsaUNlcnQgVmFsaWRhdGlvbiBOZXR3b3JrMRcwFQYDVQQKEw5WYWxpQ2VydCwg -SW5jLjE1MDMGA1UECxMsVmFsaUNlcnQgQ2xhc3MgMiBQb2xpY3kgVmFsaWRhdGlv -biBBdXRob3JpdHkxITAfBgNVBAMTGGh0dHA6Ly93d3cudmFsaWNlcnQuY29tLzEg -MB4GCSqGSIb3DQEJARYRaW5mb0B2YWxpY2VydC5jb22CAQEwDwYDVR0TAQH/BAUw -AwEB/zAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly9vY3NwLmdv -ZGFkZHkuY29tMEQGA1UdHwQ9MDswOaA3oDWGM2h0dHA6Ly9jZXJ0aWZpY2F0ZXMu -Z29kYWRkeS5jb20vcmVwb3NpdG9yeS9yb290LmNybDBLBgNVHSAERDBCMEAGBFUd -IAAwODA2BggrBgEFBQcCARYqaHR0cDovL2NlcnRpZmljYXRlcy5nb2RhZGR5LmNv -bS9yZXBvc2l0b3J5MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAAOBgQC1 -QPmnHfbq/qQaQlpE9xXUhUaJwL6e4+PrxeNYiY+Sn1eocSxI0YGyeR+sBjUZsE4O -WBsUs5iB0QQeyAfJg594RAoYC5jcdnplDQ1tgMQLARzLrUc+cb53S8wGd9D0Vmsf -SxOaFIqII6hR8INMqzW/Rn453HWkrugp++85j09VZw== +# Issuer: CN=COMODO ECC Certification Authority O=COMODO CA Limited +# Subject: CN=COMODO ECC Certification Authority O=COMODO CA Limited +# Label: "COMODO ECC Certification Authority" +# Serial: 41578283867086692638256921589707938090 +# MD5 Fingerprint: 7c:62:ff:74:9d:31:53:5e:68:4a:d5:78:aa:1e:bf:23 +# SHA1 Fingerprint: 9f:74:4e:9f:2b:4d:ba:ec:0f:31:2c:50:b6:56:3b:8e:2d:93:c3:11 +# SHA256 Fingerprint: 17:93:92:7a:06:14:54:97:89:ad:ce:2f:8f:34:f7:f0:b6:6d:0f:3a:e3:a3:b8:4d:21:ec:15:db:ba:4f:ad:c7 +-----BEGIN CERTIFICATE----- +MIICiTCCAg+gAwIBAgIQH0evqmIAcFBUTAGem2OZKjAKBggqhkjOPQQDAzCBhTEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMT +IkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwMzA2MDAw +MDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdy +ZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09N +T0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlv +biBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQDR3svdcmCFYX7deSR +FtSrYpn1PlILBs5BAH+X4QokPB0BBO490o0JlwzgdeT6+3eKKvUDYEs2ixYjFq0J +cfRK9ChQtP6IHG4/bC8vCVlbpVsLM5niwz2J+Wos77LTBumjQjBAMB0GA1UdDgQW +BBR1cacZSBm8nZ3qQUfflMRId5nTeTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/ +BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjEA7wNbeqy3eApyt4jf/7VGFAkK+qDm +fQjGGoe9GKhzvSbKYAydzpmfz1wPMOG+FDHqAjAU9JM8SaczepBGR7NjfRObTrdv +GDeAU/7dIOA1mjbRxwG55tzd8/8dLDoWV9mSOdY= -----END CERTIFICATE----- + +# Issuer: CN=TC TrustCenter Class 2 CA II O=TC TrustCenter GmbH OU=TC TrustCenter Class 2 CA +# Subject: CN=TC TrustCenter Class 2 CA II O=TC TrustCenter GmbH OU=TC TrustCenter Class 2 CA +# Label: "TC TrustCenter Class 2 CA II" +# Serial: 941389028203453866782103406992443 +# MD5 Fingerprint: ce:78:33:5c:59:78:01:6e:18:ea:b9:36:a0:b9:2e:23 +# SHA1 Fingerprint: ae:50:83:ed:7c:f4:5c:bc:8f:61:c6:21:fe:68:5d:79:42:21:15:6e +# SHA256 Fingerprint: e6:b8:f8:76:64:85:f8:07:ae:7f:8d:ac:16:70:46:1f:07:c0:a1:3e:ef:3a:1f:f7:17:53:8d:7a:ba:d3:91:b4 -----BEGIN CERTIFICATE----- -MIIC5zCCAlACAQEwDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0 -IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAz -BgNVBAsTLFZhbGlDZXJ0IENsYXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9y -aXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG -9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTk5MDYyNjAwMTk1NFoXDTE5MDYy -NjAwMTk1NFowgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0IFZhbGlkYXRpb24gTmV0d29y -azEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENs -YXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9yaXR5MSEwHwYDVQQDExhodHRw -Oi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG9w0BCQEWEWluZm9AdmFsaWNl -cnQuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDOOnHK5avIWZJV16vY -dA757tn2VUdZZUcOBVXc65g2PFxTXdMwzzjsvUGJ7SVCCSRrCl6zfN1SLUzm1NZ9 -WlmpZdRJEy0kTRxQb7XBhVQ7/nHk01xC+YDgkRoKWzk2Z/M/VXwbP7RfZHM047QS -v4dk+NoS/zcnwbNDu+97bi5p9wIDAQABMA0GCSqGSIb3DQEBBQUAA4GBADt/UG9v -UJSZSWI4OB9L+KXIPqeCgfYrx+jFzug6EILLGACOTb2oWH+heQC1u+mNr0HZDzTu -IYEZoDJJKPTEjlbVUjP9UNV+mWwD5MlM/Mtsq2azSiGM5bUMMj4QssxsodyamEwC -W/POuZ6lcg5Ktz885hZo+L7tdEy8W9ViH0Pd +MIIEqjCCA5KgAwIBAgIOLmoAAQACH9dSISwRXDswDQYJKoZIhvcNAQEFBQAwdjEL +MAkGA1UEBhMCREUxHDAaBgNVBAoTE1RDIFRydXN0Q2VudGVyIEdtYkgxIjAgBgNV +BAsTGVRDIFRydXN0Q2VudGVyIENsYXNzIDIgQ0ExJTAjBgNVBAMTHFRDIFRydXN0 +Q2VudGVyIENsYXNzIDIgQ0EgSUkwHhcNMDYwMTEyMTQzODQzWhcNMjUxMjMxMjI1 +OTU5WjB2MQswCQYDVQQGEwJERTEcMBoGA1UEChMTVEMgVHJ1c3RDZW50ZXIgR21i +SDEiMCAGA1UECxMZVEMgVHJ1c3RDZW50ZXIgQ2xhc3MgMiBDQTElMCMGA1UEAxMc +VEMgVHJ1c3RDZW50ZXIgQ2xhc3MgMiBDQSBJSTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAKuAh5uO8MN8h9foJIIRszzdQ2Lu+MNF2ujhoF/RKrLqk2jf +tMjWQ+nEdVl//OEd+DFwIxuInie5e/060smp6RQvkL4DUsFJzfb95AhmC1eKokKg +uNV/aVyQMrKXDcpK3EY+AlWJU+MaWss2xgdW94zPEfRMuzBwBJWl9jmM/XOBCH2J +XjIeIqkiRUuwZi4wzJ9l/fzLganx4Duvo4bRierERXlQXa7pIXSSTYtZgo+U4+lK +8edJsBTj9WLL1XK9H7nSn6DNqPoByNkN39r8R52zyFTfSUrxIan+GE7uSNQZu+99 +5OKdy1u2bv/jzVrndIIFuoAlOMvkaZ6vQaoahPUCAwEAAaOCATQwggEwMA8GA1Ud +EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBTjq1RMgKHbVkO3 +kUrL84J6E1wIqzCB7QYDVR0fBIHlMIHiMIHfoIHcoIHZhjVodHRwOi8vd3d3LnRy +dXN0Y2VudGVyLmRlL2NybC92Mi90Y19jbGFzc18yX2NhX0lJLmNybIaBn2xkYXA6 +Ly93d3cudHJ1c3RjZW50ZXIuZGUvQ049VEMlMjBUcnVzdENlbnRlciUyMENsYXNz +JTIwMiUyMENBJTIwSUksTz1UQyUyMFRydXN0Q2VudGVyJTIwR21iSCxPVT1yb290 +Y2VydHMsREM9dHJ1c3RjZW50ZXIsREM9ZGU/Y2VydGlmaWNhdGVSZXZvY2F0aW9u +TGlzdD9iYXNlPzANBgkqhkiG9w0BAQUFAAOCAQEAjNfffu4bgBCzg/XbEeprS6iS +GNn3Bzn1LL4GdXpoUxUc6krtXvwjshOg0wn/9vYua0Fxec3ibf2uWWuFHbhOIprt +ZjluS5TmVfwLG4t3wVMTZonZKNaL80VKY7f9ewthXbhtvsPcW3nS7Yblok2+XnR8 +au0WOB9/WIFaGusyiC2y8zl3gK9etmF1KdsjTYjKUCjLhdLTEKJZbtOTVAB6okaV +hgWcqRmY5TFyDADiZ9lA4CQze28suVyrZZ0srHbqNZn1l7kPJOzHdiEoZa5X6AeI +dUpWoNIFOqTmjZKILPPy4cHGYdtBxceb9w4aUUXCYWvcZCcXjFq32nQozZfkvQ== +-----END CERTIFICATE----- + +# Issuer: CN=TC TrustCenter Class 3 CA II O=TC TrustCenter GmbH OU=TC TrustCenter Class 3 CA +# Subject: CN=TC TrustCenter Class 3 CA II O=TC TrustCenter GmbH OU=TC TrustCenter Class 3 CA +# Label: "TC TrustCenter Class 3 CA II" +# Serial: 1506523511417715638772220530020799 +# MD5 Fingerprint: 56:5f:aa:80:61:12:17:f6:67:21:e6:2b:6d:61:56:8e +# SHA1 Fingerprint: 80:25:ef:f4:6e:70:c8:d4:72:24:65:84:fe:40:3b:8a:8d:6a:db:f5 +# SHA256 Fingerprint: 8d:a0:84:fc:f9:9c:e0:77:22:f8:9b:32:05:93:98:06:fa:5c:b8:11:e1:c8:13:f6:a1:08:c7:d3:36:b3:40:8e +-----BEGIN CERTIFICATE----- +MIIEqjCCA5KgAwIBAgIOSkcAAQAC5aBd1j8AUb8wDQYJKoZIhvcNAQEFBQAwdjEL +MAkGA1UEBhMCREUxHDAaBgNVBAoTE1RDIFRydXN0Q2VudGVyIEdtYkgxIjAgBgNV +BAsTGVRDIFRydXN0Q2VudGVyIENsYXNzIDMgQ0ExJTAjBgNVBAMTHFRDIFRydXN0 +Q2VudGVyIENsYXNzIDMgQ0EgSUkwHhcNMDYwMTEyMTQ0MTU3WhcNMjUxMjMxMjI1 +OTU5WjB2MQswCQYDVQQGEwJERTEcMBoGA1UEChMTVEMgVHJ1c3RDZW50ZXIgR21i +SDEiMCAGA1UECxMZVEMgVHJ1c3RDZW50ZXIgQ2xhc3MgMyBDQTElMCMGA1UEAxMc +VEMgVHJ1c3RDZW50ZXIgQ2xhc3MgMyBDQSBJSTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBALTgu1G7OVyLBMVMeRwjhjEQY0NVJz/GRcekPewJDRoeIMJW +Ht4bNwcwIi9v8Qbxq63WyKthoy9DxLCyLfzDlml7forkzMA5EpBCYMnMNWju2l+Q +Vl/NHE1bWEnrDgFPZPosPIlY2C8u4rBo6SI7dYnWRBpl8huXJh0obazovVkdKyT2 +1oQDZogkAHhg8fir/gKya/si+zXmFtGt9i4S5Po1auUZuV3bOx4a+9P/FRQI2Alq +ukWdFHlgfa9Aigdzs5OW03Q0jTo3Kd5c7PXuLjHCINy+8U9/I1LZW+Jk2ZyqBwi1 +Rb3R0DHBq1SfqdLDYmAD8bs5SpJKPQq5ncWg/jcCAwEAAaOCATQwggEwMA8GA1Ud +EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBTUovyfs8PYA9NX +XAek0CSnwPIA1DCB7QYDVR0fBIHlMIHiMIHfoIHcoIHZhjVodHRwOi8vd3d3LnRy +dXN0Y2VudGVyLmRlL2NybC92Mi90Y19jbGFzc18zX2NhX0lJLmNybIaBn2xkYXA6 +Ly93d3cudHJ1c3RjZW50ZXIuZGUvQ049VEMlMjBUcnVzdENlbnRlciUyMENsYXNz +JTIwMyUyMENBJTIwSUksTz1UQyUyMFRydXN0Q2VudGVyJTIwR21iSCxPVT1yb290 +Y2VydHMsREM9dHJ1c3RjZW50ZXIsREM9ZGU/Y2VydGlmaWNhdGVSZXZvY2F0aW9u +TGlzdD9iYXNlPzANBgkqhkiG9w0BAQUFAAOCAQEANmDkcPcGIEPZIxpC8vijsrlN +irTzwppVMXzEO2eatN9NDoqTSheLG43KieHPOh6sHfGcMrSOWXaiQYUlN6AT0PV8 +TtXqluJucsG7Kv5sbviRmEb8yRtXW+rIGjs/sFGYPAfaLFkB2otE6OF0/ado3VS6 +g0bsyEa1+K+XwDsJHI/OcpY9M1ZwvJbL2NV9IJqDnxrcOfHFcqMRA/07QlIp2+gB +95tejNaNhk4Z+rwcvsUhpYeeeC422wlxo3I0+GzjBgnyXlal092Y+tTmBvTwtiBj +S+opvaqCZh77gaqnN60TGOaSw4HBM7uIHqHn4rS9MWwOUT1v+5ZWgOI2F9Hc5A== -----END CERTIFICATE----- -GeoTrust Global CA -================== +# Issuer: CN=TC TrustCenter Universal CA I O=TC TrustCenter GmbH OU=TC TrustCenter Universal CA +# Subject: CN=TC TrustCenter Universal CA I O=TC TrustCenter GmbH OU=TC TrustCenter Universal CA +# Label: "TC TrustCenter Universal CA I" +# Serial: 601024842042189035295619584734726 +# MD5 Fingerprint: 45:e1:a5:72:c5:a9:36:64:40:9e:f5:e4:58:84:67:8c +# SHA1 Fingerprint: 6b:2f:34:ad:89:58:be:62:fd:b0:6b:5c:ce:bb:9d:d9:4f:4e:39:f3 +# SHA256 Fingerprint: eb:f3:c0:2a:87:89:b1:fb:7d:51:19:95:d6:63:b7:29:06:d9:13:ce:0d:5e:10:56:8a:8a:77:e2:58:61:67:e7 +-----BEGIN CERTIFICATE----- +MIID3TCCAsWgAwIBAgIOHaIAAQAC7LdggHiNtgYwDQYJKoZIhvcNAQEFBQAweTEL +MAkGA1UEBhMCREUxHDAaBgNVBAoTE1RDIFRydXN0Q2VudGVyIEdtYkgxJDAiBgNV +BAsTG1RDIFRydXN0Q2VudGVyIFVuaXZlcnNhbCBDQTEmMCQGA1UEAxMdVEMgVHJ1 +c3RDZW50ZXIgVW5pdmVyc2FsIENBIEkwHhcNMDYwMzIyMTU1NDI4WhcNMjUxMjMx +MjI1OTU5WjB5MQswCQYDVQQGEwJERTEcMBoGA1UEChMTVEMgVHJ1c3RDZW50ZXIg +R21iSDEkMCIGA1UECxMbVEMgVHJ1c3RDZW50ZXIgVW5pdmVyc2FsIENBMSYwJAYD +VQQDEx1UQyBUcnVzdENlbnRlciBVbml2ZXJzYWwgQ0EgSTCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBAKR3I5ZEr5D0MacQ9CaHnPM42Q9e3s9B6DGtxnSR +JJZ4Hgmgm5qVSkr1YnwCqMqs+1oEdjneX/H5s7/zA1hV0qq34wQi0fiU2iIIAI3T +fCZdzHd55yx4Oagmcw6iXSVphU9VDprvxrlE4Vc93x9UIuVvZaozhDrzznq+VZeu +jRIPFDPiUHDDSYcTvFHe15gSWu86gzOSBnWLknwSaHtwag+1m7Z3W0hZneTvWq3z +wZ7U10VOylY0Ibw+F1tvdwxIAUMpsN0/lm7mlaoMwCC2/T42J5zjXM9OgdwZu5GQ +fezmlwQek8wiSdeXhrYTCjxDI3d+8NzmzSQfO4ObNDqDNOMCAwEAAaNjMGEwHwYD +VR0jBBgwFoAUkqR1LKSevoFE63n8isWVpesQdXMwDwYDVR0TAQH/BAUwAwEB/zAO +BgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFJKkdSyknr6BROt5/IrFlaXrEHVzMA0G +CSqGSIb3DQEBBQUAA4IBAQAo0uCG1eb4e/CX3CJrO5UUVg8RMKWaTzqwOuAGy2X1 +7caXJ/4l8lfmXpWMPmRgFVp/Lw0BxbFg/UU1z/CyvwbZ71q+s2IhtNerNXxTPqYn +8aEt2hojnczd7Dwtnic0XQ/CNnm8yUpiLe1r2X1BQ3y2qsrtYbE3ghUJGooWMNjs +ydZHcnhLEEYUjl8Or+zHL6sQ17bxbuyGssLoDZJz3KL0Dzq/YSMQiZxIQG5wALPT +ujdEWBF6AmqI8Dc08BnprNRlc/ZpjGSUOnmFKbAWKwyCPwacx/0QK54PLLae4xW/ +2TYcuiUaUj0a7CIMHOCkoj3w6DnPgcB77V0fb8XQC9eY +-----END CERTIFICATE----- +# Issuer: CN=Cybertrust Global Root O=Cybertrust, Inc +# Subject: CN=Cybertrust Global Root O=Cybertrust, Inc +# Label: "Cybertrust Global Root" +# Serial: 4835703278459682877484360 +# MD5 Fingerprint: 72:e4:4a:87:e3:69:40:80:77:ea:bc:e3:f4:ff:f0:e1 +# SHA1 Fingerprint: 5f:43:e5:b1:bf:f8:78:8c:ac:1c:c7:ca:4a:9a:c6:22:2b:cc:34:c6 +# SHA256 Fingerprint: 96:0a:df:00:63:e9:63:56:75:0c:29:65:dd:0a:08:67:da:0b:9c:bd:6e:77:71:4a:ea:fb:23:49:ab:39:3d:a3 -----BEGIN CERTIFICATE----- -MIIDfTCCAuagAwIBAgIDErvmMA0GCSqGSIb3DQEBBQUAME4xCzAJBgNVBAYTAlVT -MRAwDgYDVQQKEwdFcXVpZmF4MS0wKwYDVQQLEyRFcXVpZmF4IFNlY3VyZSBDZXJ0 -aWZpY2F0ZSBBdXRob3JpdHkwHhcNMDIwNTIxMDQwMDAwWhcNMTgwODIxMDQwMDAw -WjBCMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UE -AxMSR2VvVHJ1c3QgR2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEA2swYYzD99BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9m -OSm9BXiLnTjoBbdqfnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIu -T8rxh0PBFpVXLVDviS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6c -JmTM386DGXHKTubU1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmR -Cw7+OC7RHQWa9k0+bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5asz -PeE4uwc2hGKceeoWMPRfwCvocWvk+QIDAQABo4HwMIHtMB8GA1UdIwQYMBaAFEjm -aPkr0rKV10fYIyAQTzOYkJ/UMB0GA1UdDgQWBBTAephojYn7qwVkDBF9qn1luMrM -TjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjA6BgNVHR8EMzAxMC+g -LaArhilodHRwOi8vY3JsLmdlb3RydXN0LmNvbS9jcmxzL3NlY3VyZWNhLmNybDBO -BgNVHSAERzBFMEMGBFUdIAAwOzA5BggrBgEFBQcCARYtaHR0cHM6Ly93d3cuZ2Vv -dHJ1c3QuY29tL3Jlc291cmNlcy9yZXBvc2l0b3J5MA0GCSqGSIb3DQEBBQUAA4GB -AHbhEm5OSxYShjAGsoEIz/AIx8dxfmbuwu3UOx//8PDITtZDOLC5MH0Y0FWDomrL -NhGc6Ehmo21/uBPUR/6LWlxz/K7ZGzIZOKuXNBSqltLroxwUCEm2u+WR74M26x1W -b8ravHNjkOR/ez4iyz0H7V84dJzjA1BOoa+Y7mHyhD8S +MIIDoTCCAomgAwIBAgILBAAAAAABD4WqLUgwDQYJKoZIhvcNAQEFBQAwOzEYMBYG +A1UEChMPQ3liZXJ0cnVzdCwgSW5jMR8wHQYDVQQDExZDeWJlcnRydXN0IEdsb2Jh +bCBSb290MB4XDTA2MTIxNTA4MDAwMFoXDTIxMTIxNTA4MDAwMFowOzEYMBYGA1UE +ChMPQ3liZXJ0cnVzdCwgSW5jMR8wHQYDVQQDExZDeWJlcnRydXN0IEdsb2JhbCBS +b290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA+Mi8vRRQZhP/8NN5 +7CPytxrHjoXxEnOmGaoQ25yiZXRadz5RfVb23CO21O1fWLE3TdVJDm71aofW0ozS +J8bi/zafmGWgE07GKmSb1ZASzxQG9Dvj1Ci+6A74q05IlG2OlTEQXO2iLb3VOm2y +HLtgwEZLAfVJrn5GitB0jaEMAs7u/OePuGtm839EAL9mJRQr3RAwHQeWP032a7iP +t3sMpTjr3kfb1V05/Iin89cqdPHoWqI7n1C6poxFNcJQZZXcY4Lv3b93TZxiyWNz +FtApD0mpSPCzqrdsxacwOUBdrsTiXSZT8M4cIwhhqJQZugRiQOwfOHB3EgZxpzAY +XSUnpQIDAQABo4GlMIGiMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/ +MB0GA1UdDgQWBBS2CHsNesysIEyGVjJez6tuhS1wVzA/BgNVHR8EODA2MDSgMqAw +hi5odHRwOi8vd3d3Mi5wdWJsaWMtdHJ1c3QuY29tL2NybC9jdC9jdHJvb3QuY3Js +MB8GA1UdIwQYMBaAFLYIew16zKwgTIZWMl7Pq26FLXBXMA0GCSqGSIb3DQEBBQUA +A4IBAQBW7wojoFROlZfJ+InaRcHUowAl9B8Tq7ejhVhpwjCt2BWKLePJzYFa+HMj +Wqd8BfP9IjsO0QbE2zZMcwSO5bAi5MXzLqXZI+O4Tkogp24CJJ8iYGd7ix1yCcUx +XOl5n4BHPa2hCwcUPUf/A2kaDAtE52Mlp3+yybh2hO0j9n0Hq0V+09+zv+mKts2o +omcrUtW3ZfA5TGOgkXmTUg9U3YO7n9GPp1Nzw8v/MOx8BLjYRB+TX3EJIrduPuoc +A06dGiBh+4E37F78CkWr1+cXVdCg6mCbpvbjjFspwgZgFJ0tl0ypkxWdYcQBX0jW +WL1WMRJOEcgh4LMRkWXbtKaIOM5V +-----END CERTIFICATE----- + +# Issuer: CN=GeoTrust Primary Certification Authority - G3 O=GeoTrust Inc. OU=(c) 2008 GeoTrust Inc. - For authorized use only +# Subject: CN=GeoTrust Primary Certification Authority - G3 O=GeoTrust Inc. OU=(c) 2008 GeoTrust Inc. - For authorized use only +# Label: "GeoTrust Primary Certification Authority - G3" +# Serial: 28809105769928564313984085209975885599 +# MD5 Fingerprint: b5:e8:34:36:c9:10:44:58:48:70:6d:2e:83:d4:b8:05 +# SHA1 Fingerprint: 03:9e:ed:b8:0b:e7:a0:3c:69:53:89:3b:20:d2:d9:32:3a:4c:2a:fd +# SHA256 Fingerprint: b4:78:b8:12:25:0d:f8:78:63:5c:2a:a7:ec:7d:15:5e:aa:62:5e:e8:29:16:e2:cd:29:43:61:88:6c:d1:fb:d4 +-----BEGIN CERTIFICATE----- +MIID/jCCAuagAwIBAgIQFaxulBmyeUtB9iepwxgPHzANBgkqhkiG9w0BAQsFADCB +mDELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUdlb1RydXN0IEluYy4xOTA3BgNVBAsT +MChjKSAyMDA4IEdlb1RydXN0IEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25s +eTE2MDQGA1UEAxMtR2VvVHJ1c3QgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhv +cml0eSAtIEczMB4XDTA4MDQwMjAwMDAwMFoXDTM3MTIwMTIzNTk1OVowgZgxCzAJ +BgNVBAYTAlVTMRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMTkwNwYDVQQLEzAoYykg +MjAwOCBHZW9UcnVzdCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxNjA0 +BgNVBAMTLUdlb1RydXN0IFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg +LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANziXmJYHTNXOTIz ++uvLh4yn1ErdBojqZI4xmKU4kB6Yzy5jK/BGvESyiaHAKAxJcCGVn2TAppMSAmUm +hsalifD614SgcK9PGpc/BkTVyetyEH3kMSj7HGHmKAdEc5IiaacDiGydY8hS2pgn +5whMcD60yRLBxWeDXTPzAxHsatBT4tG6NmCUgLthY2xbF37fQJQeqw3CIShwiP/W +JmxsYAQlTlV+fe+/lEjetx3dcI0FX4ilm/LC7urRQEFtYjgdVgbFA0dRIBn8exAL +DmKudlW/X3e+PkkBUz2YJQN2JFodtNuJ6nnltrM7P7pMKEF/BqxqjsHQ9gUdfeZC +huOl1UcCAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYw +HQYDVR0OBBYEFMR5yo6hTgMdHNxr2zFblD4/MH8tMA0GCSqGSIb3DQEBCwUAA4IB +AQAtxRPPVoB7eni9n64smefv2t+UXglpp+duaIy9cr5HqQ6XErhK8WTTOd8lNNTB +zU6B8A8ExCSzNJbGpqow32hhc9f5joWJ7w5elShKKiePEI4ufIbEAp7aDHdlDkQN +kv39sxY2+hENHYwOB4lqKVb3cvTdFZx3NWZXqxNT2I7BQMXXExZacse3aQHEerGD +AWh9jUGhlBjBJVz88P6DAod8DQ3PLghcSkANPuyBYeYk28rgDi0Hsj5W3I31QYUH +SJsMC8tJP33st/3LjWeJGqvtux6jAAgIFyqCXDFdRootD4abdNlF+9RAsXqqaC2G +spki4cErx5z481+oghLrGREt +-----END CERTIFICATE----- + +# Issuer: CN=thawte Primary Root CA - G2 O=thawte, Inc. OU=(c) 2007 thawte, Inc. - For authorized use only +# Subject: CN=thawte Primary Root CA - G2 O=thawte, Inc. OU=(c) 2007 thawte, Inc. - For authorized use only +# Label: "thawte Primary Root CA - G2" +# Serial: 71758320672825410020661621085256472406 +# MD5 Fingerprint: 74:9d:ea:60:24:c4:fd:22:53:3e:cc:3a:72:d9:29:4f +# SHA1 Fingerprint: aa:db:bc:22:23:8f:c4:01:a1:27:bb:38:dd:f4:1d:db:08:9e:f0:12 +# SHA256 Fingerprint: a4:31:0d:50:af:18:a6:44:71:90:37:2a:86:af:af:8b:95:1f:fb:43:1d:83:7f:1e:56:88:b4:59:71:ed:15:57 +-----BEGIN CERTIFICATE----- +MIICiDCCAg2gAwIBAgIQNfwmXNmET8k9Jj1Xm67XVjAKBggqhkjOPQQDAzCBhDEL +MAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjE4MDYGA1UECxMvKGMp +IDIwMDcgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxJDAi +BgNVBAMTG3RoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EgLSBHMjAeFw0wNzExMDUwMDAw +MDBaFw0zODAxMTgyMzU5NTlaMIGEMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMdGhh +d3RlLCBJbmMuMTgwNgYDVQQLEy8oYykgMjAwNyB0aGF3dGUsIEluYy4gLSBGb3Ig +YXV0aG9yaXplZCB1c2Ugb25seTEkMCIGA1UEAxMbdGhhd3RlIFByaW1hcnkgUm9v +dCBDQSAtIEcyMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEotWcgnuVnfFSeIf+iha/ +BebfowJPDQfGAFG6DAJSLSKkQjnE/o/qycG+1E3/n3qe4rF8mq2nhglzh9HnmuN6 +papu+7qzcMBniKI11KOasf2twu8x+qi58/sIxpHR+ymVo0IwQDAPBgNVHRMBAf8E +BTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUmtgAMADna3+FGO6Lts6K +DPgR4bswCgYIKoZIzj0EAwMDaQAwZgIxAN344FdHW6fmCsO99YCKlzUNG4k8VIZ3 +KMqh9HneteY4sPBlcIx/AlTCv//YoT7ZzwIxAMSNlPzcU9LcnXgWHxUzI1NS41ox +XZ3Krr0TKUQNJ1uo52icEvdYPy5yAlejj6EULg== +-----END CERTIFICATE----- + +# Issuer: CN=thawte Primary Root CA - G3 O=thawte, Inc. OU=Certification Services Division/(c) 2008 thawte, Inc. - For authorized use only +# Subject: CN=thawte Primary Root CA - G3 O=thawte, Inc. OU=Certification Services Division/(c) 2008 thawte, Inc. - For authorized use only +# Label: "thawte Primary Root CA - G3" +# Serial: 127614157056681299805556476275995414779 +# MD5 Fingerprint: fb:1b:5d:43:8a:94:cd:44:c6:76:f2:43:4b:47:e7:31 +# SHA1 Fingerprint: f1:8b:53:8d:1b:e9:03:b6:a6:f0:56:43:5b:17:15:89:ca:f3:6b:f2 +# SHA256 Fingerprint: 4b:03:f4:58:07:ad:70:f2:1b:fc:2c:ae:71:c9:fd:e4:60:4c:06:4c:f5:ff:b6:86:ba:e5:db:aa:d7:fd:d3:4c +-----BEGIN CERTIFICATE----- +MIIEKjCCAxKgAwIBAgIQYAGXt0an6rS0mtZLL/eQ+zANBgkqhkiG9w0BAQsFADCB +rjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf +Q2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYGA1UECxMvKGMpIDIw +MDggdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxJDAiBgNV +BAMTG3RoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EgLSBHMzAeFw0wODA0MDIwMDAwMDBa +Fw0zNzEyMDEyMzU5NTlaMIGuMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMdGhhd3Rl +LCBJbmMuMSgwJgYDVQQLEx9DZXJ0aWZpY2F0aW9uIFNlcnZpY2VzIERpdmlzaW9u +MTgwNgYDVQQLEy8oYykgMjAwOCB0aGF3dGUsIEluYy4gLSBGb3IgYXV0aG9yaXpl +ZCB1c2Ugb25seTEkMCIGA1UEAxMbdGhhd3RlIFByaW1hcnkgUm9vdCBDQSAtIEcz +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsr8nLPvb2FvdeHsbnndm +gcs+vHyu86YnmjSjaDFxODNi5PNxZnmxqWWjpYvVj2AtP0LMqmsywCPLLEHd5N/8 +YZzic7IilRFDGF/Eth9XbAoFWCLINkw6fKXRz4aviKdEAhN0cXMKQlkC+BsUa0Lf +b1+6a4KinVvnSr0eAXLbS3ToO39/fR8EtCab4LRarEc9VbjXsCZSKAExQGbY2SS9 +9irY7CFJXJv2eul/VTV+lmuNk5Mny5K76qxAwJ/C+IDPXfRa3M50hqY+bAtTyr2S +zhkGcuYMXDhpxwTWvGzOW/b3aJzcJRVIiKHpqfiYnODz1TEoYRFsZ5aNOZnLwkUk +OQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNV +HQ4EFgQUrWyqlGCc7eT/+j4KdCtjA/e2Wb8wDQYJKoZIhvcNAQELBQADggEBABpA +2JVlrAmSicY59BDlqQ5mU1143vokkbvnRFHfxhY0Cu9qRFHqKweKA3rD6z8KLFIW +oCtDuSWQP3CpMyVtRRooOyfPqsMpQhvfO0zAMzRbQYi/aytlryjvsvXDqmbOe1bu +t8jLZ8HJnBoYuMTDSQPxYA5QzUbF83d597YV4Djbxy8ooAw/dyZ02SUS2jHaGh7c +KUGRIjxpp7sC8rZcJwOJ9Abqm+RyguOhCcHpABnTPtRwa7pxpqpYrvS76Wy274fM +m7v/OeZWYdMKp8RcTGB7BXcmer/YB1IsYvdwY9k5vG8cwnncdimvzsUsZAReiDZu +MdRAGmI0Nj81Aa6sY6A= -----END CERTIFICATE----- +# Issuer: CN=GeoTrust Primary Certification Authority - G2 O=GeoTrust Inc. OU=(c) 2007 GeoTrust Inc. - For authorized use only +# Subject: CN=GeoTrust Primary Certification Authority - G2 O=GeoTrust Inc. OU=(c) 2007 GeoTrust Inc. - For authorized use only +# Label: "GeoTrust Primary Certification Authority - G2" +# Serial: 80682863203381065782177908751794619243 +# MD5 Fingerprint: 01:5e:d8:6b:bd:6f:3d:8e:a1:31:f8:12:e0:98:73:6a +# SHA1 Fingerprint: 8d:17:84:d5:37:f3:03:7d:ec:70:fe:57:8b:51:9a:99:e6:10:d7:b0 +# SHA256 Fingerprint: 5e:db:7a:c4:3b:82:a0:6a:87:61:e8:d7:be:49:79:eb:f2:61:1f:7d:d7:9b:f9:1c:1c:6b:56:6a:21:9e:d7:66 +-----BEGIN CERTIFICATE----- +MIICrjCCAjWgAwIBAgIQPLL0SAoA4v7rJDteYD7DazAKBggqhkjOPQQDAzCBmDEL +MAkGA1UEBhMCVVMxFjAUBgNVBAoTDUdlb1RydXN0IEluYy4xOTA3BgNVBAsTMChj +KSAyMDA3IEdlb1RydXN0IEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTE2 +MDQGA1UEAxMtR2VvVHJ1c3QgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0 +eSAtIEcyMB4XDTA3MTEwNTAwMDAwMFoXDTM4MDExODIzNTk1OVowgZgxCzAJBgNV +BAYTAlVTMRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMTkwNwYDVQQLEzAoYykgMjAw +NyBHZW9UcnVzdCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxNjA0BgNV +BAMTLUdlb1RydXN0IFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBH +MjB2MBAGByqGSM49AgEGBSuBBAAiA2IABBWx6P0DFUPlrOuHNxFi79KDNlJ9RVcL +So17VDs6bl8VAsBQps8lL33KSLjHUGMcKiEIfJo22Av+0SbFWDEwKCXzXV2juLal +tJLtbCyf691DiaI8S0iRHVDsJt/WYC69IaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAO +BgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFBVfNVdRVfslsq0DafwBo/q+EVXVMAoG +CCqGSM49BAMDA2cAMGQCMGSWWaboCd6LuvpaiIjwH5HTRqjySkwCY/tsXzjbLkGT +qQ7mndwxHLKgpxgceeHHNgIwOlavmnRs9vuD4DPTCF+hnMJbn0bWtsuRBmOiBucz +rD6ogRLQy7rQkgu2npaqBA+K +-----END CERTIFICATE----- + +# Issuer: CN=VeriSign Universal Root Certification Authority O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2008 VeriSign, Inc. - For authorized use only +# Subject: CN=VeriSign Universal Root Certification Authority O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2008 VeriSign, Inc. - For authorized use only +# Label: "VeriSign Universal Root Certification Authority" +# Serial: 85209574734084581917763752644031726877 +# MD5 Fingerprint: 8e:ad:b5:01:aa:4d:81:e4:8c:1d:d1:e1:14:00:95:19 +# SHA1 Fingerprint: 36:79:ca:35:66:87:72:30:4d:30:a5:fb:87:3b:0f:a7:7b:b7:0d:54 +# SHA256 Fingerprint: 23:99:56:11:27:a5:71:25:de:8c:ef:ea:61:0d:df:2f:a0:78:b5:c8:06:7f:4e:82:82:90:bf:b8:60:e8:4b:3c +-----BEGIN CERTIFICATE----- +MIIEuTCCA6GgAwIBAgIQQBrEZCGzEyEDDrvkEhrFHTANBgkqhkiG9w0BAQsFADCB +vTELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwOCBWZXJp +U2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MTgwNgYDVQQDEy9W +ZXJpU2lnbiBVbml2ZXJzYWwgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAe +Fw0wODA0MDIwMDAwMDBaFw0zNzEyMDEyMzU5NTlaMIG9MQswCQYDVQQGEwJVUzEX +MBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlTaWduIFRydXN0 +IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAyMDA4IFZlcmlTaWduLCBJbmMuIC0gRm9y +IGF1dGhvcml6ZWQgdXNlIG9ubHkxODA2BgNVBAMTL1ZlcmlTaWduIFVuaXZlcnNh +bCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAx2E3XrEBNNti1xWb/1hajCMj1mCOkdeQmIN65lgZOIzF +9uVkhbSicfvtvbnazU0AtMgtc6XHaXGVHzk8skQHnOgO+k1KxCHfKWGPMiJhgsWH +H26MfF8WIFFE0XBPV+rjHOPMee5Y2A7Cs0WTwCznmhcrewA3ekEzeOEz4vMQGn+H +LL729fdC4uW/h2KJXwBL38Xd5HVEMkE6HnFuacsLdUYI0crSK5XQz/u5QGtkjFdN +/BMReYTtXlT2NJ8IAfMQJQYXStrxHXpma5hgZqTZ79IugvHw7wnqRMkVauIDbjPT +rJ9VAMf2CGqUuV/c4DPxhGD5WycRtPwW8rtWaoAljQIDAQABo4GyMIGvMA8GA1Ud +EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMG0GCCsGAQUFBwEMBGEwX6FdoFsw +WTBXMFUWCWltYWdlL2dpZjAhMB8wBwYFKw4DAhoEFI/l0xqGrI2Oa8PPgGrUSBgs +exkuMCUWI2h0dHA6Ly9sb2dvLnZlcmlzaWduLmNvbS92c2xvZ28uZ2lmMB0GA1Ud +DgQWBBS2d/ppSEefUxLVwuoHMnYH0ZcHGTANBgkqhkiG9w0BAQsFAAOCAQEASvj4 +sAPmLGd75JR3Y8xuTPl9Dg3cyLk1uXBPY/ok+myDjEedO2Pzmvl2MpWRsXe8rJq+ +seQxIcaBlVZaDrHC1LGmWazxY8u4TB1ZkErvkBYoH1quEPuBUDgMbMzxPcP1Y+Oz +4yHJJDnp/RVmRvQbEdBNc6N9Rvk97ahfYtTxP/jgdFcrGJ2BtMQo2pSXpXDrrB2+ +BxHw1dvd5Yzw1TKwg+ZX4o+/vqGqvz0dtdQ46tewXDpPaj+PwGZsY6rp2aQW9IHR +lRQOfc2VNNnSj3BzgXucfr2YYdhFh5iQxeuGMMY1v/D/w1WIg0vvBZIGcfK4mJO3 +7M2CYfE45k+XmCpajQ== +-----END CERTIFICATE----- + +# Issuer: CN=VeriSign Class 3 Public Primary Certification Authority - G4 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2007 VeriSign, Inc. - For authorized use only +# Subject: CN=VeriSign Class 3 Public Primary Certification Authority - G4 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2007 VeriSign, Inc. - For authorized use only +# Label: "VeriSign Class 3 Public Primary Certification Authority - G4" +# Serial: 63143484348153506665311985501458640051 +# MD5 Fingerprint: 3a:52:e1:e7:fd:6f:3a:e3:6f:f3:6f:99:1b:f9:22:41 +# SHA1 Fingerprint: 22:d5:d8:df:8f:02:31:d1:8d:f7:9d:b7:cf:8a:2d:64:c9:3f:6c:3a +# SHA256 Fingerprint: 69:dd:d7:ea:90:bb:57:c9:3e:13:5d:c8:5e:a6:fc:d5:48:0b:60:32:39:bd:c4:54:fc:75:8b:2a:26:cf:7f:79 +-----BEGIN CERTIFICATE----- +MIIDhDCCAwqgAwIBAgIQL4D+I4wOIg9IZxIokYesszAKBggqhkjOPQQDAzCByjEL +MAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZW +ZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNyBWZXJpU2ln +biwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJp +U2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9y +aXR5IC0gRzQwHhcNMDcxMTA1MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCByjELMAkG +A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZWZXJp +U2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNyBWZXJpU2lnbiwg +SW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJpU2ln +biBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5 +IC0gRzQwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASnVnp8Utpkmw4tXNherJI9/gHm +GUo9FANL+mAnINmDiWn6VMaaGF5VKmTeBvaNSjutEDxlPZCIBIngMGGzrl0Bp3ve +fLK+ymVhAIau2o970ImtTR1ZmkGxvEeA3J5iw/mjgbIwga8wDwYDVR0TAQH/BAUw +AwEB/zAOBgNVHQ8BAf8EBAMCAQYwbQYIKwYBBQUHAQwEYTBfoV2gWzBZMFcwVRYJ +aW1hZ2UvZ2lmMCEwHzAHBgUrDgMCGgQUj+XTGoasjY5rw8+AatRIGCx7GS4wJRYj +aHR0cDovL2xvZ28udmVyaXNpZ24uY29tL3ZzbG9nby5naWYwHQYDVR0OBBYEFLMW +kf3upm7ktS5Jj4d4gYDs5bG1MAoGCCqGSM49BAMDA2gAMGUCMGYhDBgmYFo4e1ZC +4Kf8NoRRkSAsdk1DPcQdhCPQrNZ8NQbOzWm9kA3bbEhCHQ6qQgIxAJw9SDkjOVga +FRJZap7v1VmyHVIsmXHNxynfGyphe3HR3vPA5Q06Sqotp9iGKt0uEA== +-----END CERTIFICATE----- + +# Issuer: O=VeriSign, Inc. OU=Class 3 Public Primary Certification Authority +# Subject: O=VeriSign, Inc. OU=Class 3 Public Primary Certification Authority +# Label: "Verisign Class 3 Public Primary Certification Authority" +# Serial: 80507572722862485515306429940691309246 +# MD5 Fingerprint: ef:5a:f1:33:ef:f1:cd:bb:51:02:ee:12:14:4b:96:c4 +# SHA1 Fingerprint: a1:db:63:93:91:6f:17:e4:18:55:09:40:04:15:c7:02:40:b0:ae:6b +# SHA256 Fingerprint: a4:b6:b3:99:6f:c2:f3:06:b3:fd:86:81:bd:63:41:3d:8c:50:09:cc:4f:a3:29:c2:cc:f0:e2:fa:1b:14:03:05 +-----BEGIN CERTIFICATE----- +MIICPDCCAaUCEDyRMcsf9tAbDpq40ES/Er4wDQYJKoZIhvcNAQEFBQAwXzELMAkG +A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz +cyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2 +MDEyOTAwMDAwMFoXDTI4MDgwMjIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV +BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAzIFB1YmxpYyBQcmlt +YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN +ADCBiQKBgQDJXFme8huKARS0EN8EQNvjV69qRUCPhAwL0TPZ2RHP7gJYHyX3KqhE +BarsAx94f56TuZoAqiN91qyFomNFx3InzPRMxnVx0jnvT0Lwdd8KkMaOIG+YD/is +I19wKTakyYbnsZogy1Olhec9vn2a/iRFM9x2Fe0PonFkTGUugWhFpwIDAQABMA0G +CSqGSIb3DQEBBQUAA4GBABByUqkFFBkyCEHwxWsKzH4PIRnN5GfcX6kb5sroc50i +2JhucwNhkcV8sEVAbkSdjbCxlnRhLQ2pRdKkkirWmnWXbj9T/UWZYB2oK0z5XqcJ +2HUw19JlYD1n1khVdWk/kfVIC0dpImmClr7JyDiGSnoscxlIaU5rfGW/D/xwzoiQ +-----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R3 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R3 +# Label: "GlobalSign Root CA - R3" +# Serial: 4835703278459759426209954 +# MD5 Fingerprint: c5:df:b8:49:ca:05:13:55:ee:2d:ba:1a:c3:3e:b0:28 +# SHA1 Fingerprint: d6:9b:56:11:48:f0:1c:77:c5:45:78:c1:09:26:df:5b:85:69:76:ad +# SHA256 Fingerprint: cb:b5:22:d7:b7:f1:27:ad:6a:01:13:86:5b:df:1c:d4:10:2e:7d:07:59:af:63:5a:7c:f4:72:0d:c9:63:c5:3b +-----BEGIN CERTIFICATE----- +MIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4G +A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp +Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDkwMzE4MTAwMDAwWhcNMjkwMzE4 +MTAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMzETMBEG +A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAMwldpB5BngiFvXAg7aEyiie/QV2EcWtiHL8 +RgJDx7KKnQRfJMsuS+FggkbhUqsMgUdwbN1k0ev1LKMPgj0MK66X17YUhhB5uzsT +gHeMCOFJ0mpiLx9e+pZo34knlTifBtc+ycsmWQ1z3rDI6SYOgxXG71uL0gRgykmm +KPZpO/bLyCiR5Z2KYVc3rHQU3HTgOu5yLy6c+9C7v/U9AOEGM+iCK65TpjoWc4zd +QQ4gOsC0p6Hpsk+QLjJg6VfLuQSSaGjlOCZgdbKfd/+RFO+uIEn8rUAVSNECMWEZ +XriX7613t2Saer9fwRPvm2L7DWzgVGkWqQPabumDk3F2xmmFghcCAwEAAaNCMEAw +DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI/wS3+o +LkUkrk1Q+mOai97i3Ru8MA0GCSqGSIb3DQEBCwUAA4IBAQBLQNvAUKr+yAzv95ZU +RUm7lgAJQayzE4aGKAczymvmdLm6AC2upArT9fHxD4q/c2dKg8dEe3jgr25sbwMp +jjM5RcOO5LlXbKr8EpbsU8Yt5CRsuZRj+9xTaGdWPoO4zzUhw8lo/s7awlOqzJCK +6fBdRoyV3XpYKBovHd7NADdBj+1EbddTKJd+82cEHhXXipa0095MJ6RMG3NzdvQX +mcIfeg7jLQitChws/zyrVQ4PkX4268NXSb7hLi18YIvDQVETI53O9zJrlAGomecs +Mx86OyXShkDOOyyGeMlhLxS67ttVb9+E7gUJTb0o2HLO02JQZR7rkpeDMdmztcpH +WD9f +-----END CERTIFICATE----- + +# Issuer: CN=TC TrustCenter Universal CA III O=TC TrustCenter GmbH OU=TC TrustCenter Universal CA +# Subject: CN=TC TrustCenter Universal CA III O=TC TrustCenter GmbH OU=TC TrustCenter Universal CA +# Label: "TC TrustCenter Universal CA III" +# Serial: 2010889993983507346460533407902964 +# MD5 Fingerprint: 9f:dd:db:ab:ff:8e:ff:45:21:5f:f0:6c:9d:8f:fe:2b +# SHA1 Fingerprint: 96:56:cd:7b:57:96:98:95:d0:e1:41:46:68:06:fb:b8:c6:11:06:87 +# SHA256 Fingerprint: 30:9b:4a:87:f6:ca:56:c9:31:69:aa:a9:9c:6d:98:88:54:d7:89:2b:d5:43:7e:2d:07:b2:9c:be:da:55:d3:5d +-----BEGIN CERTIFICATE----- +MIID4TCCAsmgAwIBAgIOYyUAAQACFI0zFQLkbPQwDQYJKoZIhvcNAQEFBQAwezEL +MAkGA1UEBhMCREUxHDAaBgNVBAoTE1RDIFRydXN0Q2VudGVyIEdtYkgxJDAiBgNV +BAsTG1RDIFRydXN0Q2VudGVyIFVuaXZlcnNhbCBDQTEoMCYGA1UEAxMfVEMgVHJ1 +c3RDZW50ZXIgVW5pdmVyc2FsIENBIElJSTAeFw0wOTA5MDkwODE1MjdaFw0yOTEy +MzEyMzU5NTlaMHsxCzAJBgNVBAYTAkRFMRwwGgYDVQQKExNUQyBUcnVzdENlbnRl +ciBHbWJIMSQwIgYDVQQLExtUQyBUcnVzdENlbnRlciBVbml2ZXJzYWwgQ0ExKDAm +BgNVBAMTH1RDIFRydXN0Q2VudGVyIFVuaXZlcnNhbCBDQSBJSUkwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDC2pxisLlxErALyBpXsq6DFJmzNEubkKLF +5+cvAqBNLaT6hdqbJYUtQCggbergvbFIgyIpRJ9Og+41URNzdNW88jBmlFPAQDYv +DIRlzg9uwliT6CwLOunBjvvya8o84pxOjuT5fdMnnxvVZ3iHLX8LR7PH6MlIfK8v +zArZQe+f/prhsq75U7Xl6UafYOPfjdN/+5Z+s7Vy+EutCHnNaYlAJ/Uqwa1D7KRT +yGG299J5KmcYdkhtWyUB0SbFt1dpIxVbYYqt8Bst2a9c8SaQaanVDED1M4BDj5yj +dipFtK+/fz6HP3bFzSreIMUWWMv5G/UPyw0RUmS40nZid4PxWJ//AgMBAAGjYzBh +MB8GA1UdIwQYMBaAFFbn4VslQ4Dg9ozhcbyO5YAvxEjiMA8GA1UdEwEB/wQFMAMB +Af8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBRW5+FbJUOA4PaM4XG8juWAL8RI +4jANBgkqhkiG9w0BAQUFAAOCAQEAg8ev6n9NCjw5sWi+e22JLumzCecYV42Fmhfz +dkJQEw/HkG8zrcVJYCtsSVgZ1OK+t7+rSbyUyKu+KGwWaODIl0YgoGhnYIg5IFHY +aAERzqf2EQf27OysGh+yZm5WZ2B6dF7AbZc2rrUNXWZzwCUyRdhKBgePxLcHsU0G +DeGl6/R1yrqc0L2z0zIkTO5+4nYES0lT2PLpVDP85XEfPRRclkvxOvIAu2y0+pZV +CIgJwcyRGSmwIC3/yzikQOEXvnlhgP8HA4ZMTnsGnxGGjYnuJ8Tb4rwZjgvDwxPH +LQNjO9Po5KIqwoIIlBZU8O8fJ5AluA0OKBtHd0e9HKgl8ZS0Zg== +-----END CERTIFICATE----- + +# Issuer: CN=Go Daddy Root Certificate Authority - G2 O=GoDaddy.com, Inc. +# Subject: CN=Go Daddy Root Certificate Authority - G2 O=GoDaddy.com, Inc. +# Label: "Go Daddy Root Certificate Authority - G2" +# Serial: 0 +# MD5 Fingerprint: 80:3a:bc:22:c1:e6:fb:8d:9b:3b:27:4a:32:1b:9a:01 +# SHA1 Fingerprint: 47:be:ab:c9:22:ea:e8:0e:78:78:34:62:a7:9f:45:c2:54:fd:e6:8b +# SHA256 Fingerprint: 45:14:0b:32:47:eb:9c:c8:c5:b4:f0:d7:b5:30:91:f7:32:92:08:9e:6e:5a:63:e2:74:9d:d3:ac:a9:19:8e:da +-----BEGIN CERTIFICATE----- +MIIDxTCCAq2gAwIBAgIBADANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoT +EUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRp +ZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAwMFoXDTM3MTIzMTIz +NTk1OVowgYMxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6b25hMRMwEQYDVQQH +EwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjExMC8GA1UE +AxMoR28gRGFkZHkgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL9xYgjx+lk09xvJGKP3gElY6SKD +E6bFIEMBO4Tx5oVJnyfq9oQbTqC023CYxzIBsQU+B07u9PpPL1kwIuerGVZr4oAH +/PMWdYA5UXvl+TW2dE6pjYIT5LY/qQOD+qK+ihVqf94Lw7YZFAXK6sOoBJQ7Rnwy +DfMAZiLIjWltNowRGLfTshxgtDj6AozO091GB94KPutdfMh8+7ArU6SSYmlRJQVh +GkSBjCypQ5Yj36w6gZoOKcUcqeldHraenjAKOc7xiID7S13MMuyFYkMlNAJWJwGR +tDtwKj9useiciAF9n9T521NtYJ2/LOdYq7hfRvzOxBsDPAnrSTFcaUaz4EcCAwEA +AaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FDqahQcQZyi27/a9BUFuIMGU2g/eMA0GCSqGSIb3DQEBCwUAA4IBAQCZ21151fmX +WWcDYfF+OwYxdS2hII5PZYe096acvNjpL9DbWu7PdIxztDhC2gV7+AJ1uP2lsdeu +9tfeE8tTEH6KRtGX+rcuKxGrkLAngPnon1rpN5+r5N9ss4UXnT3ZJE95kTXWXwTr +gIOrmgIttRD02JDHBHNA7XIloKmf7J6raBKZV8aPEjoJpL1E/QYVN8Gb5DKj7Tjo +2GTzLH4U/ALqn83/B2gX2yKQOC16jdFU8WnjXzPKej17CuPKf1855eJ1usV2GDPO +LPAvTK33sefOT6jEm0pUBsV/fdUID+Ic/n4XuKxe9tQWskMJDE32p2u0mYRlynqI +4uJEvlz36hz1 +-----END CERTIFICATE----- + +# Issuer: CN=Starfield Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Subject: CN=Starfield Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Label: "Starfield Root Certificate Authority - G2" +# Serial: 0 +# MD5 Fingerprint: d6:39:81:c6:52:7e:96:69:fc:fc:ca:66:ed:05:f2:96 +# SHA1 Fingerprint: b5:1c:06:7c:ee:2b:0c:3d:f8:55:ab:2d:92:f4:fe:39:d4:e7:0f:0e +# SHA256 Fingerprint: 2c:e1:cb:0b:f9:d2:f9:e1:02:99:3f:be:21:51:52:c3:b2:dd:0c:ab:de:1c:68:e5:31:9b:83:91:54:db:b7:f5 +-----BEGIN CERTIFICATE----- +MIID3TCCAsWgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBjzELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT +HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAMTKVN0YXJmaWVs +ZCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAw +MFoXDTM3MTIzMTIzNTk1OVowgY8xCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6 +b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFyZmllbGQgVGVj +aG5vbG9naWVzLCBJbmMuMTIwMAYDVQQDEylTdGFyZmllbGQgUm9vdCBDZXJ0aWZp +Y2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAL3twQP89o/8ArFvW59I2Z154qK3A2FWGMNHttfKPTUuiUP3oWmb3ooa/RMg +nLRJdzIpVv257IzdIvpy3Cdhl+72WoTsbhm5iSzchFvVdPtrX8WJpRBSiUZV9Lh1 +HOZ/5FSuS/hVclcCGfgXcVnrHigHdMWdSL5stPSksPNkN3mSwOxGXn/hbVNMYq/N +Hwtjuzqd+/x5AJhhdM8mgkBj87JyahkNmcrUDnXMN/uLicFZ8WJ/X7NfZTD4p7dN +dloedl40wOiWVpmKs/B/pM293DIxfJHP4F8R+GuqSVzRmZTRouNjWwl2tVZi4Ut0 +HZbUJtQIBFnQmA4O5t78w+wfkPECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAO +BgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFHwMMh+n2TB/xH1oo2Kooc6rB1snMA0G +CSqGSIb3DQEBCwUAA4IBAQARWfolTwNvlJk7mh+ChTnUdgWUXuEok21iXQnCoKjU +sHU48TRqneSfioYmUeYs0cYtbpUgSpIB7LiKZ3sx4mcujJUDJi5DnUox9g61DLu3 +4jd/IroAow57UvtruzvE03lRTs2Q9GcHGcg8RnoNAX3FWOdt5oUwF5okxBDgBPfg +8n/Uqgr/Qh037ZTlZFkSIHc40zI+OIF1lnP6aI+xy84fxez6nH7PfrHxBy22/L/K +pL/QlwVKvOoYKAKQvVR4CSFx09F9HdkWsKlhPdAKACL8x3vLCWRFCztAgfd9fDL1 +mMpYjn0q7pBZc2T5NnReJaH1ZgUufzkVqSr7UIuOhWn0 +-----END CERTIFICATE----- + +# Issuer: CN=Starfield Services Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Subject: CN=Starfield Services Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Label: "Starfield Services Root Certificate Authority - G2" +# Serial: 0 +# MD5 Fingerprint: 17:35:74:af:7b:61:1c:eb:f4:f9:3c:e2:ee:40:f9:a2 +# SHA1 Fingerprint: 92:5a:8f:8d:2c:6d:04:e0:66:5f:59:6a:ff:22:d8:63:e8:25:6f:3f +# SHA256 Fingerprint: 56:8d:69:05:a2:c8:87:08:a4:b3:02:51:90:ed:cf:ed:b1:97:4a:60:6a:13:c6:e5:29:0f:cb:2a:e6:3e:da:b5 +-----BEGIN CERTIFICATE----- +MIID7zCCAtegAwIBAgIBADANBgkqhkiG9w0BAQsFADCBmDELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT +HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xOzA5BgNVBAMTMlN0YXJmaWVs +ZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5 +MDkwMTAwMDAwMFoXDTM3MTIzMTIzNTk1OVowgZgxCzAJBgNVBAYTAlVTMRAwDgYD +VQQIEwdBcml6b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFy +ZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTswOQYDVQQDEzJTdGFyZmllbGQgU2Vy +dmljZXMgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBANUMOsQq+U7i9b4Zl1+OiFOxHz/Lz58gE20p +OsgPfTz3a3Y4Y9k2YKibXlwAgLIvWX/2h/klQ4bnaRtSmpDhcePYLQ1Ob/bISdm2 +8xpWriu2dBTrz/sm4xq6HZYuajtYlIlHVv8loJNwU4PahHQUw2eeBGg6345AWh1K +Ts9DkTvnVtYAcMtS7nt9rjrnvDH5RfbCYM8TWQIrgMw0R9+53pBlbQLPLJGmpufe +hRhJfGZOozptqbXuNC66DQO4M99H67FrjSXZm86B0UVGMpZwh94CDklDhbZsc7tk +6mFBrMnUVN+HL8cisibMn1lUaJ/8viovxFUcdUBgF4UCVTmLfwUCAwEAAaNCMEAw +DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJxfAN+q +AdcwKziIorhtSpzyEZGDMA0GCSqGSIb3DQEBCwUAA4IBAQBLNqaEd2ndOxmfZyMI +bw5hyf2E3F/YNoHN2BtBLZ9g3ccaaNnRbobhiCPPE95Dz+I0swSdHynVv/heyNXB +ve6SbzJ08pGCL72CQnqtKrcgfU28elUSwhXqvfdqlS5sdJ/PHLTyxQGjhdByPq1z +qwubdQxtRbeOlKyWN7Wg0I8VRw7j6IPdj/3vQQF3zCepYoUz8jcI73HPdwbeyBkd +iEDPfUYd/x7H4c7/I9vG+o1VTqkC50cRRj70/b17KSa7qWFiNyi2LSr2EIZkyXCn +0q23KXB56jzaYyWf/Wi3MOxw+3WKt21gZ7IeyLnp2KhvAotnDU0mV3HaIPzBSlCN +sSi6 +-----END CERTIFICATE----- + +# Issuer: CN=AffirmTrust Commercial O=AffirmTrust +# Subject: CN=AffirmTrust Commercial O=AffirmTrust +# Label: "AffirmTrust Commercial" +# Serial: 8608355977964138876 +# MD5 Fingerprint: 82:92:ba:5b:ef:cd:8a:6f:a6:3d:55:f9:84:f6:d6:b7 +# SHA1 Fingerprint: f9:b5:b6:32:45:5f:9c:be:ec:57:5f:80:dc:e9:6e:2c:c7:b2:78:b7 +# SHA256 Fingerprint: 03:76:ab:1d:54:c5:f9:80:3c:e4:b2:e2:01:a0:ee:7e:ef:7b:57:b6:36:e8:a9:3c:9b:8d:48:60:c9:6f:5f:a7 +-----BEGIN CERTIFICATE----- +MIIDTDCCAjSgAwIBAgIId3cGJyapsXwwDQYJKoZIhvcNAQELBQAwRDELMAkGA1UE +BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZpcm1UcnVz +dCBDb21tZXJjaWFsMB4XDTEwMDEyOTE0MDYwNloXDTMwMTIzMTE0MDYwNlowRDEL +MAkGA1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZp +cm1UcnVzdCBDb21tZXJjaWFsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEA9htPZwcroRX1BiLLHwGy43NFBkRJLLtJJRTWzsO3qyxPxkEylFf6EqdbDuKP +Hx6GGaeqtS25Xw2Kwq+FNXkyLbscYjfysVtKPcrNcV/pQr6U6Mje+SJIZMblq8Yr +ba0F8PrVC8+a5fBQpIs7R6UjW3p6+DM/uO+Zl+MgwdYoic+U+7lF7eNAFxHUdPAL +MeIrJmqbTFeurCA+ukV6BfO9m2kVrn1OIGPENXY6BwLJN/3HR+7o8XYdcxXyl6S1 +yHp52UKqK39c/s4mT6NmgTWvRLpUHhwwMmWd5jyTXlBOeuM61G7MGvv50jeuJCqr +VwMiKA1JdX+3KNp1v47j3A55MQIDAQABo0IwQDAdBgNVHQ4EFgQUnZPGU4teyq8/ +nx4P5ZmVvCT2lI8wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwDQYJ +KoZIhvcNAQELBQADggEBAFis9AQOzcAN/wr91LoWXym9e2iZWEnStB03TX8nfUYG +XUPGhi4+c7ImfU+TqbbEKpqrIZcUsd6M06uJFdhrJNTxFq7YpFzUf1GO7RgBsZNj +vbz4YYCanrHOQnDiqX0GJX0nof5v7LMeJNrjS1UaADs1tDvZ110w/YETifLCBivt +Z8SOyUOyXGsViQK8YvxO8rUzqrJv0wqiUOP2O+guRMLbZjipM1ZI8W0bM40NjD9g +N53Tym1+NH4Nn3J2ixufcv1SNUFFApYvHLKac0khsUlHRUe072o0EclNmsxZt9YC +nlpOZbWUrhvfKbAW8b8Angc6F2S1BLUjIZkKlTuXfO8= +-----END CERTIFICATE----- + +# Issuer: CN=AffirmTrust Networking O=AffirmTrust +# Subject: CN=AffirmTrust Networking O=AffirmTrust +# Label: "AffirmTrust Networking" +# Serial: 8957382827206547757 +# MD5 Fingerprint: 42:65:ca:be:01:9a:9a:4c:a9:8c:41:49:cd:c0:d5:7f +# SHA1 Fingerprint: 29:36:21:02:8b:20:ed:02:f5:66:c5:32:d1:d6:ed:90:9f:45:00:2f +# SHA256 Fingerprint: 0a:81:ec:5a:92:97:77:f1:45:90:4a:f3:8d:5d:50:9f:66:b5:e2:c5:8f:cd:b5:31:05:8b:0e:17:f3:f0:b4:1b +-----BEGIN CERTIFICATE----- +MIIDTDCCAjSgAwIBAgIIfE8EORzUmS0wDQYJKoZIhvcNAQEFBQAwRDELMAkGA1UE +BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZpcm1UcnVz +dCBOZXR3b3JraW5nMB4XDTEwMDEyOTE0MDgyNFoXDTMwMTIzMTE0MDgyNFowRDEL +MAkGA1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZp +cm1UcnVzdCBOZXR3b3JraW5nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEAtITMMxcua5Rsa2FSoOujz3mUTOWUgJnLVWREZY9nZOIG41w3SfYvm4SEHi3y +YJ0wTsyEheIszx6e/jarM3c1RNg1lho9Nuh6DtjVR6FqaYvZ/Ls6rnla1fTWcbua +kCNrmreIdIcMHl+5ni36q1Mr3Lt2PpNMCAiMHqIjHNRqrSK6mQEubWXLviRmVSRL +QESxG9fhwoXA3hA/Pe24/PHxI1Pcv2WXb9n5QHGNfb2V1M6+oF4nI979ptAmDgAp +6zxG8D1gvz9Q0twmQVGeFDdCBKNwV6gbh+0t+nvujArjqWaJGctB+d1ENmHP4ndG +yH329JKBNv3bNPFyfvMMFr20FQIDAQABo0IwQDAdBgNVHQ4EFgQUBx/S55zawm6i +QLSwelAQUHTEyL0wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwDQYJ +KoZIhvcNAQEFBQADggEBAIlXshZ6qML91tmbmzTCnLQyFE2npN/svqe++EPbkTfO +tDIuUFUaNU52Q3Eg75N3ThVwLofDwR1t3Mu1J9QsVtFSUzpE0nPIxBsFZVpikpzu +QY0x2+c06lkh1QF612S4ZDnNye2v7UsDSKegmQGA3GWjNq5lWUhPgkvIZfFXHeVZ +Lgo/bNjR9eUJtGxUAArgFU2HdW23WJZa3W3SAKD0m0i+wzekujbgfIeFlxoVot4u +olu9rxj5kFDNcFn4J2dHy8egBzp90SxdbBk6ZrV9/ZFvgrG+CJPbFEfxojfHRZ48 +x3evZKiT3/Zpg4Jg8klCNO1aAFSFHBY2kgxc+qatv9s= +-----END CERTIFICATE----- + +# Issuer: CN=AffirmTrust Premium O=AffirmTrust +# Subject: CN=AffirmTrust Premium O=AffirmTrust +# Label: "AffirmTrust Premium" +# Serial: 7893706540734352110 +# MD5 Fingerprint: c4:5d:0e:48:b6:ac:28:30:4e:0a:bc:f9:38:16:87:57 +# SHA1 Fingerprint: d8:a6:33:2c:e0:03:6f:b1:85:f6:63:4f:7d:6a:06:65:26:32:28:27 +# SHA256 Fingerprint: 70:a7:3f:7f:37:6b:60:07:42:48:90:45:34:b1:14:82:d5:bf:0e:69:8e:cc:49:8d:f5:25:77:eb:f2:e9:3b:9a +-----BEGIN CERTIFICATE----- +MIIFRjCCAy6gAwIBAgIIbYwURrGmCu4wDQYJKoZIhvcNAQEMBQAwQTELMAkGA1UE +BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MRwwGgYDVQQDDBNBZmZpcm1UcnVz +dCBQcmVtaXVtMB4XDTEwMDEyOTE0MTAzNloXDTQwMTIzMTE0MTAzNlowQTELMAkG +A1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MRwwGgYDVQQDDBNBZmZpcm1U +cnVzdCBQcmVtaXVtMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxBLf +qV/+Qd3d9Z+K4/as4Tx4mrzY8H96oDMq3I0gW64tb+eT2TZwamjPjlGjhVtnBKAQ +JG9dKILBl1fYSCkTtuG+kU3fhQxTGJoeJKJPj/CihQvL9Cl/0qRY7iZNyaqoe5rZ ++jjeRFcV5fiMyNlI4g0WJx0eyIOFJbe6qlVBzAMiSy2RjYvmia9mx+n/K+k8rNrS +s8PhaJyJ+HoAVt70VZVs+7pk3WKL3wt3MutizCaam7uqYoNMtAZ6MMgpv+0GTZe5 +HMQxK9VfvFMSF5yZVylmd2EhMQcuJUmdGPLu8ytxjLW6OQdJd/zvLpKQBY0tL3d7 +70O/Nbua2Plzpyzy0FfuKE4mX4+QaAkvuPjcBukumj5Rp9EixAqnOEhss/n/fauG +V+O61oV4d7pD6kh/9ti+I20ev9E2bFhc8e6kGVQa9QPSdubhjL08s9NIS+LI+H+S +qHZGnEJlPqQewQcDWkYtuJfzt9WyVSHvutxMAJf7FJUnM7/oQ0dG0giZFmA7mn7S +5u046uwBHjxIVkkJx0w3AJ6IDsBz4W9m6XJHMD4Q5QsDyZpCAGzFlH5hxIrff4Ia +C1nEWTJ3s7xgaVY5/bQGeyzWZDbZvUjthB9+pSKPKrhC9IK31FOQeE4tGv2Bb0TX +OwF0lkLgAOIua+rF7nKsu7/+6qqo+Nz2snmKtmcCAwEAAaNCMEAwHQYDVR0OBBYE +FJ3AZ6YMItkm9UWrpmVSESfYRaxjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ +BAQDAgEGMA0GCSqGSIb3DQEBDAUAA4ICAQCzV00QYk465KzquByvMiPIs0laUZx2 +KI15qldGF9X1Uva3ROgIRL8YhNILgM3FEv0AVQVhh0HctSSePMTYyPtwni94loMg +Nt58D2kTiKV1NpgIpsbfrM7jWNa3Pt668+s0QNiigfV4Py/VpfzZotReBA4Xrf5B +8OWycvpEgjNC6C1Y91aMYj+6QrCcDFx+LmUmXFNPALJ4fqENmS2NuB2OosSw/WDQ +MKSOyARiqcTtNd56l+0OOF6SL5Nwpamcb6d9Ex1+xghIsV5n61EIJenmJWtSKZGc +0jlzCFfemQa0W50QBuHCAKi4HEoCChTQwUHK+4w1IX2COPKpVJEZNZOUbWo6xbLQ +u4mGk+ibyQ86p3q4ofB4Rvr8Ny/lioTz3/4E2aFooC8k4gmVBtWVyuEklut89pMF +u+1z6S3RdTnX5yTb2E5fQ4+e0BQ5v1VwSJlXMbSc7kqYA5YwH2AG7hsj/oFgIxpH +YoWlzBk0gG+zrBrjn/B7SK3VAdlntqlyk+otZrWyuOQ9PLLvTIzq6we/qzWaVYa8 +GKa1qF60g2xraUDTn9zxw2lrueFtCfTxqlB2Cnp9ehehVZZCmTEJ3WARjQUwfuaO +RtGdFNrHF+QFlozEJLUbzxQHskD4o55BhrwE0GuWyCqANP2/7waj3VjFhT0+j/6e +KeC2uAloGRwYQw== +-----END CERTIFICATE----- + +# Issuer: CN=AffirmTrust Premium ECC O=AffirmTrust +# Subject: CN=AffirmTrust Premium ECC O=AffirmTrust +# Label: "AffirmTrust Premium ECC" +# Serial: 8401224907861490260 +# MD5 Fingerprint: 64:b0:09:55:cf:b1:d5:99:e2:be:13:ab:a6:5d:ea:4d +# SHA1 Fingerprint: b8:23:6b:00:2f:1d:16:86:53:01:55:6c:11:a4:37:ca:eb:ff:c3:bb +# SHA256 Fingerprint: bd:71:fd:f6:da:97:e4:cf:62:d1:64:7a:dd:25:81:b0:7d:79:ad:f8:39:7e:b4:ec:ba:9c:5e:84:88:82:14:23 +-----BEGIN CERTIFICATE----- +MIIB/jCCAYWgAwIBAgIIdJclisc/elQwCgYIKoZIzj0EAwMwRTELMAkGA1UEBhMC +VVMxFDASBgNVBAoMC0FmZmlybVRydXN0MSAwHgYDVQQDDBdBZmZpcm1UcnVzdCBQ +cmVtaXVtIEVDQzAeFw0xMDAxMjkxNDIwMjRaFw00MDEyMzExNDIwMjRaMEUxCzAJ +BgNVBAYTAlVTMRQwEgYDVQQKDAtBZmZpcm1UcnVzdDEgMB4GA1UEAwwXQWZmaXJt +VHJ1c3QgUHJlbWl1bSBFQ0MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQNMF4bFZ0D +0KF5Nbc6PJJ6yhUczWLznCZcBz3lVPqj1swS6vQUX+iOGasvLkjmrBhDeKzQN8O9 +ss0s5kfiGuZjuD0uL3jET9v0D6RoTFVya5UdThhClXjMNzyR4ptlKymjQjBAMB0G +A1UdDgQWBBSaryl6wBE1NSZRMADDav5A1a7WPDAPBgNVHRMBAf8EBTADAQH/MA4G +A1UdDwEB/wQEAwIBBjAKBggqhkjOPQQDAwNnADBkAjAXCfOHiFBar8jAQr9HX/Vs +aobgxCd05DhT1wV/GzTjxi+zygk8N53X57hG8f2h4nECMEJZh0PUUd+60wkyWs6I +flc9nF9Ca/UHLbXwgpP5WW+uZPpY5Yse42O+tYHNbwKMeQ== +-----END CERTIFICATE----- + +# Issuer: CN=StartCom Certification Authority O=StartCom Ltd. OU=Secure Digital Certificate Signing +# Subject: CN=StartCom Certification Authority O=StartCom Ltd. OU=Secure Digital Certificate Signing +# Label: "StartCom Certification Authority" +# Serial: 45 +# MD5 Fingerprint: c9:3b:0d:84:41:fc:a4:76:79:23:08:57:de:10:19:16 +# SHA1 Fingerprint: a3:f1:33:3f:e2:42:bf:cf:c5:d1:4e:8f:39:42:98:40:68:10:d1:a0 +# SHA256 Fingerprint: e1:78:90:ee:09:a3:fb:f4:f4:8b:9c:41:4a:17:d6:37:b7:a5:06:47:e9:bc:75:23:22:72:7f:cc:17:42:a9:11 +-----BEGIN CERTIFICATE----- +MIIHhzCCBW+gAwIBAgIBLTANBgkqhkiG9w0BAQsFADB9MQswCQYDVQQGEwJJTDEW +MBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0YWwg +Q2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwHhcNMDYwOTE3MTk0NjM3WhcNMzYwOTE3MTk0NjM2WjB9 +MQswCQYDVQQGEwJJTDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMi +U2VjdXJlIERpZ2l0YWwgQ2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3Rh +cnRDb20gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUA +A4ICDwAwggIKAoICAQDBiNsJvGxGfHiflXu1M5DycmLWwTYgIiRezul38kMKogZk +pMyONvg45iPwbm2xPN1yo4UcodM9tDMr0y+v/uqwQVlntsQGfQqedIXWeUyAN3rf +OQVSWff0G0ZDpNKFhdLDcfN1YjS6LIp/Ho/u7TTQEceWzVI9ujPW3U3eCztKS5/C +Ji/6tRYccjV3yjxd5srhJosaNnZcAdt0FCX+7bWgiA/deMotHweXMAEtcnn6RtYT +Kqi5pquDSR3l8u/d5AGOGAqPY1MWhWKpDhk6zLVmpsJrdAfkK+F2PrRt2PZE4XNi +HzvEvqBTViVsUQn3qqvKv3b9bZvzndu/PWa8DFaqr5hIlTpL36dYUNk4dalb6kMM +Av+Z6+hsTXBbKWWc3apdzK8BMewM69KN6Oqce+Zu9ydmDBpI125C4z/eIT574Q1w ++2OqqGwaVLRcJXrJosmLFqa7LH4XXgVNWG4SHQHuEhANxjJ/GP/89PrNbpHoNkm+ +Gkhpi8KWTRoSsmkXwQqQ1vp5Iki/untp+HDH+no32NgN0nZPV/+Qt+OR0t3vwmC3 +Zzrd/qqc8NSLf3Iizsafl7b4r4qgEKjZ+xjGtrVcUjyJthkqcwEKDwOzEmDyei+B +26Nu/yYwl/WL3YlXtq09s68rxbd2AvCl1iuahhQqcvbjM4xdCUsT37uMdBNSSwID +AQABo4ICEDCCAgwwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYD +VR0OBBYEFE4L7xqkQFulF2mHMMo0aEPQQa7yMB8GA1UdIwQYMBaAFE4L7xqkQFul +F2mHMMo0aEPQQa7yMIIBWgYDVR0gBIIBUTCCAU0wggFJBgsrBgEEAYG1NwEBATCC +ATgwLgYIKwYBBQUHAgEWImh0dHA6Ly93d3cuc3RhcnRzc2wuY29tL3BvbGljeS5w +ZGYwNAYIKwYBBQUHAgEWKGh0dHA6Ly93d3cuc3RhcnRzc2wuY29tL2ludGVybWVk +aWF0ZS5wZGYwgc8GCCsGAQUFBwICMIHCMCcWIFN0YXJ0IENvbW1lcmNpYWwgKFN0 +YXJ0Q29tKSBMdGQuMAMCAQEagZZMaW1pdGVkIExpYWJpbGl0eSwgcmVhZCB0aGUg +c2VjdGlvbiAqTGVnYWwgTGltaXRhdGlvbnMqIG9mIHRoZSBTdGFydENvbSBDZXJ0 +aWZpY2F0aW9uIEF1dGhvcml0eSBQb2xpY3kgYXZhaWxhYmxlIGF0IGh0dHA6Ly93 +d3cuc3RhcnRzc2wuY29tL3BvbGljeS5wZGYwEQYJYIZIAYb4QgEBBAQDAgAHMDgG +CWCGSAGG+EIBDQQrFilTdGFydENvbSBGcmVlIFNTTCBDZXJ0aWZpY2F0aW9uIEF1 +dGhvcml0eTANBgkqhkiG9w0BAQsFAAOCAgEAjo/n3JR5fPGFf59Jb2vKXfuM/gTF +wWLRfUKKvFO3lANmMD+x5wqnUCBVJX92ehQN6wQOQOY+2IirByeDqXWmN3PH/UvS +Ta0XQMhGvjt/UfzDtgUx3M2FIk5xt/JxXrAaxrqTi3iSSoX4eA+D/i+tLPfkpLst +0OcNOrg+zvZ49q5HJMqjNTbOx8aHmNrs++myziebiMMEofYLWWivydsQD032ZGNc +pRJvkrKTlMeIFw6Ttn5ii5B/q06f/ON1FE8qMt9bDeD1e5MNq6HPh+GlBEXoPBKl +CcWw0bdT82AUuoVpaiF8H3VhFyAXe2w7QSlc4axa0c2Mm+tgHRns9+Ww2vl5GKVF +P0lDV9LdJNUso/2RjSe15esUBppMeyG7Oq0wBhjA2MFrLH9ZXF2RsXAiV+uKa0hK +1Q8p7MZAwC+ITGgBF3f0JBlPvfrhsiAhS90a2Cl9qrjeVOwhVYBsHvUwyKMQ5bLm +KhQxw4UtjJixhlpPiVktucf3HMiKf8CdBUrmQk9io20ppB+Fq9vlgcitKj1MXVuE +JnHEhV5xJMqlG2zYYdMa4FTbzrqpMrUi9nNBCV24F10OD5mQ1kfabwo6YigUZ4LZ +8dCAWZvLMdibD4x3TrVoivJs9iQOLWxwxXPR3hTQcY+203sC9uO41Alua551hDnm +fyWl8kgAwKQB2j8= +-----END CERTIFICATE----- + +# Issuer: CN=StartCom Certification Authority G2 O=StartCom Ltd. +# Subject: CN=StartCom Certification Authority G2 O=StartCom Ltd. +# Label: "StartCom Certification Authority G2" +# Serial: 59 +# MD5 Fingerprint: 78:4b:fb:9e:64:82:0a:d3:b8:4c:62:f3:64:f2:90:64 +# SHA1 Fingerprint: 31:f1:fd:68:22:63:20:ee:c6:3b:3f:9d:ea:4a:3e:53:7c:7c:39:17 +# SHA256 Fingerprint: c7:ba:65:67:de:93:a7:98:ae:1f:aa:79:1e:71:2d:37:8f:ae:1f:93:c4:39:7f:ea:44:1b:b7:cb:e6:fd:59:95 +-----BEGIN CERTIFICATE----- +MIIFYzCCA0ugAwIBAgIBOzANBgkqhkiG9w0BAQsFADBTMQswCQYDVQQGEwJJTDEW +MBQGA1UEChMNU3RhcnRDb20gTHRkLjEsMCoGA1UEAxMjU3RhcnRDb20gQ2VydGlm +aWNhdGlvbiBBdXRob3JpdHkgRzIwHhcNMTAwMTAxMDEwMDAxWhcNMzkxMjMxMjM1 +OTAxWjBTMQswCQYDVQQGEwJJTDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjEsMCoG +A1UEAxMjU3RhcnRDb20gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgRzIwggIiMA0G +CSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC2iTZbB7cgNr2Cu+EWIAOVeq8Oo1XJ +JZlKxdBWQYeQTSFgpBSHO839sj60ZwNq7eEPS8CRhXBF4EKe3ikj1AENoBB5uNsD +vfOpL9HG4A/LnooUCri99lZi8cVytjIl2bLzvWXFDSxu1ZJvGIsAQRSCb0AgJnoo +D/Uefyf3lLE3PbfHkffiAez9lInhzG7TNtYKGXmu1zSCZf98Qru23QumNK9LYP5/ +Q0kGi4xDuFby2X8hQxfqp0iVAXV16iulQ5XqFYSdCI0mblWbq9zSOdIxHWDirMxW +RST1HFSr7obdljKF+ExP6JV2tgXdNiNnvP8V4so75qbsO+wmETRIjfaAKxojAuuK +HDp2KntWFhxyKrOq42ClAJ8Em+JvHhRYW6Vsi1g8w7pOOlz34ZYrPu8HvKTlXcxN +nw3h3Kq74W4a7I/htkxNeXJdFzULHdfBR9qWJODQcqhaX2YtENwvKhOuJv4KHBnM +0D4LnMgJLvlblnpHnOl68wVQdJVznjAJ85eCXuaPOQgeWeU1FEIT/wCc976qUM/i +UUjXuG+v+E5+M5iSFGI6dWPPe/regjupuznixL0sAA7IF6wT700ljtizkC+p2il9 +Ha90OrInwMEePnWjFqmveiJdnxMaz6eg6+OGCtP95paV1yPIN93EfKo2rJgaErHg +TuixO/XWb/Ew1wIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQE +AwIBBjAdBgNVHQ4EFgQUS8W0QGutHLOlHGVuRjaJhwUMDrYwDQYJKoZIhvcNAQEL +BQADggIBAHNXPyzVlTJ+N9uWkusZXn5T50HsEbZH77Xe7XRcxfGOSeD8bpkTzZ+K +2s06Ctg6Wgk/XzTQLwPSZh0avZyQN8gMjgdalEVGKua+etqhqaRpEpKwfTbURIfX +UfEpY9Z1zRbkJ4kd+MIySP3bmdCPX1R0zKxnNBFi2QwKN4fRoxdIjtIXHfbX/dtl +6/2o1PXWT6RbdejF0mCy2wl+JYt7ulKSnj7oxXehPOBKc2thz4bcQ///If4jXSRK +9dNtD2IEBVeC2m6kMyV5Sy5UGYvMLD0w6dEG/+gyRr61M3Z3qAFdlsHB1b6uJcDJ +HgoJIIihDsnzb02CVAAgp9KP5DlUFy6NHrgbuxu9mk47EDTcnIhT76IxW1hPkWLI +wpqazRVdOKnWvvgTtZ8SafJQYqz7Fzf07rh1Z2AQ+4NQ+US1dZxAF7L+/XldblhY +XzD8AK6vM8EOTmy6p6ahfzLbOOCxchcKK5HsamMm7YnUeMx0HgX4a/6ManY5Ka5l +IxKVCCIcl85bBu4M4ru8H0ST9tg4RQUh7eStqxK2A6RCLi3ECToDZ2mEmuFZkIoo +hdVddLHRDiBYmxOlsGOm7XtH/UVVMKTumtTm4ofvmMkyghEpIrwACjFeLQ/Ajulr +so8uBtjRkcfGEvRM/TAXw8HaOFvjqermobp573PYtlNXLfbQ4ddI +-----END CERTIFICATE----- From 9e564c49b39230a2d9f0db30a8bb5137f9e6783e Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 3 Sep 2014 16:46:42 +0200 Subject: [PATCH 096/202] Update library: requests --- libs/requests/__init__.py | 4 +- libs/requests/adapters.py | 83 ++++-- libs/requests/auth.py | 8 +- libs/requests/certs.py | 13 +- libs/requests/compat.py | 2 - libs/requests/exceptions.py | 17 +- libs/requests/models.py | 25 +- libs/requests/packages/README.rst | 8 - libs/requests/packages/urllib3/__init__.py | 24 +- libs/requests/packages/urllib3/_collections.py | 8 +- libs/requests/packages/urllib3/connection.py | 154 +++++++----- libs/requests/packages/urllib3/connectionpool.py | 259 +++++++++++-------- libs/requests/packages/urllib3/contrib/ntlmpool.py | 6 - .../requests/packages/urllib3/contrib/pyopenssl.py | 232 ++++------------- libs/requests/packages/urllib3/exceptions.py | 56 ++++- libs/requests/packages/urllib3/fields.py | 26 +- libs/requests/packages/urllib3/filepost.py | 15 +- .../packages/urllib3/packages/ordered_dict.py | 1 - libs/requests/packages/urllib3/poolmanager.py | 43 ++-- libs/requests/packages/urllib3/request.py | 44 ++-- libs/requests/packages/urllib3/response.py | 97 ++++--- libs/requests/packages/urllib3/util/__init__.py | 9 +- libs/requests/packages/urllib3/util/connection.py | 58 ++++- libs/requests/packages/urllib3/util/request.py | 19 +- libs/requests/packages/urllib3/util/response.py | 17 +- libs/requests/packages/urllib3/util/retry.py | 279 +++++++++++++++++++++ libs/requests/packages/urllib3/util/ssl_.py | 5 +- libs/requests/packages/urllib3/util/timeout.py | 98 ++++---- libs/requests/packages/urllib3/util/url.py | 19 +- libs/requests/sessions.py | 75 +++--- libs/requests/status_codes.py | 3 +- libs/requests/structures.py | 2 +- libs/requests/utils.py | 3 +- 33 files changed, 1068 insertions(+), 644 deletions(-) delete mode 100644 libs/requests/packages/README.rst create mode 100644 libs/requests/packages/urllib3/util/retry.py diff --git a/libs/requests/__init__.py b/libs/requests/__init__.py index bba1900..33a2b27 100644 --- a/libs/requests/__init__.py +++ b/libs/requests/__init__.py @@ -42,8 +42,8 @@ is at . """ __title__ = 'requests' -__version__ = '2.3.0' -__build__ = 0x020300 +__version__ = '2.4.0' +__build__ = 0x020400 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2014 Kenneth Reitz' diff --git a/libs/requests/adapters.py b/libs/requests/adapters.py index eb7a2d2..3c1e979 100644 --- a/libs/requests/adapters.py +++ b/libs/requests/adapters.py @@ -11,20 +11,23 @@ and maintain connections. import socket from .models import Response +from .packages.urllib3 import Retry from .packages.urllib3.poolmanager import PoolManager, proxy_from_url from .packages.urllib3.response import HTTPResponse from .packages.urllib3.util import Timeout as TimeoutSauce -from .compat import urlparse, basestring, urldefrag, unquote +from .compat import urlparse, basestring, urldefrag from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, prepend_scheme_if_needed, get_auth_from_url) from .structures import CaseInsensitiveDict -from .packages.urllib3.exceptions import MaxRetryError -from .packages.urllib3.exceptions import TimeoutError -from .packages.urllib3.exceptions import SSLError as _SSLError +from .packages.urllib3.exceptions import ConnectTimeoutError from .packages.urllib3.exceptions import HTTPError as _HTTPError +from .packages.urllib3.exceptions import MaxRetryError from .packages.urllib3.exceptions import ProxyError as _ProxyError +from .packages.urllib3.exceptions import ReadTimeoutError +from .packages.urllib3.exceptions import SSLError as _SSLError from .cookies import extract_cookies_to_jar -from .exceptions import ConnectionError, Timeout, SSLError, ProxyError +from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, + ProxyError) from .auth import _basic_auth_str DEFAULT_POOLBLOCK = False @@ -101,14 +104,17 @@ class HTTPAdapter(BaseAdapter): self.init_poolmanager(self._pool_connections, self._pool_maxsize, block=self._pool_block) - def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK): - """Initializes a urllib3 PoolManager. This method should not be called - from user code, and is only exposed for use when subclassing the + def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs): + """Initializes a urllib3 PoolManager. + + This method should not be called from user code, and is only + exposed for use when subclassing the :class:`HTTPAdapter `. :param connections: The number of urllib3 connection pools to cache. :param maxsize: The maximum number of connections to save in the pool. :param block: Block when no free connections are available. + :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager. """ # save these values for pickling self._pool_connections = connections @@ -116,7 +122,30 @@ class HTTPAdapter(BaseAdapter): self._pool_block = block self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, - block=block) + block=block, **pool_kwargs) + + def proxy_manager_for(self, proxy, **proxy_kwargs): + """Return urllib3 ProxyManager for the given proxy. + + This method should not be called from user code, and is only + exposed for use when subclassing the + :class:`HTTPAdapter `. + + :param proxy: The proxy to return a urllib3 ProxyManager for. + :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager. + :returns: ProxyManager + """ + if not proxy in self.proxy_manager: + proxy_headers = self.proxy_headers(proxy) + self.proxy_manager[proxy] = proxy_from_url( + proxy, + proxy_headers=proxy_headers, + num_pools=self._pool_connections, + maxsize=self._pool_maxsize, + block=self._pool_block, + **proxy_kwargs) + + return self.proxy_manager[proxy] def cert_verify(self, conn, url, verify, cert): """Verify a SSL certificate. This method should not be called from user @@ -204,17 +233,8 @@ class HTTPAdapter(BaseAdapter): if proxy: proxy = prepend_scheme_if_needed(proxy, 'http') - proxy_headers = self.proxy_headers(proxy) - - if not proxy in self.proxy_manager: - self.proxy_manager[proxy] = proxy_from_url( - proxy, - proxy_headers=proxy_headers, - num_pools=self._pool_connections, - maxsize=self._pool_maxsize, - block=self._pool_block) - - conn = self.proxy_manager[proxy].connection_from_url(url) + proxy_manager = self.proxy_manager_for(proxy) + conn = proxy_manager.connection_from_url(url) else: # Only scheme should be lower case parsed = urlparse(url) @@ -297,6 +317,7 @@ class HTTPAdapter(BaseAdapter): :param request: The :class:`PreparedRequest ` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) The timeout on the request. + :type timeout: float or tuple (connect timeout, read timeout), eg (3.1, 20) :param verify: (optional) Whether to verify SSL certificates. :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. @@ -310,7 +331,18 @@ class HTTPAdapter(BaseAdapter): chunked = not (request.body is None or 'Content-Length' in request.headers) - timeout = TimeoutSauce(connect=timeout, read=timeout) + if isinstance(timeout, tuple): + try: + connect, read = timeout + timeout = TimeoutSauce(connect=connect, read=read) + except ValueError as e: + # this may raise a string formatting error. + err = ("Invalid timeout {0}. Pass a (connect, read) " + "timeout tuple, or a single float to set " + "both timeouts to the same value".format(timeout)) + raise ValueError(err) + else: + timeout = TimeoutSauce(connect=timeout, read=timeout) try: if not chunked: @@ -323,7 +355,7 @@ class HTTPAdapter(BaseAdapter): assert_same_host=False, preload_content=False, decode_content=False, - retries=self.max_retries, + retries=Retry(self.max_retries, read=False), timeout=timeout ) @@ -372,6 +404,9 @@ class HTTPAdapter(BaseAdapter): raise ConnectionError(sockerr, request=request) except MaxRetryError as e: + if isinstance(e.reason, ConnectTimeoutError): + raise ConnectTimeout(e, request=request) + raise ConnectionError(e, request=request) except _ProxyError as e: @@ -380,8 +415,8 @@ class HTTPAdapter(BaseAdapter): except (_SSLError, _HTTPError) as e: if isinstance(e, _SSLError): raise SSLError(e, request=request) - elif isinstance(e, TimeoutError): - raise Timeout(e, request=request) + elif isinstance(e, ReadTimeoutError): + raise ReadTimeout(e, request=request) else: raise diff --git a/libs/requests/auth.py b/libs/requests/auth.py index 9f831b7..9b6426d 100644 --- a/libs/requests/auth.py +++ b/libs/requests/auth.py @@ -16,7 +16,7 @@ from base64 import b64encode from .compat import urlparse, str from .cookies import extract_cookies_to_jar -from .utils import parse_dict_header +from .utils import parse_dict_header, to_native_string CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded' CONTENT_TYPE_MULTI_PART = 'multipart/form-data' @@ -25,7 +25,11 @@ CONTENT_TYPE_MULTI_PART = 'multipart/form-data' def _basic_auth_str(username, password): """Returns a Basic Auth string.""" - return 'Basic ' + b64encode(('%s:%s' % (username, password)).encode('latin1')).strip().decode('latin1') + authstr = 'Basic ' + to_native_string( + b64encode(('%s:%s' % (username, password)).encode('latin1')).strip() + ) + + return authstr class AuthBase(object): diff --git a/libs/requests/certs.py b/libs/requests/certs.py index bc00826..07e6475 100644 --- a/libs/requests/certs.py +++ b/libs/requests/certs.py @@ -11,14 +11,15 @@ If you are packaging Requests, e.g., for a Linux distribution or a managed environment, you can change the definition of where() to return a separately packaged CA bundle. """ - import os.path - -def where(): - """Return the preferred certificate bundle.""" - # vendored bundle inside Requests - return os.path.join(os.path.dirname(__file__), 'cacert.pem') +try: + from certifi import where +except ImportError: + def where(): + """Return the preferred certificate bundle.""" + # vendored bundle inside Requests + return os.path.join(os.path.dirname(__file__), 'cacert.pem') if __name__ == '__main__': print(where()) diff --git a/libs/requests/compat.py b/libs/requests/compat.py index 84d703b..be5a1ed 100644 --- a/libs/requests/compat.py +++ b/libs/requests/compat.py @@ -92,7 +92,6 @@ if is_py2: from Cookie import Morsel from StringIO import StringIO from .packages.urllib3.packages.ordered_dict import OrderedDict - from httplib import IncompleteRead builtin_str = str bytes = str @@ -108,7 +107,6 @@ elif is_py3: from http.cookies import Morsel from io import StringIO from collections import OrderedDict - from http.client import IncompleteRead builtin_str = str str = str diff --git a/libs/requests/exceptions.py b/libs/requests/exceptions.py index a4ee9d6..6dbd98a 100644 --- a/libs/requests/exceptions.py +++ b/libs/requests/exceptions.py @@ -44,7 +44,22 @@ class SSLError(ConnectionError): class Timeout(RequestException): - """The request timed out.""" + """The request timed out. + + Catching this error will catch both :exc:`ConnectTimeout` and + :exc:`ReadTimeout` errors. + """ + + +class ConnectTimeout(ConnectionError, Timeout): + """The request timed out while trying to connect to the server. + + Requests that produce this error are safe to retry + """ + + +class ReadTimeout(Timeout): + """The server did not send any data in the allotted amount of time.""" class URLRequired(RequestException): diff --git a/libs/requests/models.py b/libs/requests/models.py index 5aad8ce..03ff627 100644 --- a/libs/requests/models.py +++ b/libs/requests/models.py @@ -19,26 +19,28 @@ from .cookies import cookiejar_from_dict, get_cookie_header from .packages.urllib3.fields import RequestField from .packages.urllib3.filepost import encode_multipart_formdata from .packages.urllib3.util import parse_url -from .packages.urllib3.exceptions import DecodeError +from .packages.urllib3.exceptions import ( + DecodeError, ReadTimeoutError, ProtocolError) from .exceptions import ( HTTPError, RequestException, MissingSchema, InvalidURL, - ChunkedEncodingError, ContentDecodingError) + ChunkedEncodingError, ContentDecodingError, ConnectionError) from .utils import ( guess_filename, get_auth_from_url, requote_uri, stream_decode_response_unicode, to_key_val_list, parse_header_links, iter_slices, guess_json_utf, super_len, to_native_string) from .compat import ( cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO, - is_py2, chardet, json, builtin_str, basestring, IncompleteRead) + is_py2, chardet, json, builtin_str, basestring) from .status_codes import codes #: The set of HTTP status codes that indicate an automatically #: processable redirect. REDIRECT_STATI = ( - codes.moved, # 301 - codes.found, # 302 - codes.other, # 303 - codes.temporary_moved, # 307 + codes.moved, # 301 + codes.found, # 302 + codes.other, # 303 + codes.temporary_redirect, # 307 + codes.permanent_redirect, # 308 ) DEFAULT_REDIRECT_LIMIT = 30 CONTENT_CHUNK_SIZE = 10 * 1024 @@ -610,6 +612,11 @@ class Response(object): return ('location' in self.headers and self.status_code in REDIRECT_STATI) @property + def is_permanent_redirect(self): + """True if this Response one of the permanant versions of redirect""" + return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect)) + + @property def apparent_encoding(self): """The apparent encoding, provided by the chardet library""" return chardet.detect(self.content)['encoding'] @@ -630,10 +637,12 @@ class Response(object): try: for chunk in self.raw.stream(chunk_size, decode_content=True): yield chunk - except IncompleteRead as e: + except ProtocolError as e: raise ChunkedEncodingError(e) except DecodeError as e: raise ContentDecodingError(e) + except ReadTimeoutError as e: + raise ConnectionError(e) except AttributeError: # Standard file-like object. while True: diff --git a/libs/requests/packages/README.rst b/libs/requests/packages/README.rst deleted file mode 100644 index c42f376..0000000 --- a/libs/requests/packages/README.rst +++ /dev/null @@ -1,8 +0,0 @@ -If you are planning to submit a pull request to requests with any changes in -this library do not go any further. These are independent libraries which we -vendor into requests. Any changes necessary to these libraries must be made in -them and submitted as separate pull requests to those libraries. - -urllib3 pull requests go here: https://github.com/shazow/urllib3 - -chardet pull requests go here: https://github.com/chardet/chardet diff --git a/libs/requests/packages/urllib3/__init__.py b/libs/requests/packages/urllib3/__init__.py index 73071f7..4b36b5a 100644 --- a/libs/requests/packages/urllib3/__init__.py +++ b/libs/requests/packages/urllib3/__init__.py @@ -1,9 +1,3 @@ -# urllib3/__init__.py -# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -# -# This module is part of urllib3 and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - """ urllib3 - Thread-safe connection pooling and re-using. """ @@ -23,7 +17,10 @@ from . import exceptions from .filepost import encode_multipart_formdata from .poolmanager import PoolManager, ProxyManager, proxy_from_url from .response import HTTPResponse -from .util import make_headers, get_host, Timeout +from .util.request import make_headers +from .util.url import get_host +from .util.timeout import Timeout +from .util.retry import Retry # Set default logging handler to avoid "No handler found" warnings. @@ -51,8 +48,19 @@ def add_stderr_logger(level=logging.DEBUG): handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) logger.addHandler(handler) logger.setLevel(level) - logger.debug('Added an stderr logging handler to logger: %s' % __name__) + logger.debug('Added a stderr logging handler to logger: %s' % __name__) return handler # ... Clean up. del NullHandler + + +# Set security warning to only go off once by default. +import warnings +warnings.simplefilter('module', exceptions.SecurityWarning) + +def disable_warnings(category=exceptions.HTTPWarning): + """ + Helper for quickly disabling all urllib3 warnings. + """ + warnings.simplefilter('ignore', category) diff --git a/libs/requests/packages/urllib3/_collections.py b/libs/requests/packages/urllib3/_collections.py index 9cea3a4..d77ebb8 100644 --- a/libs/requests/packages/urllib3/_collections.py +++ b/libs/requests/packages/urllib3/_collections.py @@ -1,9 +1,3 @@ -# urllib3/_collections.py -# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -# -# This module is part of urllib3 and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - from collections import Mapping, MutableMapping try: from threading import RLock @@ -116,7 +110,7 @@ class HTTPHeaderDict(MutableMapping): A ``dict`` like container for storing HTTP Headers. Field names are stored and compared case-insensitively in compliance with - RFC 2616. Iteration provides the first case-sensitive key seen for each + RFC 7230. Iteration provides the first case-sensitive key seen for each case-insensitive pair. Using ``__setitem__`` syntax overwrites fields that compare equal diff --git a/libs/requests/packages/urllib3/connection.py b/libs/requests/packages/urllib3/connection.py index 5feb332..c6e1959 100644 --- a/libs/requests/packages/urllib3/connection.py +++ b/libs/requests/packages/urllib3/connection.py @@ -1,95 +1,133 @@ -# urllib3/connection.py -# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -# -# This module is part of urllib3 and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - +import datetime import sys import socket from socket import timeout as SocketTimeout +import warnings -try: # Python 3 +try: # Python 3 from http.client import HTTPConnection as _HTTPConnection, HTTPException except ImportError: from httplib import HTTPConnection as _HTTPConnection, HTTPException + class DummyConnection(object): "Used to detect a failed ConnectionCls import." pass -try: # Compiled with SSL? - ssl = None + +try: # Compiled with SSL? HTTPSConnection = DummyConnection + import ssl + BaseSSLError = ssl.SSLError +except (ImportError, AttributeError): # Platform-specific: No SSL. + ssl = None class BaseSSLError(BaseException): pass - try: # Python 3 - from http.client import HTTPSConnection as _HTTPSConnection - except ImportError: - from httplib import HTTPSConnection as _HTTPSConnection - - import ssl - BaseSSLError = ssl.SSLError - -except (ImportError, AttributeError): # Platform-specific: No SSL. - pass from .exceptions import ( ConnectTimeoutError, + SystemTimeWarning, ) from .packages.ssl_match_hostname import match_hostname from .packages import six -from .util import ( - assert_fingerprint, + +from .util.ssl_ import ( resolve_cert_reqs, resolve_ssl_version, ssl_wrap_socket, + assert_fingerprint, ) +from .util import connection + port_by_scheme = { 'http': 80, 'https': 443, } +RECENT_DATE = datetime.date(2014, 1, 1) + class HTTPConnection(_HTTPConnection, object): """ Based on httplib.HTTPConnection but provides an extra constructor backwards-compatibility layer between older and newer Pythons. + + Additional keyword parameters are used to configure attributes of the connection. + Accepted parameters include: + + - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool` + - ``source_address``: Set the source address for the current connection. + + .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x + + - ``socket_options``: Set specific options on the underlying socket. If not specified, then + defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling + Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy. + + For example, if you wish to enable TCP Keep Alive in addition to the defaults, + you might pass:: + + HTTPConnection.default_socket_options + [ + (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), + ] + + Or you may want to disable the defaults by passing an empty list (e.g., ``[]``). """ default_port = port_by_scheme['http'] - # By default, disable Nagle's Algorithm. - tcp_nodelay = 1 + #: Disable Nagle's algorithm by default. + #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]`` + default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)] + + #: Whether this connection verifies the host's certificate. + is_verified = False def __init__(self, *args, **kw): if six.PY3: # Python 3 kw.pop('strict', None) - if sys.version_info < (2, 7): # Python 2.6 and older - kw.pop('source_address', None) # Pre-set source_address in case we have an older Python like 2.6. self.source_address = kw.get('source_address') + if sys.version_info < (2, 7): # Python 2.6 + # _HTTPConnection on Python 2.6 will balk at this keyword arg, but + # not newer versions. We can still use it when creating a + # connection though, so we pop it *after* we have saved it as + # self.source_address. + kw.pop('source_address', None) + + #: The socket options provided by the user. If no options are + #: provided, we use the default options. + self.socket_options = kw.pop('socket_options', self.default_socket_options) + # Superclass also sets self.source_address in Python 2.7+. - _HTTPConnection.__init__(self, *args, **kw) + _HTTPConnection.__init__(self, *args, **kw) def _new_conn(self): """ Establish a socket connection and set nodelay settings on it. - :return: a new socket connection + :return: New socket connection. """ - extra_args = [] - if self.source_address: # Python 2.7+ - extra_args.append(self.source_address) + extra_kw = {} + if self.source_address: + extra_kw['source_address'] = self.source_address - conn = socket.create_connection( - (self.host, self.port), self.timeout, *extra_args) - conn.setsockopt( - socket.IPPROTO_TCP, socket.TCP_NODELAY, self.tcp_nodelay) + if self.socket_options: + extra_kw['socket_options'] = self.socket_options + + try: + conn = connection.create_connection( + (self.host, self.port), self.timeout, **extra_kw) + + except SocketTimeout: + raise ConnectTimeoutError( + self, "Connection to %s timed out. (connect timeout=%s)" % + (self.host, self.timeout)) return conn @@ -101,6 +139,8 @@ class HTTPConnection(_HTTPConnection, object): if getattr(self, '_tunnel_host', None): # TODO: Fix tunnel so it doesn't depend on self.sock state. self._tunnel() + # Mark this connection as not reusable + self.auto_open = 0 def connect(self): conn = self._new_conn() @@ -137,7 +177,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): cert_reqs = None ca_certs = None ssl_version = None - conn_kw = {} + assert_fingerprint = None def set_cert(self, key_file=None, cert_file=None, cert_reqs=None, ca_certs=None, @@ -152,18 +192,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): def connect(self): # Add certificate verification - - try: - sock = socket.create_connection( - address=(self.host, self.port), timeout=self.timeout, - **self.conn_kw) - except SocketTimeout: - raise ConnectTimeoutError( - self, "Connection to %s timed out. (connect timeout=%s)" % - (self.host, self.timeout)) - - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, - self.tcp_nodelay) + conn = self._new_conn() resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs) resolved_ssl_version = resolve_ssl_version(self.ssl_version) @@ -173,29 +202,42 @@ class VerifiedHTTPSConnection(HTTPSConnection): # _tunnel_host was added in Python 2.6.3 # (See: http://hg.python.org/cpython/rev/0f57b30a152f) - self.sock = sock + self.sock = conn # Calls self._set_hostport(), so self.host is # self._tunnel_host below. self._tunnel() + # Mark this connection as not reusable + self.auto_open = 0 # Override the host with the one we're requesting data from. hostname = self._tunnel_host + is_time_off = datetime.date.today() < RECENT_DATE + if is_time_off: + warnings.warn(( + 'System time is way off (before {0}). This will probably ' + 'lead to SSL verification errors').format(RECENT_DATE), + SystemTimeWarning + ) + # Wrap socket using verification with the root certs in # trusted_root_certs - self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file, + self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file, cert_reqs=resolved_cert_reqs, ca_certs=self.ca_certs, server_hostname=hostname, ssl_version=resolved_ssl_version) - if resolved_cert_reqs != ssl.CERT_NONE: - if self.assert_fingerprint: - assert_fingerprint(self.sock.getpeercert(binary_form=True), - self.assert_fingerprint) - elif self.assert_hostname is not False: - match_hostname(self.sock.getpeercert(), - self.assert_hostname or hostname) + if self.assert_fingerprint: + assert_fingerprint(self.sock.getpeercert(binary_form=True), + self.assert_fingerprint) + elif resolved_cert_reqs != ssl.CERT_NONE \ + and self.assert_hostname is not False: + match_hostname(self.sock.getpeercert(), + self.assert_hostname or hostname) + + self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED + or self.assert_fingerprint is not None) if ssl: diff --git a/libs/requests/packages/urllib3/connectionpool.py b/libs/requests/packages/urllib3/connectionpool.py index 95a53a7..9cc2a95 100644 --- a/libs/requests/packages/urllib3/connectionpool.py +++ b/libs/requests/packages/urllib3/connectionpool.py @@ -1,17 +1,12 @@ -# urllib3/connectionpool.py -# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -# -# This module is part of urllib3 and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - -import sys import errno import logging +import sys +import warnings from socket import error as SocketError, timeout as SocketTimeout import socket -try: # Python 3 +try: # Python 3 from queue import LifoQueue, Empty, Full except ImportError: from Queue import LifoQueue, Empty, Full @@ -20,16 +15,16 @@ except ImportError: from .exceptions import ( ClosedPoolError, - ConnectionError, - ConnectTimeoutError, + ProtocolError, EmptyPoolError, HostChangedError, - LocationParseError, + LocationValueError, MaxRetryError, + ProxyError, + ReadTimeoutError, SSLError, TimeoutError, - ReadTimeoutError, - ProxyError, + InsecureRequestWarning, ) from .packages.ssl_match_hostname import CertificateError from .packages import six @@ -41,11 +36,11 @@ from .connection import ( ) from .request import RequestMethods from .response import HTTPResponse -from .util import ( - get_host, - is_connection_dropped, - Timeout, -) + +from .util.connection import is_connection_dropped +from .util.retry import Retry +from .util.timeout import Timeout +from .util.url import get_host xrange = six.moves.xrange @@ -54,8 +49,8 @@ log = logging.getLogger(__name__) _Default = object() -## Pool objects +## Pool objects class ConnectionPool(object): """ Base class for all connection pools, such as @@ -66,13 +61,11 @@ class ConnectionPool(object): QueueCls = LifoQueue def __init__(self, host, port=None): - if host is None: - raise LocationParseError(host) + if not host: + raise LocationValueError("No host specified.") # httplib doesn't like it when we include brackets in ipv6 addresses - host = host.strip('[]') - - self.host = host + self.host = host.strip('[]') self.port = port def __str__(self): @@ -82,6 +75,7 @@ class ConnectionPool(object): # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 _blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK]) + class HTTPConnectionPool(ConnectionPool, RequestMethods): """ Thread-safe connection pool for one host. @@ -126,6 +120,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Headers to include with all requests, unless other headers are given explicitly. + :param retries: + Retry configuration to use by default with requests in this pool. + :param _proxy: Parsed proxy URL, should not be used directly, instead, see :class:`urllib3.connectionpool.ProxyManager`" @@ -133,6 +130,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): :param _proxy_headers: A dictionary with proxy headers, should not be used directly, instead, see :class:`urllib3.connectionpool.ProxyManager`" + + :param \**conn_kw: + Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`, + :class:`urllib3.connection.HTTPSConnection` instances. """ scheme = 'http' @@ -140,18 +141,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): def __init__(self, host, port=None, strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False, - headers=None, _proxy=None, _proxy_headers=None, **conn_kw): + headers=None, retries=None, + _proxy=None, _proxy_headers=None, + **conn_kw): ConnectionPool.__init__(self, host, port) RequestMethods.__init__(self, headers) self.strict = strict - # This is for backwards compatibility and can be removed once a timeout - # can only be set to a Timeout object if not isinstance(timeout, Timeout): timeout = Timeout.from_float(timeout) + if retries is None: + retries = Retry.DEFAULT + self.timeout = timeout + self.retries = retries self.pool = self.QueueCls(maxsize) self.block = block @@ -166,11 +171,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # These are mostly for testing and debugging purposes. self.num_connections = 0 self.num_requests = 0 - - if sys.version_info < (2, 7): # Python 2.6 and older - conn_kw.pop('source_address', None) self.conn_kw = conn_kw + if self.proxy: + # Enable Nagle's algorithm for proxies, to avoid packet fragmentation. + # We cannot know if the user has added default socket options, so we cannot replace the + # list. + self.conn_kw.setdefault('socket_options', []) + def _new_conn(self): """ Return a fresh :class:`HTTPConnection`. @@ -182,10 +190,6 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): conn = self.ConnectionCls(host=self.host, port=self.port, timeout=self.timeout.connect_timeout, strict=self.strict, **self.conn_kw) - if self.proxy is not None: - # Enable Nagle's algorithm for proxies, to avoid packet - # fragmentation. - conn.tcp_nodelay = 0 return conn def _get_conn(self, timeout=None): @@ -204,7 +208,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): try: conn = self.pool.get(block=self.block, timeout=timeout) - except AttributeError: # self.pool is None + except AttributeError: # self.pool is None raise ClosedPoolError(self, "Pool is closed.") except Empty: @@ -218,6 +222,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if conn and is_connection_dropped(conn): log.info("Resetting dropped connection: %s" % self.host) conn.close() + if getattr(conn, 'auto_open', 1) == 0: + # This is a proxied connection that has been mutated by + # httplib._tunnel() and cannot be reused (since it would + # attempt to bypass the proxy) + conn = None return conn or self._new_conn() @@ -237,7 +246,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): """ try: self.pool.put(conn, block=False) - return # Everything is dandy, done. + return # Everything is dandy, done. except AttributeError: # self.pool is None. pass @@ -251,6 +260,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if conn: conn.close() + def _validate_conn(self, conn): + """ + Called right before a request is made, after the socket is created. + """ + pass + def _get_timeout(self, timeout): """ Helper that always returns a :class:`urllib3.util.Timeout` """ if timeout is _Default: @@ -282,23 +297,21 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): self.num_requests += 1 timeout_obj = self._get_timeout(timeout) + timeout_obj.start_connect() + conn.timeout = timeout_obj.connect_timeout - try: - timeout_obj.start_connect() - conn.timeout = timeout_obj.connect_timeout - # conn.request() calls httplib.*.request, not the method in - # urllib3.request. It also calls makefile (recv) on the socket. - conn.request(method, url, **httplib_request_kw) - except SocketTimeout: - raise ConnectTimeoutError( - self, "Connection to %s timed out. (connect timeout=%s)" % - (self.host, timeout_obj.connect_timeout)) + # Trigger any extra validation we need to do. + self._validate_conn(conn) + + # conn.request() calls httplib.*.request, not the method in + # urllib3.request. It also calls makefile (recv) on the socket. + conn.request(method, url, **httplib_request_kw) # Reset the timeout for the recv() on the socket read_timeout = timeout_obj.read_timeout # App Engine doesn't have a sock attr - if hasattr(conn, 'sock'): + if getattr(conn, 'sock', None): # In Python 3 socket.py will catch EAGAIN and return None when you # try and read into the file pointer created by http.client, which # instead raises a BadStatusLine exception. Instead of catching @@ -306,18 +319,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # timeouts, check for a zero timeout before making the request. if read_timeout == 0: raise ReadTimeoutError( - self, url, - "Read timed out. (read timeout=%s)" % read_timeout) + self, url, "Read timed out. (read timeout=%s)" % read_timeout) if read_timeout is Timeout.DEFAULT_TIMEOUT: conn.sock.settimeout(socket.getdefaulttimeout()) - else: # None or a value + else: # None or a value conn.sock.settimeout(read_timeout) # Receive the response from the server try: - try: # Python 2.7+, use buffering of HTTP responses + try: # Python 2.7+, use buffering of HTTP responses httplib_response = conn.getresponse(buffering=True) - except TypeError: # Python 2.6 and older + except TypeError: # Python 2.6 and older httplib_response = conn.getresponse() except SocketTimeout: raise ReadTimeoutError( @@ -329,17 +341,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # http://bugs.python.org/issue10272 if 'timed out' in str(e) or \ 'did not complete (read)' in str(e): # Python 2.6 - raise ReadTimeoutError(self, url, "Read timed out.") + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % read_timeout) raise - except SocketError as e: # Platform-specific: Python 2 + except SocketError as e: # Platform-specific: Python 2 # See the above comment about EAGAIN in Python 3. In Python 2 we # have to specifically catch it and throw the timeout error if e.errno in _blocking_errnos: raise ReadTimeoutError( - self, url, - "Read timed out. (read timeout=%s)" % read_timeout) + self, url, "Read timed out. (read timeout=%s)" % read_timeout) raise @@ -364,7 +376,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): conn.close() except Empty: - pass # Done. + pass # Done. def is_same_host(self, url): """ @@ -385,7 +397,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): return (scheme, host, port) == (self.scheme, self.host, self.port) - def urlopen(self, method, url, body=None, headers=None, retries=3, + def urlopen(self, method, url, body=None, headers=None, retries=None, redirect=True, assert_same_host=True, timeout=_Default, pool_timeout=None, release_conn=None, **response_kw): """ @@ -419,9 +431,20 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): these headers completely replace any pool-specific headers. :param retries: - Number of retries to allow before raising a MaxRetryError exception. - If `False`, then retries are disabled and any exception is raised - immediately. + Configure the number of retries to allow before raising a + :class:`~urllib3.exceptions.MaxRetryError` exception. + + Pass ``None`` to retry until you receive a response. Pass a + :class:`~urllib3.util.retry.Retry` object for fine-grained control + over different types of retries. + Pass an integer number to retry connection errors that many times, + but no other types of errors. Pass zero to never retry. + + If ``False``, then retries are disabled and any exception is raised + immediately. Also, instead of raising a MaxRetryError on redirects, + the redirect response will be returned. + + :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, @@ -460,15 +483,15 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if headers is None: headers = self.headers - if retries < 0 and retries is not False: - raise MaxRetryError(self, url) + if not isinstance(retries, Retry): + retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = response_kw.get('preload_content', True) # Check host if assert_same_host and not self.is_same_host(url): - raise HostChangedError(self, url, retries - 1) + raise HostChangedError(self, url, retries) conn = None @@ -484,10 +507,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): err = None try: - # Request a connection from the queue + # Request a connection from the queue. conn = self._get_conn(timeout=pool_timeout) - # Make the request on the httplib connection object + # Make the request on the httplib connection object. httplib_response = self._make_request(conn, method, url, timeout=timeout, body=body, headers=headers) @@ -526,21 +549,15 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): conn.close() conn = None - if not retries: - if isinstance(e, TimeoutError): - # TimeoutError is exempt from MaxRetryError-wrapping. - # FIXME: ... Not sure why. Add a reason here. - raise - - # Wrap unexpected exceptions with the most appropriate - # module-level exception and re-raise. - if isinstance(e, SocketError) and self.proxy: - raise ProxyError('Cannot connect to proxy.', e) + stacktrace = sys.exc_info()[2] + if isinstance(e, SocketError) and self.proxy: + e = ProxyError('Cannot connect to proxy.', e) + elif isinstance(e, (SocketError, HTTPException)): + e = ProtocolError('Connection aborted.', e) - if retries is False: - raise ConnectionError('Connection failed.', e) - - raise MaxRetryError(self, url, e) + retries = retries.increment(method, url, error=e, + _pool=self, _stacktrace=stacktrace) + retries.sleep() # Keep track of the error for the retry warning. err = e @@ -554,23 +571,43 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if not conn: # Try again - log.warning("Retrying (%d attempts remain) after connection " + log.warning("Retrying (%r) after connection " "broken by '%r': %s" % (retries, err, url)) - return self.urlopen(method, url, body, headers, retries - 1, + return self.urlopen(method, url, body, headers, retries, redirect, assert_same_host, timeout=timeout, pool_timeout=pool_timeout, release_conn=release_conn, **response_kw) # Handle redirect? redirect_location = redirect and response.get_redirect_location() - if redirect_location and retries is not False: + if redirect_location: if response.status == 303: method = 'GET' + + try: + retries = retries.increment(method, url, response=response, _pool=self) + except MaxRetryError: + if retries.raise_on_redirect: + raise + return response + log.info("Redirecting %s -> %s" % (url, redirect_location)) return self.urlopen(method, redirect_location, body, headers, - retries - 1, redirect, assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, **response_kw) + retries=retries, redirect=redirect, + assert_same_host=assert_same_host, + timeout=timeout, pool_timeout=pool_timeout, + release_conn=release_conn, **response_kw) + + # Check if we should retry the HTTP response. + if retries.is_forced_retry(method, status_code=response.status): + retries = retries.increment(method, url, response=response, _pool=self) + retries.sleep() + log.info("Forced retry: %s" % url) + return self.urlopen(method, url, body, headers, + retries=retries, redirect=redirect, + assert_same_host=assert_same_host, + timeout=timeout, pool_timeout=pool_timeout, + release_conn=release_conn, **response_kw) return response @@ -597,19 +634,17 @@ class HTTPSConnectionPool(HTTPConnectionPool): ConnectionCls = HTTPSConnection def __init__(self, host, port=None, - strict=False, timeout=None, maxsize=1, - block=False, headers=None, + strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, + block=False, headers=None, retries=None, _proxy=None, _proxy_headers=None, key_file=None, cert_file=None, cert_reqs=None, ca_certs=None, ssl_version=None, assert_hostname=None, assert_fingerprint=None, **conn_kw): - if sys.version_info < (2, 7): # Python 2.6 or older - conn_kw.pop('source_address', None) - HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize, - block, headers, _proxy, _proxy_headers, **conn_kw) + block, headers, retries, _proxy, _proxy_headers, + **conn_kw) self.key_file = key_file self.cert_file = cert_file self.cert_reqs = cert_reqs @@ -617,7 +652,6 @@ class HTTPSConnectionPool(HTTPConnectionPool): self.ssl_version = ssl_version self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint - self.conn_kw = conn_kw def _prepare_conn(self, conn): """ @@ -633,7 +667,6 @@ class HTTPSConnectionPool(HTTPConnectionPool): assert_hostname=self.assert_hostname, assert_fingerprint=self.assert_fingerprint) conn.ssl_version = self.ssl_version - conn.conn_kw = self.conn_kw if self.proxy is not None: # Python 2.7+ @@ -641,7 +674,12 @@ class HTTPSConnectionPool(HTTPConnectionPool): set_tunnel = conn.set_tunnel except AttributeError: # Platform-specific: Python 2.6 set_tunnel = conn._set_tunnel - set_tunnel(self.host, self.port, self.proxy_headers) + + if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older + set_tunnel(self.host, self.port) + else: + set_tunnel(self.host, self.port, self.proxy_headers) + # Establish tunnel connection early, because otherwise httplib # would improperly set Host: header to proxy's IP:port. conn.connect() @@ -667,21 +705,30 @@ class HTTPSConnectionPool(HTTPConnectionPool): actual_host = self.proxy.host actual_port = self.proxy.port - extra_params = {} - if not six.PY3: # Python 2 - extra_params['strict'] = self.strict - extra_params.update(self.conn_kw) - conn = self.ConnectionCls(host=actual_host, port=actual_port, timeout=self.timeout.connect_timeout, - **extra_params) - if self.proxy is not None: - # Enable Nagle's algorithm for proxies, to avoid packet - # fragmentation. - conn.tcp_nodelay = 0 + strict=self.strict, **self.conn_kw) return self._prepare_conn(conn) + def _validate_conn(self, conn): + """ + Called right before a request is made, after the socket is created. + """ + super(HTTPSConnectionPool, self)._validate_conn(conn) + + # Force connect early to allow us to validate the connection. + if not getattr(conn, 'sock', None): # AppEngine might not have `.sock` + conn.connect() + + if not conn.is_verified: + warnings.warn(( + 'Unverified HTTPS request is being made. ' + 'Adding certificate verification is strongly advised. See: ' + 'https://urllib3.readthedocs.org/en/latest/security.html ' + '(This warning will only appear once by default.)'), + InsecureRequestWarning) + def connection_from_url(url, **kw): """ @@ -698,7 +745,7 @@ def connection_from_url(url, **kw): :class:`.ConnectionPool`. Useful for specifying things like timeout, maxsize, headers, etc. - Example: :: + Example:: >>> conn = connection_from_url('http://google.com/') >>> r = conn.request('GET', '/') diff --git a/libs/requests/packages/urllib3/contrib/ntlmpool.py b/libs/requests/packages/urllib3/contrib/ntlmpool.py index b8cd933..c6b266f 100644 --- a/libs/requests/packages/urllib3/contrib/ntlmpool.py +++ b/libs/requests/packages/urllib3/contrib/ntlmpool.py @@ -1,9 +1,3 @@ -# urllib3/contrib/ntlmpool.py -# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -# -# This module is part of urllib3 and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - """ NTLM authenticating pool, contributed by erikcederstran diff --git a/libs/requests/packages/urllib3/contrib/pyopenssl.py b/libs/requests/packages/urllib3/contrib/pyopenssl.py index 21a12c6..24de9e4 100644 --- a/libs/requests/packages/urllib3/contrib/pyopenssl.py +++ b/libs/requests/packages/urllib3/contrib/pyopenssl.py @@ -46,15 +46,18 @@ Module Variables ''' -from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT -from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName +try: + from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT + from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName +except SyntaxError as e: + raise ImportError(e) + import OpenSSL.SSL from pyasn1.codec.der import decoder as der_decoder from pyasn1.type import univ, constraint from socket import _fileobject, timeout import ssl import select -from cStringIO import StringIO from .. import connection from .. import util @@ -155,196 +158,43 @@ def get_subj_alt_name(peer_cert): return dns_name -class fileobject(_fileobject): - - def _wait_for_sock(self): - rd, wd, ed = select.select([self._sock], [], [], - self._sock.gettimeout()) - if not rd: - raise timeout() - - - def read(self, size=-1): - # Use max, disallow tiny reads in a loop as they are very inefficient. - # We never leave read() with any leftover data from a new recv() call - # in our internal buffer. - rbufsize = max(self._rbufsize, self.default_bufsize) - # Our use of StringIO rather than lists of string objects returned by - # recv() minimizes memory usage and fragmentation that occurs when - # rbufsize is large compared to the typical return value of recv(). - buf = self._rbuf - buf.seek(0, 2) # seek end - if size < 0: - # Read until EOF - self._rbuf = StringIO() # reset _rbuf. we consume it via buf. - while True: - try: - data = self._sock.recv(rbufsize) - except OpenSSL.SSL.WantReadError: - self._wait_for_sock() - continue - if not data: - break - buf.write(data) - return buf.getvalue() - else: - # Read until size bytes or EOF seen, whichever comes first - buf_len = buf.tell() - if buf_len >= size: - # Already have size bytes in our buffer? Extract and return. - buf.seek(0) - rv = buf.read(size) - self._rbuf = StringIO() - self._rbuf.write(buf.read()) - return rv - - self._rbuf = StringIO() # reset _rbuf. we consume it via buf. - while True: - left = size - buf_len - # recv() will malloc the amount of memory given as its - # parameter even though it often returns much less data - # than that. The returned data string is short lived - # as we copy it into a StringIO and free it. This avoids - # fragmentation issues on many platforms. - try: - data = self._sock.recv(left) - except OpenSSL.SSL.WantReadError: - self._wait_for_sock() - continue - if not data: - break - n = len(data) - if n == size and not buf_len: - # Shortcut. Avoid buffer data copies when: - # - We have no data in our buffer. - # AND - # - Our call to recv returned exactly the - # number of bytes we were asked to read. - return data - if n == left: - buf.write(data) - del data # explicit free - break - assert n <= left, "recv(%d) returned %d bytes" % (left, n) - buf.write(data) - buf_len += n - del data # explicit free - #assert buf_len == buf.tell() - return buf.getvalue() - - def readline(self, size=-1): - buf = self._rbuf - buf.seek(0, 2) # seek end - if buf.tell() > 0: - # check if we already have it in our buffer - buf.seek(0) - bline = buf.readline(size) - if bline.endswith('\n') or len(bline) == size: - self._rbuf = StringIO() - self._rbuf.write(buf.read()) - return bline - del bline - if size < 0: - # Read until \n or EOF, whichever comes first - if self._rbufsize <= 1: - # Speed up unbuffered case - buf.seek(0) - buffers = [buf.read()] - self._rbuf = StringIO() # reset _rbuf. we consume it via buf. - data = None - recv = self._sock.recv - while True: - try: - while data != "\n": - data = recv(1) - if not data: - break - buffers.append(data) - except OpenSSL.SSL.WantReadError: - self._wait_for_sock() - continue - break - return "".join(buffers) - - buf.seek(0, 2) # seek end - self._rbuf = StringIO() # reset _rbuf. we consume it via buf. - while True: - try: - data = self._sock.recv(self._rbufsize) - except OpenSSL.SSL.WantReadError: - self._wait_for_sock() - continue - if not data: - break - nl = data.find('\n') - if nl >= 0: - nl += 1 - buf.write(data[:nl]) - self._rbuf.write(data[nl:]) - del data - break - buf.write(data) - return buf.getvalue() - else: - # Read until size bytes or \n or EOF seen, whichever comes first - buf.seek(0, 2) # seek end - buf_len = buf.tell() - if buf_len >= size: - buf.seek(0) - rv = buf.read(size) - self._rbuf = StringIO() - self._rbuf.write(buf.read()) - return rv - self._rbuf = StringIO() # reset _rbuf. we consume it via buf. - while True: - try: - data = self._sock.recv(self._rbufsize) - except OpenSSL.SSL.WantReadError: - self._wait_for_sock() - continue - if not data: - break - left = size - buf_len - # did we just receive a newline? - nl = data.find('\n', 0, left) - if nl >= 0: - nl += 1 - # save the excess data to _rbuf - self._rbuf.write(data[nl:]) - if buf_len: - buf.write(data[:nl]) - break - else: - # Shortcut. Avoid data copy through buf when returning - # a substring of our first recv(). - return data[:nl] - n = len(data) - if n == size and not buf_len: - # Shortcut. Avoid data copy through buf when - # returning exactly all of our first recv(). - return data - if n >= left: - buf.write(data[:left]) - self._rbuf.write(data[left:]) - break - buf.write(data) - buf_len += n - #assert buf_len == buf.tell() - return buf.getvalue() - - class WrappedSocket(object): - '''API-compatibility wrapper for Python OpenSSL's Connection-class.''' + '''API-compatibility wrapper for Python OpenSSL's Connection-class. - def __init__(self, connection, socket): + Note: _makefile_refs, _drop() and _reuse() are needed for the garbage + collector of pypy. + ''' + + def __init__(self, connection, socket, suppress_ragged_eofs=True): self.connection = connection self.socket = socket + self.suppress_ragged_eofs = suppress_ragged_eofs + self._makefile_refs = 0 def fileno(self): return self.socket.fileno() def makefile(self, mode, bufsize=-1): - return fileobject(self.connection, mode, bufsize) + self._makefile_refs += 1 + return _fileobject(self, mode, bufsize, close=True) + + def recv(self, *args, **kwargs): + try: + data = self.connection.recv(*args, **kwargs) + except OpenSSL.SSL.SysCallError as e: + if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): + return b'' + else: + raise + except OpenSSL.SSL.WantReadError: + rd, wd, ed = select.select( + [self.socket], [], [], self.socket.gettimeout()) + if not rd: + raise timeout('The read operation timed out') + else: + return self.recv(*args, **kwargs) + else: + return data def settimeout(self, timeout): return self.socket.settimeout(timeout) @@ -353,7 +203,10 @@ class WrappedSocket(object): return self.connection.sendall(data) def close(self): - return self.connection.shutdown() + if self._makefile_refs < 1: + return self.connection.shutdown() + else: + self._makefile_refs -= 1 def getpeercert(self, binary_form=False): x509 = self.connection.get_peer_certificate() @@ -376,6 +229,15 @@ class WrappedSocket(object): ] } + def _reuse(self): + self._makefile_refs += 1 + + def _drop(self): + if self._makefile_refs < 1: + self.close() + else: + self._makefile_refs -= 1 + def _verify_callback(cnx, x509, err_no, err_depth, return_code): return err_no == 0 diff --git a/libs/requests/packages/urllib3/exceptions.py b/libs/requests/packages/urllib3/exceptions.py index b4df831..7519ba9 100644 --- a/libs/requests/packages/urllib3/exceptions.py +++ b/libs/requests/packages/urllib3/exceptions.py @@ -1,9 +1,3 @@ -# urllib3/exceptions.py -# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -# -# This module is part of urllib3 and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - ## Base Exceptions @@ -11,6 +5,11 @@ class HTTPError(Exception): "Base exception used by this module." pass +class HTTPWarning(Warning): + "Base warning used by this module." + pass + + class PoolError(HTTPError): "Base exception for errors caused within a pool." @@ -44,27 +43,38 @@ class ProxyError(HTTPError): pass -class ConnectionError(HTTPError): - "Raised when a normal connection fails." +class DecodeError(HTTPError): + "Raised when automatic decoding based on Content-Type fails." pass -class DecodeError(HTTPError): - "Raised when automatic decoding based on Content-Type fails." +class ProtocolError(HTTPError): + "Raised when something unexpected happens mid-request/response." pass +#: Renamed to ProtocolError but aliased for backwards compatibility. +ConnectionError = ProtocolError + + ## Leaf Exceptions class MaxRetryError(RequestError): - "Raised when the maximum number of retries is exceeded." + """Raised when the maximum number of retries is exceeded. + + :param pool: The connection pool + :type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool` + :param string url: The requested Url + :param exceptions.Exception reason: The underlying error + + """ def __init__(self, pool, url, reason=None): self.reason = reason message = "Max retries exceeded with url: %s" % url if reason: - message += " (Caused by %s: %s)" % (type(reason), reason) + message += " (Caused by %r)" % reason else: message += " (Caused by redirect)" @@ -116,7 +126,12 @@ class ClosedPoolError(PoolError): pass -class LocationParseError(ValueError, HTTPError): +class LocationValueError(ValueError, HTTPError): + "Raised when there is something wrong with a given URL input." + pass + + +class LocationParseError(LocationValueError): "Raised when get_host or similar fails to parse the URL input." def __init__(self, location): @@ -124,3 +139,18 @@ class LocationParseError(ValueError, HTTPError): HTTPError.__init__(self, message) self.location = location + + +class SecurityWarning(HTTPWarning): + "Warned when perfoming security reducing actions" + pass + + +class InsecureRequestWarning(SecurityWarning): + "Warned when making an unverified HTTPS request." + pass + + +class SystemTimeWarning(SecurityWarning): + "Warned when system time is suspected to be wrong" + pass diff --git a/libs/requests/packages/urllib3/fields.py b/libs/requests/packages/urllib3/fields.py index da79e92..c853f8d 100644 --- a/libs/requests/packages/urllib3/fields.py +++ b/libs/requests/packages/urllib3/fields.py @@ -1,9 +1,3 @@ -# urllib3/fields.py -# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -# -# This module is part of urllib3 and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - import email.utils import mimetypes @@ -78,9 +72,10 @@ class RequestField(object): """ A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters. - Supports constructing :class:`~urllib3.fields.RequestField` from parameter - of key/value strings AND key/filetuple. A filetuple is a (filename, data, MIME type) - tuple where the MIME type is optional. For example: :: + Supports constructing :class:`~urllib3.fields.RequestField` from + parameter of key/value strings AND key/filetuple. A filetuple is a + (filename, data, MIME type) tuple where the MIME type is optional. + For example:: 'foo': 'bar', 'fakefile': ('foofile.txt', 'contents of foofile'), @@ -125,8 +120,8 @@ class RequestField(object): 'Content-Disposition' fields. :param header_parts: - A sequence of (k, v) typles or a :class:`dict` of (k, v) to format as - `k1="v1"; k2="v2"; ...`. + A sequence of (k, v) typles or a :class:`dict` of (k, v) to format + as `k1="v1"; k2="v2"; ...`. """ parts = [] iterable = header_parts @@ -158,7 +153,8 @@ class RequestField(object): lines.append('\r\n') return '\r\n'.join(lines) - def make_multipart(self, content_disposition=None, content_type=None, content_location=None): + def make_multipart(self, content_disposition=None, content_type=None, + content_location=None): """ Makes this request field into a multipart request field. @@ -172,6 +168,10 @@ class RequestField(object): """ self.headers['Content-Disposition'] = content_disposition or 'form-data' - self.headers['Content-Disposition'] += '; '.join(['', self._render_parts((('name', self._name), ('filename', self._filename)))]) + self.headers['Content-Disposition'] += '; '.join([ + '', self._render_parts( + (('name', self._name), ('filename', self._filename)) + ) + ]) self.headers['Content-Type'] = content_type self.headers['Content-Location'] = content_location diff --git a/libs/requests/packages/urllib3/filepost.py b/libs/requests/packages/urllib3/filepost.py index e8b30bd..0fbf488 100644 --- a/libs/requests/packages/urllib3/filepost.py +++ b/libs/requests/packages/urllib3/filepost.py @@ -1,11 +1,4 @@ -# urllib3/filepost.py -# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -# -# This module is part of urllib3 and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - import codecs -import mimetypes from uuid import uuid4 from io import BytesIO @@ -38,10 +31,10 @@ def iter_field_objects(fields): i = iter(fields) for field in i: - if isinstance(field, RequestField): - yield field - else: - yield RequestField.from_tuples(*field) + if isinstance(field, RequestField): + yield field + else: + yield RequestField.from_tuples(*field) def iter_fields(fields): diff --git a/libs/requests/packages/urllib3/packages/ordered_dict.py b/libs/requests/packages/urllib3/packages/ordered_dict.py index 7f8ee15..4479363 100644 --- a/libs/requests/packages/urllib3/packages/ordered_dict.py +++ b/libs/requests/packages/urllib3/packages/ordered_dict.py @@ -2,7 +2,6 @@ # Passes Python2.7's test suite and incorporates all the latest updates. # Copyright 2009 Raymond Hettinger, released under the MIT License. # http://code.activestate.com/recipes/576693/ - try: from thread import get_ident as _get_ident except ImportError: diff --git a/libs/requests/packages/urllib3/poolmanager.py b/libs/requests/packages/urllib3/poolmanager.py index f18ff2b..515dc96 100644 --- a/libs/requests/packages/urllib3/poolmanager.py +++ b/libs/requests/packages/urllib3/poolmanager.py @@ -1,9 +1,3 @@ -# urllib3/poolmanager.py -# Copyright 2008-2014 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -# -# This module is part of urllib3 and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - import logging try: # Python 3 @@ -14,8 +8,10 @@ except ImportError: from ._collections import RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool from .connectionpool import port_by_scheme +from .exceptions import LocationValueError from .request import RequestMethods -from .util import parse_url +from .util.url import parse_url +from .util.retry import Retry __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] @@ -49,7 +45,7 @@ class PoolManager(RequestMethods): Additional parameters are used to create fresh :class:`urllib3.connectionpool.ConnectionPool` instances. - Example: :: + Example:: >>> manager = PoolManager(num_pools=2) >>> r = manager.request('GET', 'http://google.com/') @@ -102,10 +98,11 @@ class PoolManager(RequestMethods): ``urllib3.connectionpool.port_by_scheme``. """ - scheme = scheme or 'http' + if not host: + raise LocationValueError("No host specified.") + scheme = scheme or 'http' port = port or port_by_scheme.get(scheme, 80) - pool_key = (scheme, host, port) with self.pools.lock: @@ -118,6 +115,7 @@ class PoolManager(RequestMethods): # Make a fresh ConnectionPool of the desired type pool = self._new_pool(scheme, host, port) self.pools[pool_key] = pool + return pool def connection_from_url(self, url): @@ -161,13 +159,18 @@ class PoolManager(RequestMethods): # Support relative URLs for redirecting. redirect_location = urljoin(url, redirect_location) - # RFC 2616, Section 10.3.4 + # RFC 7231, Section 6.4.4 if response.status == 303: method = 'GET' - log.info("Redirecting %s -> %s" % (url, redirect_location)) - kw['retries'] = kw.get('retries', 3) - 1 # Persist retries countdown + retries = kw.get('retries') + if not isinstance(retries, Retry): + retries = Retry.from_int(retries, redirect=redirect) + + kw['retries'] = retries.increment(method, redirect_location) kw['redirect'] = redirect + + log.info("Redirecting %s -> %s" % (url, redirect_location)) return self.urlopen(method, redirect_location, **kw) @@ -208,12 +211,16 @@ class ProxyManager(PoolManager): if not proxy.port: port = port_by_scheme.get(proxy.scheme, 80) proxy = proxy._replace(port=port) + + assert proxy.scheme in ("http", "https"), \ + 'Not supported proxy scheme %s' % proxy.scheme + self.proxy = proxy self.proxy_headers = proxy_headers or {} - assert self.proxy.scheme in ("http", "https"), \ - 'Not supported proxy scheme %s' % self.proxy.scheme + connection_pool_kw['_proxy'] = self.proxy connection_pool_kw['_proxy_headers'] = self.proxy_headers + super(ProxyManager, self).__init__( num_pools, headers, **connection_pool_kw) @@ -248,10 +255,10 @@ class ProxyManager(PoolManager): # For proxied HTTPS requests, httplib sets the necessary headers # on the CONNECT to the proxy. For HTTP, we'll definitely # need to set 'Host' at the very least. - kw['headers'] = self._set_proxy_headers(url, kw.get('headers', - self.headers)) + headers = kw.get('headers', self.headers) + kw['headers'] = self._set_proxy_headers(url, headers) - return super(ProxyManager, self).urlopen(method, url, redirect, **kw) + return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw) def proxy_from_url(url, **kw): diff --git a/libs/requests/packages/urllib3/request.py b/libs/requests/packages/urllib3/request.py index 2a92cc2..51fe238 100644 --- a/libs/requests/packages/urllib3/request.py +++ b/libs/requests/packages/urllib3/request.py @@ -1,9 +1,3 @@ -# urllib3/request.py -# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -# -# This module is part of urllib3 and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - try: from urllib.parse import urlencode except ImportError: @@ -26,8 +20,8 @@ class RequestMethods(object): Specifically, - :meth:`.request_encode_url` is for sending requests whose fields are encoded - in the URL (such as GET, HEAD, DELETE). + :meth:`.request_encode_url` is for sending requests whose fields are + encoded in the URL (such as GET, HEAD, DELETE). :meth:`.request_encode_body` is for sending requests whose fields are encoded in the *body* of the request using multipart or www-form-urlencoded @@ -51,7 +45,7 @@ class RequestMethods(object): def urlopen(self, method, url, body=None, headers=None, encode_multipart=True, multipart_boundary=None, - **kw): # Abstract + **kw): # Abstract raise NotImplemented("Classes extending RequestMethods must implement " "their own ``urlopen`` method.") @@ -61,8 +55,8 @@ class RequestMethods(object): ``fields`` based on the ``method`` used. This is a convenience method that requires the least amount of manual - effort. It can be used in most situations, while still having the option - to drop down to more specific methods when necessary, such as + effort. It can be used in most situations, while still having the + option to drop down to more specific methods when necessary, such as :meth:`request_encode_url`, :meth:`request_encode_body`, or even the lowest level :meth:`urlopen`. """ @@ -70,12 +64,12 @@ class RequestMethods(object): if method in self._encode_url_methods: return self.request_encode_url(method, url, fields=fields, - headers=headers, - **urlopen_kw) + headers=headers, + **urlopen_kw) else: return self.request_encode_body(method, url, fields=fields, - headers=headers, - **urlopen_kw) + headers=headers, + **urlopen_kw) def request_encode_url(self, method, url, fields=None, **urlopen_kw): """ @@ -94,18 +88,18 @@ class RequestMethods(object): the body. This is useful for request methods like POST, PUT, PATCH, etc. When ``encode_multipart=True`` (default), then - :meth:`urllib3.filepost.encode_multipart_formdata` is used to encode the - payload with the appropriate content type. Otherwise + :meth:`urllib3.filepost.encode_multipart_formdata` is used to encode + the payload with the appropriate content type. Otherwise :meth:`urllib.urlencode` is used with the 'application/x-www-form-urlencoded' content type. Multipart encoding must be used when posting files, and it's reasonably - safe to use it in other times too. However, it may break request signing, - such as with OAuth. + safe to use it in other times too. However, it may break request + signing, such as with OAuth. Supports an optional ``fields`` parameter of key/value strings AND key/filetuple. A filetuple is a (filename, data, MIME type) tuple where - the MIME type is optional. For example: :: + the MIME type is optional. For example:: fields = { 'foo': 'bar', @@ -119,17 +113,17 @@ class RequestMethods(object): When uploading a file, providing a filename (the first parameter of the tuple) is optional but recommended to best mimick behavior of browsers. - Note that if ``headers`` are supplied, the 'Content-Type' header will be - overwritten because it depends on the dynamic random boundary string + Note that if ``headers`` are supplied, the 'Content-Type' header will + be overwritten because it depends on the dynamic random boundary string which is used to compose the body of the request. The random boundary string can be explicitly set with the ``multipart_boundary`` parameter. """ if encode_multipart: - body, content_type = encode_multipart_formdata(fields or {}, - boundary=multipart_boundary) + body, content_type = encode_multipart_formdata( + fields or {}, boundary=multipart_boundary) else: body, content_type = (urlencode(fields or {}), - 'application/x-www-form-urlencoded') + 'application/x-www-form-urlencoded') if headers is None: headers = self.headers diff --git a/libs/requests/packages/urllib3/response.py b/libs/requests/packages/urllib3/response.py index db44182..e69de95 100644 --- a/libs/requests/packages/urllib3/response.py +++ b/libs/requests/packages/urllib3/response.py @@ -1,22 +1,14 @@ -# urllib3/response.py -# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -# -# This module is part of urllib3 and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - - -import logging import zlib import io +from socket import timeout as SocketTimeout from ._collections import HTTPHeaderDict -from .exceptions import DecodeError +from .exceptions import ProtocolError, DecodeError, ReadTimeoutError from .packages.six import string_types as basestring, binary_type -from .util import is_fp_closed +from .connection import HTTPException, BaseSSLError +from .util.response import is_fp_closed -log = logging.getLogger(__name__) - class DeflateDecoder(object): @@ -56,7 +48,10 @@ class HTTPResponse(io.IOBase): HTTP Response container. Backwards-compatible to httplib's HTTPResponse but the response ``body`` is - loaded and decoded on-demand when the ``data`` property is accessed. + loaded and decoded on-demand when the ``data`` property is accessed. This + class is also compatible with the Python standard library's :mod:`io` + module, and can hence be treated as a readable object in the context of that + framework. Extra parameters for behaviour not present in httplib.HTTPResponse: @@ -91,11 +86,14 @@ class HTTPResponse(io.IOBase): self.decode_content = decode_content self._decoder = None - self._body = body if body and isinstance(body, basestring) else None + self._body = None self._fp = None self._original_response = original_response self._fp_bytes_read = 0 + if body and isinstance(body, (basestring, binary_type)): + self._body = body + self._pool = pool self._connection = connection @@ -163,8 +161,8 @@ class HTTPResponse(io.IOBase): after having ``.read()`` the file object. (Overridden if ``amt`` is set.) """ - # Note: content-encoding value should be case-insensitive, per RFC 2616 - # Section 3.5 + # Note: content-encoding value should be case-insensitive, per RFC 7230 + # Section 3.2 content_encoding = self.headers.get('content-encoding', '').lower() if self._decoder is None: if content_encoding in self.CONTENT_DECODERS: @@ -178,23 +176,42 @@ class HTTPResponse(io.IOBase): flush_decoder = False try: - if amt is None: - # cStringIO doesn't like amt=None - data = self._fp.read() - flush_decoder = True - else: - cache_content = False - data = self._fp.read(amt) - if amt != 0 and not data: # Platform-specific: Buggy versions of Python. - # Close the connection when no data is returned - # - # This is redundant to what httplib/http.client _should_ - # already do. However, versions of python released before - # December 15, 2012 (http://bugs.python.org/issue16298) do not - # properly close the connection in all cases. There is no harm - # in redundantly calling close. - self._fp.close() + try: + if amt is None: + # cStringIO doesn't like amt=None + data = self._fp.read() flush_decoder = True + else: + cache_content = False + data = self._fp.read(amt) + if amt != 0 and not data: # Platform-specific: Buggy versions of Python. + # Close the connection when no data is returned + # + # This is redundant to what httplib/http.client _should_ + # already do. However, versions of python released before + # December 15, 2012 (http://bugs.python.org/issue16298) do + # not properly close the connection in all cases. There is + # no harm in redundantly calling close. + self._fp.close() + flush_decoder = True + + except SocketTimeout: + # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but + # there is yet no clean way to get at it from this context. + raise ReadTimeoutError(self._pool, None, 'Read timed out.') + + except BaseSSLError as e: + # FIXME: Is there a better way to differentiate between SSLErrors? + if not 'read operation timed out' in str(e): # Defensive: + # This shouldn't happen but just in case we're missing an edge + # case, let's avoid swallowing SSL errors. + raise + + raise ReadTimeoutError(self._pool, None, 'Read timed out.') + + except HTTPException as e: + # This includes IncompleteRead. + raise ProtocolError('Connection broken: %r' % e, e) self._fp_bytes_read += len(data) @@ -204,8 +221,7 @@ class HTTPResponse(io.IOBase): except (IOError, zlib.error) as e: raise DecodeError( "Received response with content-encoding: %s, but " - "failed to decode it." % content_encoding, - e) + "failed to decode it." % content_encoding, e) if flush_decoder and decode_content and self._decoder: buf = self._decoder.decompress(binary_type()) @@ -242,7 +258,6 @@ class HTTPResponse(io.IOBase): if data: yield data - @classmethod def from_httplib(ResponseCls, r, **response_kw): """ @@ -297,7 +312,7 @@ class HTTPResponse(io.IOBase): elif hasattr(self._fp, "fileno"): return self._fp.fileno() else: - raise IOError("The file-like object this HTTPResponse is wrapped " + raise IOError("The file-like object this HTTPResponse is wrapped " "around has no file descriptor") def flush(self): @@ -305,4 +320,14 @@ class HTTPResponse(io.IOBase): return self._fp.flush() def readable(self): + # This method is required for `io` module compatibility. return True + + def readinto(self, b): + # This method is required for `io` module compatibility. + temp = self.read(len(b)) + if len(temp) == 0: + return 0 + else: + b[:len(temp)] = temp + return len(temp) diff --git a/libs/requests/packages/urllib3/util/__init__.py b/libs/requests/packages/urllib3/util/__init__.py index a40185e..8becc81 100644 --- a/libs/requests/packages/urllib3/util/__init__.py +++ b/libs/requests/packages/urllib3/util/__init__.py @@ -1,9 +1,4 @@ -# urllib3/util/__init__.py -# Copyright 2008-2014 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -# -# This module is part of urllib3 and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - +# For backwards compatibility, provide imports that used to be here. from .connection import is_connection_dropped from .request import make_headers from .response import is_fp_closed @@ -19,6 +14,8 @@ from .timeout import ( current_time, Timeout, ) + +from .retry import Retry from .url import ( get_host, parse_url, diff --git a/libs/requests/packages/urllib3/util/connection.py b/libs/requests/packages/urllib3/util/connection.py index 8deeab5..2156993 100644 --- a/libs/requests/packages/urllib3/util/connection.py +++ b/libs/requests/packages/urllib3/util/connection.py @@ -1,4 +1,4 @@ -from socket import error as SocketError +import socket try: from select import poll, POLLIN except ImportError: # `poll` doesn't exist on OSX and other platforms @@ -8,6 +8,7 @@ except ImportError: # `poll` doesn't exist on OSX and other platforms except ImportError: # `select` doesn't exist on AppEngine. select = False + def is_connection_dropped(conn): # Platform-specific """ Returns True if the connection is dropped and should be closed. @@ -22,7 +23,7 @@ def is_connection_dropped(conn): # Platform-specific if sock is False: # Platform-specific: AppEngine return False if sock is None: # Connection already closed (such as by httplib). - return False + return True if not poll: if not select: # Platform-specific: AppEngine @@ -30,7 +31,7 @@ def is_connection_dropped(conn): # Platform-specific try: return select([sock], [], [], 0.0)[0] - except SocketError: + except socket.error: return True # This version is better on platforms that support it. @@ -42,4 +43,55 @@ def is_connection_dropped(conn): # Platform-specific return True +# This function is copied from socket.py in the Python 2.7 standard +# library test suite. Added to its signature is only `socket_options`. +def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None, socket_options=None): + """Connect to *address* and return the socket object. + + Convenience function. Connect to *address* (a 2-tuple ``(host, + port)``) and return the socket object. Passing the optional + *timeout* parameter will set the timeout on the socket instance + before attempting to connect. If no *timeout* is supplied, the + global default timeout setting returned by :func:`getdefaulttimeout` + is used. If *source_address* is set it must be a tuple of (host, port) + for the socket to bind as a source address before making the connection. + An host of '' or port 0 tells the OS to use the default. + """ + + host, port = address + err = None + for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + sock = None + try: + sock = socket.socket(af, socktype, proto) + + # If provided, set socket level options before connecting. + # This is the only addition urllib3 makes to this function. + _set_socket_options(sock, socket_options) + + if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: + sock.settimeout(timeout) + if source_address: + sock.bind(source_address) + sock.connect(sa) + return sock + + except socket.error as _: + err = _ + if sock is not None: + sock.close() + + if err is not None: + raise err + else: + raise socket.error("getaddrinfo returns an empty list") + + +def _set_socket_options(sock, options): + if options is None: + return + for opt in options: + sock.setsockopt(*opt) diff --git a/libs/requests/packages/urllib3/util/request.py b/libs/requests/packages/urllib3/util/request.py index d48d651..bc64f6b 100644 --- a/libs/requests/packages/urllib3/util/request.py +++ b/libs/requests/packages/urllib3/util/request.py @@ -1,13 +1,12 @@ from base64 import b64encode -from ..packages import six - +from ..packages.six import b ACCEPT_ENCODING = 'gzip,deflate' def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, - basic_auth=None, proxy_basic_auth=None): + basic_auth=None, proxy_basic_auth=None, disable_cache=None): """ Shortcuts for generating request headers. @@ -32,7 +31,10 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, Colon-separated username:password string for 'proxy-authorization: basic ...' auth header. - Example: :: + :param disable_cache: + If ``True``, adds 'cache-control: no-cache' header. + + Example:: >>> make_headers(keep_alive=True, user_agent="Batman/1.0") {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} @@ -57,12 +59,13 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, if basic_auth: headers['authorization'] = 'Basic ' + \ - b64encode(six.b(basic_auth)).decode('utf-8') + b64encode(b(basic_auth)).decode('utf-8') if proxy_basic_auth: headers['proxy-authorization'] = 'Basic ' + \ - b64encode(six.b(proxy_basic_auth)).decode('utf-8') - - return headers + b64encode(b(proxy_basic_auth)).decode('utf-8') + if disable_cache: + headers['cache-control'] = 'no-cache' + return headers diff --git a/libs/requests/packages/urllib3/util/response.py b/libs/requests/packages/urllib3/util/response.py index d0325bc..45fff55 100644 --- a/libs/requests/packages/urllib3/util/response.py +++ b/libs/requests/packages/urllib3/util/response.py @@ -5,9 +5,18 @@ def is_fp_closed(obj): :param obj: The file-like object to check. """ - if hasattr(obj, 'fp'): - # Object is a container for another file-like object that gets released - # on exhaustion (e.g. HTTPResponse) + + try: + # Check via the official file-like-object way. + return obj.closed + except AttributeError: + pass + + try: + # Check if the object is a container for another file-like object that + # gets released on exhaustion (e.g. HTTPResponse). return obj.fp is None + except AttributeError: + pass - return obj.closed + raise ValueError("Unable to determine whether fp is closed.") diff --git a/libs/requests/packages/urllib3/util/retry.py b/libs/requests/packages/urllib3/util/retry.py new file mode 100644 index 0000000..eb560df --- /dev/null +++ b/libs/requests/packages/urllib3/util/retry.py @@ -0,0 +1,279 @@ +import time +import logging + +from ..exceptions import ( + ProtocolError, + ConnectTimeoutError, + ReadTimeoutError, + MaxRetryError, +) +from ..packages import six + + +log = logging.getLogger(__name__) + + +class Retry(object): + """ Retry configuration. + + Each retry attempt will create a new Retry object with updated values, so + they can be safely reused. + + Retries can be defined as a default for a pool:: + + retries = Retry(connect=5, read=2, redirect=5) + http = PoolManager(retries=retries) + response = http.request('GET', 'http://example.com/') + + Or per-request (which overrides the default for the pool):: + + response = http.request('GET', 'http://example.com/', retries=Retry(10)) + + Retries can be disabled by passing ``False``:: + + response = http.request('GET', 'http://example.com/', retries=False) + + Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless + retries are disabled, in which case the causing exception will be raised. + + + :param int total: + Total number of retries to allow. Takes precedence over other counts. + + Set to ``None`` to remove this constraint and fall back on other + counts. It's a good idea to set this to some sensibly-high value to + account for unexpected edge cases and avoid infinite retry loops. + + Set to ``0`` to fail on the first retry. + + Set to ``False`` to disable and imply ``raise_on_redirect=False``. + + :param int connect: + How many connection-related errors to retry on. + + These are errors raised before the request is sent to the remote server, + which we assume has not triggered the server to process the request. + + Set to ``0`` to fail on the first retry of this type. + + :param int read: + How many times to retry on read errors. + + These errors are raised after the request was sent to the server, so the + request may have side-effects. + + Set to ``0`` to fail on the first retry of this type. + + :param int redirect: + How many redirects to perform. Limit this to avoid infinite redirect + loops. + + A redirect is a HTTP response with a status code 301, 302, 303, 307 or + 308. + + Set to ``0`` to fail on the first retry of this type. + + Set to ``False`` to disable and imply ``raise_on_redirect=False``. + + :param iterable method_whitelist: + Set of uppercased HTTP method verbs that we should retry on. + + By default, we only retry on methods which are considered to be + indempotent (multiple requests with the same parameters end with the + same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`. + + :param iterable status_forcelist: + A set of HTTP status codes that we should force a retry on. + + By default, this is disabled with ``None``. + + :param float backoff_factor: + A backoff factor to apply between attempts. urllib3 will sleep for:: + + {backoff factor} * (2 ^ ({number of total retries} - 1)) + + seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep + for [0.1s, 0.2s, 0.4s, ...] between retries. It will never be longer + than :attr:`Retry.MAX_BACKOFF`. + + By default, backoff is disabled (set to 0). + + :param bool raise_on_redirect: Whether, if the number of redirects is + exhausted, to raise a MaxRetryError, or to return a response with a + response code in the 3xx range. + """ + + DEFAULT_METHOD_WHITELIST = frozenset([ + 'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE']) + + #: Maximum backoff time. + BACKOFF_MAX = 120 + + def __init__(self, total=10, connect=None, read=None, redirect=None, + method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, + backoff_factor=0, raise_on_redirect=True, _observed_errors=0): + + self.total = total + self.connect = connect + self.read = read + + if redirect is False or total is False: + redirect = 0 + raise_on_redirect = False + + self.redirect = redirect + self.status_forcelist = status_forcelist or set() + self.method_whitelist = method_whitelist + self.backoff_factor = backoff_factor + self.raise_on_redirect = raise_on_redirect + self._observed_errors = _observed_errors # TODO: use .history instead? + + def new(self, **kw): + params = dict( + total=self.total, + connect=self.connect, read=self.read, redirect=self.redirect, + method_whitelist=self.method_whitelist, + status_forcelist=self.status_forcelist, + backoff_factor=self.backoff_factor, + raise_on_redirect=self.raise_on_redirect, + _observed_errors=self._observed_errors, + ) + params.update(kw) + return type(self)(**params) + + @classmethod + def from_int(cls, retries, redirect=True, default=None): + """ Backwards-compatibility for the old retries format.""" + if retries is None: + retries = default if default is not None else cls.DEFAULT + + if isinstance(retries, Retry): + return retries + + redirect = bool(redirect) and None + new_retries = cls(retries, redirect=redirect) + log.debug("Converted retries value: %r -> %r" % (retries, new_retries)) + return new_retries + + def get_backoff_time(self): + """ Formula for computing the current backoff + + :rtype: float + """ + if self._observed_errors <= 1: + return 0 + + backoff_value = self.backoff_factor * (2 ** (self._observed_errors - 1)) + return min(self.BACKOFF_MAX, backoff_value) + + def sleep(self): + """ Sleep between retry attempts using an exponential backoff. + + By default, the backoff factor is 0 and this method will return + immediately. + """ + backoff = self.get_backoff_time() + if backoff <= 0: + return + time.sleep(backoff) + + def _is_connection_error(self, err): + """ Errors when we're fairly sure that the server did not receive the + request, so it should be safe to retry. + """ + return isinstance(err, ConnectTimeoutError) + + def _is_read_error(self, err): + """ Errors that occur after the request has been started, so we can't + assume that the server did not process any of it. + """ + return isinstance(err, (ReadTimeoutError, ProtocolError)) + + def is_forced_retry(self, method, status_code): + """ Is this method/response retryable? (Based on method/codes whitelists) + """ + if self.method_whitelist and method.upper() not in self.method_whitelist: + return False + + return self.status_forcelist and status_code in self.status_forcelist + + def is_exhausted(self): + """ Are we out of retries? + """ + retry_counts = (self.total, self.connect, self.read, self.redirect) + retry_counts = list(filter(None, retry_counts)) + if not retry_counts: + return False + + return min(retry_counts) < 0 + + def increment(self, method=None, url=None, response=None, error=None, _pool=None, _stacktrace=None): + """ Return a new Retry object with incremented retry counters. + + :param response: A response object, or None, if the server did not + return a response. + :type response: :class:`~urllib3.response.HTTPResponse` + :param Exception error: An error encountered during the request, or + None if the response was received successfully. + + :return: A new ``Retry`` object. + """ + if self.total is False and error: + # Disabled, indicate to re-raise the error. + raise six.reraise(type(error), error, _stacktrace) + + total = self.total + if total is not None: + total -= 1 + + _observed_errors = self._observed_errors + connect = self.connect + read = self.read + redirect = self.redirect + + if error and self._is_connection_error(error): + # Connect retry? + if connect is False: + raise six.reraise(type(error), error, _stacktrace) + elif connect is not None: + connect -= 1 + _observed_errors += 1 + + elif error and self._is_read_error(error): + # Read retry? + if read is False: + raise six.reraise(type(error), error, _stacktrace) + elif read is not None: + read -= 1 + _observed_errors += 1 + + elif response and response.get_redirect_location(): + # Redirect retry? + if redirect is not None: + redirect -= 1 + + else: + # FIXME: Nothing changed, scenario doesn't make sense. + _observed_errors += 1 + + new_retry = self.new( + total=total, + connect=connect, read=read, redirect=redirect, + _observed_errors=_observed_errors) + + if new_retry.is_exhausted(): + raise MaxRetryError(_pool, url, error) + + log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry)) + + return new_retry + + + def __repr__(self): + return ('{cls.__name__}(total={self.total}, connect={self.connect}, ' + 'read={self.read}, redirect={self.redirect})').format( + cls=type(self), self=self) + + +# For backwards compatibility (equivalent to pre-v1.9): +Retry.DEFAULT = Retry(3) diff --git a/libs/requests/packages/urllib3/util/ssl_.py b/libs/requests/packages/urllib3/util/ssl_.py index dee4b87..9cfe2d2 100644 --- a/libs/requests/packages/urllib3/util/ssl_.py +++ b/libs/requests/packages/urllib3/util/ssl_.py @@ -34,10 +34,9 @@ def assert_fingerprint(cert, fingerprint): } fingerprint = fingerprint.replace(':', '').lower() + digest_length, odd = divmod(len(fingerprint), 2) - digest_length, rest = divmod(len(fingerprint), 2) - - if rest or digest_length not in hashfunc_map: + if odd or digest_length not in hashfunc_map: raise SSLError('Fingerprint is of invalid length.') # We need encode() here for py32; works on py2 and p33. diff --git a/libs/requests/packages/urllib3/util/timeout.py b/libs/requests/packages/urllib3/util/timeout.py index 4f947cb..ea7027f 100644 --- a/libs/requests/packages/urllib3/util/timeout.py +++ b/libs/requests/packages/urllib3/util/timeout.py @@ -1,32 +1,49 @@ +# The default socket timeout, used by httplib to indicate that no timeout was +# specified by the user from socket import _GLOBAL_DEFAULT_TIMEOUT import time from ..exceptions import TimeoutStateError +# A sentinel value to indicate that no timeout was specified by the user in +# urllib3 +_Default = object() def current_time(): """ - Retrieve the current time, this function is mocked out in unit testing. + Retrieve the current time. This function is mocked out in unit testing. """ return time.time() -_Default = object() -# The default timeout to use for socket connections. This is the attribute used -# by httplib to define the default timeout +class Timeout(object): + """ Timeout configuration. + Timeouts can be defined as a default for a pool:: -class Timeout(object): - """ - Utility object for storing timeout values. + timeout = Timeout(connect=2.0, read=7.0) + http = PoolManager(timeout=timeout) + response = http.request('GET', 'http://example.com/') + + Or per-request (which overrides the default for the pool):: + + response = http.request('GET', 'http://example.com/', timeout=Timeout(10)) + + Timeouts can be disabled by setting all the parameters to ``None``:: - Example usage: + no_timeout = Timeout(connect=None, read=None) + response = http.request('GET', 'http://example.com/, timeout=no_timeout) - .. code-block:: python - timeout = urllib3.util.Timeout(connect=2.0, read=7.0) - pool = HTTPConnectionPool('www.google.com', 80, timeout=timeout) - pool.request(...) # Etc, etc + :param total: + This combines the connect and read timeouts into one; the read timeout + will be set to the time leftover from the connect attempt. In the + event that both a connect timeout and a total are specified, or a read + timeout and a total are specified, the shorter timeout will be applied. + + Defaults to None. + + :type total: integer, float, or None :param connect: The maximum amount of time to wait for a connection attempt to a server @@ -47,25 +64,15 @@ class Timeout(object): :type read: integer, float, or None - :param total: - This combines the connect and read timeouts into one; the read timeout - will be set to the time leftover from the connect attempt. In the - event that both a connect timeout and a total are specified, or a read - timeout and a total are specified, the shorter timeout will be applied. - - Defaults to None. - - :type total: integer, float, or None - .. note:: Many factors can affect the total amount of time for urllib3 to return - an HTTP response. Specifically, Python's DNS resolver does not obey the - timeout specified on the socket. Other factors that can affect total - request time include high CPU load, high swap, the program running at a - low priority level, or other behaviors. The observed running time for - urllib3 to return a response may be greater than the value passed to - `total`. + an HTTP response. + + For example, Python's DNS resolver does not obey the timeout specified + on the socket. Other factors that can affect total request time include + high CPU load, high swap, the program running at a low priority level, + or other behaviors. In addition, the read and total timeouts only measure the time between read operations on the socket connecting the client and the server, @@ -73,8 +80,8 @@ class Timeout(object): response. For most requests, the timeout is raised because the server has not sent the first byte in the specified time. This is not always the case; if a server streams one byte every fifteen seconds, a timeout - of 20 seconds will not ever trigger, even though the request will - take several minutes to complete. + of 20 seconds will not trigger, even though the request will take + several minutes to complete. If your goal is to cut off any request after a set amount of wall clock time, consider having a second "watcher" thread to cut off a slow @@ -94,17 +101,16 @@ class Timeout(object): return '%s(connect=%r, read=%r, total=%r)' % ( type(self).__name__, self._connect, self._read, self.total) - @classmethod def _validate_timeout(cls, value, name): - """ Check that a timeout attribute is valid + """ Check that a timeout attribute is valid. :param value: The timeout value to validate - :param name: The name of the timeout attribute to validate. This is used - for clear error messages - :return: the value - :raises ValueError: if the type is not an integer or a float, or if it - is a numeric value less than zero + :param name: The name of the timeout attribute to validate. This is + used to specify in error messages. + :return: The validated and casted version of the given value. + :raises ValueError: If the type is not an integer or a float, or if it + is a numeric value less than zero. """ if value is _Default: return cls.DEFAULT_TIMEOUT @@ -123,7 +129,7 @@ class Timeout(object): raise ValueError("Attempted to set %s timeout to %s, but the " "timeout cannot be set to a value less " "than 0." % (name, value)) - except TypeError: # Python 3 + except TypeError: # Python 3 raise ValueError("Timeout value %s was %s, but it must be an " "int or float." % (name, value)) @@ -135,12 +141,12 @@ class Timeout(object): The timeout value used by httplib.py sets the same timeout on the connect(), and recv() socket requests. This creates a :class:`Timeout` - object that sets the individual timeouts to the ``timeout`` value passed - to this function. + object that sets the individual timeouts to the ``timeout`` value + passed to this function. - :param timeout: The legacy timeout value + :param timeout: The legacy timeout value. :type timeout: integer, float, sentinel default object, or None - :return: a Timeout object + :return: Timeout object :rtype: :class:`Timeout` """ return Timeout(read=timeout, connect=timeout) @@ -174,7 +180,7 @@ class Timeout(object): def get_connect_duration(self): """ Gets the time elapsed since the call to :meth:`start_connect`. - :return: the elapsed time + :return: Elapsed time. :rtype: float :raises urllib3.exceptions.TimeoutStateError: if you attempt to get duration for a timer that hasn't been started. @@ -191,7 +197,7 @@ class Timeout(object): This will be a positive float or integer, the value None (never timeout), or the default system timeout. - :return: the connect timeout + :return: Connect timeout. :rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None """ if self.total is None: @@ -214,7 +220,7 @@ class Timeout(object): established, a :exc:`~urllib3.exceptions.TimeoutStateError` will be raised. - :return: the value to use for the read timeout + :return: Value to use for the read timeout. :rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None :raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect` has not yet been called on this object. @@ -223,7 +229,7 @@ class Timeout(object): self.total is not self.DEFAULT_TIMEOUT and self._read is not None and self._read is not self.DEFAULT_TIMEOUT): - # in case the connect timeout has not yet been established. + # In case the connect timeout has not yet been established. if self._start_connect is None: return self._read return max(0, min(self.total - self.get_connect_duration(), diff --git a/libs/requests/packages/urllib3/util/url.py b/libs/requests/packages/urllib3/util/url.py index 362d216..487d456 100644 --- a/libs/requests/packages/urllib3/util/url.py +++ b/libs/requests/packages/urllib3/util/url.py @@ -3,15 +3,20 @@ from collections import namedtuple from ..exceptions import LocationParseError -class Url(namedtuple('Url', ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'])): +url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] + + +class Url(namedtuple('Url', url_attrs)): """ Datastructure for representing an HTTP URL. Used as a return value for :func:`parse_url`. """ slots = () - def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None): - return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment) + def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, + query=None, fragment=None): + return super(Url, cls).__new__(cls, scheme, auth, host, port, path, + query, fragment) @property def hostname(self): @@ -43,7 +48,7 @@ def split_first(s, delims): If not found, then the first part is the full input string. - Example: :: + Example:: >>> split_first('foo/bar?baz', '?/=') ('foo', 'bar?baz', '/') @@ -76,7 +81,7 @@ def parse_url(url): Partly backwards-compatible with :mod:`urlparse`. - Example: :: + Example:: >>> parse_url('http://google.com/mail/') Url(scheme='http', host='google.com', port=None, path='/', ...) @@ -91,6 +96,10 @@ def parse_url(url): # Additionally, this implementations does silly things to be optimal # on CPython. + if not url: + # Empty + return Url() + scheme = None auth = None host = None diff --git a/libs/requests/sessions.py b/libs/requests/sessions.py index df85a25..508b0ef 100644 --- a/libs/requests/sessions.py +++ b/libs/requests/sessions.py @@ -91,10 +91,17 @@ class SessionRedirectMixin(object): """Receives a Response. Returns a generator of Responses.""" i = 0 + hist = [] # keep track of history while resp.is_redirect: prepared_request = req.copy() + if i > 0: + # Update history and keep track of redirects. + hist.append(resp) + new_hist = list(hist) + resp.history = new_hist + try: resp.content # Consume socket so it can be released except (ChunkedEncodingError, ContentDecodingError, RuntimeError): @@ -118,7 +125,7 @@ class SessionRedirectMixin(object): parsed = urlparse(url) url = parsed.geturl() - # Facilitate non-RFC2616-compliant 'location' headers + # Facilitate relative 'location' headers, as allowed by RFC 7231. # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') # Compliant with RFC3986, we percent encode the url. if not urlparse(url).netloc: @@ -127,8 +134,11 @@ class SessionRedirectMixin(object): url = requote_uri(url) prepared_request.url = to_native_string(url) + # cache the url + if resp.is_permanent_redirect: + self.redirect_cache[req.url] = prepared_request.url - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 + # http://tools.ietf.org/html/rfc7231#section-6.4.4 if (resp.status_code == codes.see_other and method != 'HEAD'): method = 'GET' @@ -146,7 +156,7 @@ class SessionRedirectMixin(object): prepared_request.method = method # https://github.com/kennethreitz/requests/issues/1084 - if resp.status_code not in (codes.temporary, codes.resume): + if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect): if 'Content-Length' in prepared_request.headers: del prepared_request.headers['Content-Length'] @@ -263,7 +273,7 @@ class Session(SessionRedirectMixin): __attrs__ = [ 'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks', 'params', 'verify', 'cert', 'prefetch', 'adapters', 'stream', - 'trust_env', 'max_redirects'] + 'trust_env', 'max_redirects', 'redirect_cache'] def __init__(self): @@ -316,6 +326,8 @@ class Session(SessionRedirectMixin): self.mount('https://', HTTPAdapter()) self.mount('http://', HTTPAdapter()) + self.redirect_cache = {} + def __enter__(self): return self @@ -423,36 +435,16 @@ class Session(SessionRedirectMixin): proxies = proxies or {} - # Gather clues from the surrounding environment. - if self.trust_env: - # Set environment's proxies. - env_proxies = get_environ_proxies(url) or {} - for (k, v) in env_proxies.items(): - proxies.setdefault(k, v) - - # Look for configuration. - if not verify and verify is not False: - verify = os.environ.get('REQUESTS_CA_BUNDLE') - - # Curl compatibility. - if not verify and verify is not False: - verify = os.environ.get('CURL_CA_BUNDLE') - - # Merge all the kwargs. - proxies = merge_setting(proxies, self.proxies) - stream = merge_setting(stream, self.stream) - verify = merge_setting(verify, self.verify) - cert = merge_setting(cert, self.cert) + settings = self.merge_environment_settings( + prep.url, proxies, stream, verify, cert + ) # Send the request. send_kwargs = { - 'stream': stream, 'timeout': timeout, - 'verify': verify, - 'cert': cert, - 'proxies': proxies, 'allow_redirects': allow_redirects, } + send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) return resp @@ -540,6 +532,9 @@ class Session(SessionRedirectMixin): if not isinstance(request, PreparedRequest): raise ValueError('You can only send PreparedRequests.') + while request.url in self.redirect_cache: + request.url = self.redirect_cache.get(request.url) + # Set up variables needed for resolve_redirects and dispatching of hooks allow_redirects = kwargs.pop('allow_redirects', True) stream = kwargs.get('stream') @@ -597,6 +592,30 @@ class Session(SessionRedirectMixin): return r + def merge_environment_settings(self, url, proxies, stream, verify, cert): + """Check the environment and merge it with some settings.""" + # Gather clues from the surrounding environment. + if self.trust_env: + # Set environment's proxies. + env_proxies = get_environ_proxies(url) or {} + for (k, v) in env_proxies.items(): + proxies.setdefault(k, v) + + # Look for requests environment configuration and be compatible + # with cURL. + if verify is True or verify is None: + verify = (os.environ.get('REQUESTS_CA_BUNDLE') or + os.environ.get('CURL_CA_BUNDLE')) + + # Merge all the kwargs. + proxies = merge_setting(proxies, self.proxies) + stream = merge_setting(stream, self.stream) + verify = merge_setting(verify, self.verify) + cert = merge_setting(cert, self.cert) + + return {'verify': verify, 'proxies': proxies, 'stream': stream, + 'cert': cert} + def get_adapter(self, url): """Returns the appropriate connnection adapter for the given URL.""" for (prefix, adapter) in self.adapters.items(): diff --git a/libs/requests/status_codes.py b/libs/requests/status_codes.py index ed7a866..e0887f2 100644 --- a/libs/requests/status_codes.py +++ b/libs/requests/status_codes.py @@ -30,7 +30,8 @@ _codes = { 305: ('use_proxy',), 306: ('switch_proxy',), 307: ('temporary_redirect', 'temporary_moved', 'temporary'), - 308: ('resume_incomplete', 'resume'), + 308: ('permanent_redirect', + 'resume_incomplete', 'resume',), # These 2 to be removed in 3.0 # Client Error. 400: ('bad_request', 'bad'), diff --git a/libs/requests/structures.py b/libs/requests/structures.py index 66cdad8..3e5f2fa 100644 --- a/libs/requests/structures.py +++ b/libs/requests/structures.py @@ -23,7 +23,7 @@ class CaseInsensitiveDict(collections.MutableMapping): case of the last key to be set, and ``iter(instance)``, ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()`` will contain case-sensitive keys. However, querying and contains - testing is case insensitive: + testing is case insensitive:: cid = CaseInsensitiveDict() cid['Accept'] = 'application/json' diff --git a/libs/requests/utils.py b/libs/requests/utils.py index 68e50cf..2c6bb09 100644 --- a/libs/requests/utils.py +++ b/libs/requests/utils.py @@ -554,7 +554,8 @@ def default_headers(): return CaseInsensitiveDict({ 'User-Agent': default_user_agent(), 'Accept-Encoding': ', '.join(('gzip', 'deflate')), - 'Accept': '*/*' + 'Accept': '*/*', + 'Connection': 'keep-alive' }) From 1de457fa8d2d7ba6e489738d65d95fb6d865449a Mon Sep 17 00:00:00 2001 From: Troy Olson Date: Wed, 3 Sep 2014 12:33:04 -0700 Subject: [PATCH 097/202] Update iptorrents.py Adds another URL parameter when searching on iptorrents for brrip. This allows it to find brrip that are also classified as 720p/1080p. See this thread for info: https://couchpota.to/forum/viewtopic.php?f=5&t=4032&sid=7892abbaaca9dad8bd3cc27cafb7fd33 And this prior pull request for more info: https://github.com/RuudBurger/CouchPotatoServer/pull/3696 --- couchpotato/core/media/movie/providers/torrent/iptorrents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/providers/torrent/iptorrents.py b/couchpotato/core/media/movie/providers/torrent/iptorrents.py index 89aeee8..1c75feb 100644 --- a/couchpotato/core/media/movie/providers/torrent/iptorrents.py +++ b/couchpotato/core/media/movie/providers/torrent/iptorrents.py @@ -13,7 +13,7 @@ class IPTorrents(MovieProvider, Base): ([87], ['3d']), ([48], ['720p', '1080p', 'bd50']), ([72], ['cam', 'ts', 'tc', 'r5', 'scr']), - ([7], ['dvdrip', 'brrip']), + ([7,48], ['dvdrip', 'brrip']), ([6], ['dvdr']), ] From bc6d19700429cf5a6aab611018be4fb55aef5087 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 7 Sep 2014 21:40:54 +0200 Subject: [PATCH 098/202] Don't score identifier quality.guess twice --- couchpotato/core/plugins/quality/main.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index baeca67..1c83614 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -293,10 +293,6 @@ class QualityPlugin(Plugin): log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) score += points.get(tag_type) - if list(set(qualities) & set(words)): - log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) - score += points.get(tag_type) - # Check extention for ext in quality.get('ext', []): if ext == extension: From ab253f90302e64ee09ca6b37358da0ad338bc78a Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 7 Sep 2014 21:41:59 +0200 Subject: [PATCH 099/202] Add quality test --- couchpotato/core/plugins/quality/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 1c83614..f5d890a 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -482,6 +482,8 @@ class QualityPlugin(Plugin): 'Movie Name (2014).mkv': {'size': 4500, 'quality': '720p', 'extra': {'titles': ['Movie Name 2014 720p Bluray']}}, 'Movie Name (2015).mkv': {'size': 500, 'quality': '1080p', 'extra': {'resolution_width': 1920}}, 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, + 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, + 'Movie Name.2014.720p Web-Dl Aac2.0 h264-ReleaseGroup': {'size': 3800, 'quality': 'brrip'}, } correct = 0 From 4879bc62516167cfab16a21cb818ce65c3cf1c06 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 7 Sep 2014 21:42:45 +0200 Subject: [PATCH 100/202] Move hdtv & hdrip to alternative for brrips --- couchpotato/core/plugins/quality/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index f5d890a..3daac01 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -26,7 +26,7 @@ class QualityPlugin(Plugin): {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'median_size': 40000, 'label': 'BR-Disk', 'alternative': ['bd25', ('br', 'disk')], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']}, {'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'median_size': 10000, 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts'], 'tags': ['m2ts', 'x264', 'h264']}, {'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'median_size': 5500, 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']}, - {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'median_size': 2000, 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip')], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, + {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'median_size': 2000, 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip'), 'hdtv', 'hdrip'], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['webdl', ('web', 'dl')]}, {'identifier': 'dvdr', 'size': (3000, 10000), 'median_size': 4500, 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']}, {'identifier': 'dvdrip', 'size': (600, 2400), 'median_size': 1500, 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, {'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]}, From 4f646094b5dcdc03f759cd1e637a24bdba6322fd Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 7 Sep 2014 21:50:08 +0200 Subject: [PATCH 101/202] Add quality test --- couchpotato/core/plugins/quality/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 3daac01..b5d8238 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -484,6 +484,7 @@ class QualityPlugin(Plugin): 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, 'Movie Name.2014.720p Web-Dl Aac2.0 h264-ReleaseGroup': {'size': 3800, 'quality': 'brrip'}, + 'Movie Name.2014.720p.WEBRip.x264.AC3-MAJESTiC': {'size': 3000, 'quality': 'scr'}, } correct = 0 From af8806e2928dacb7df638127fbfd9071fd14565c Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 7 Sep 2014 21:50:20 +0200 Subject: [PATCH 102/202] Move webrip to scr alternative --- couchpotato/core/plugins/quality/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index b5d8238..352cf79 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -29,7 +29,7 @@ class QualityPlugin(Plugin): {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'median_size': 2000, 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip'), 'hdtv', 'hdrip'], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['webdl', ('web', 'dl')]}, {'identifier': 'dvdr', 'size': (3000, 10000), 'median_size': 4500, 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']}, {'identifier': 'dvdrip', 'size': (600, 2400), 'median_size': 1500, 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, - {'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]}, + {'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr', 'webrip', ('web', 'rip')], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': []}, {'identifier': 'r5', 'size': (600, 1000), 'median_size': 700, 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p'], 'ext':[]}, {'identifier': 'tc', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p'], 'ext':[]}, {'identifier': 'ts', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p'], 'ext':[]}, From ca24bf031c05faf83e5d8cfbaa17c75afa22fb28 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 8 Sep 2014 19:21:31 +0200 Subject: [PATCH 103/202] Change quality test --- couchpotato/core/plugins/quality/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 352cf79..f3fbefe 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -484,7 +484,7 @@ class QualityPlugin(Plugin): 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, 'Movie Name.2014.720p Web-Dl Aac2.0 h264-ReleaseGroup': {'size': 3800, 'quality': 'brrip'}, - 'Movie Name.2014.720p.WEBRip.x264.AC3-MAJESTiC': {'size': 3000, 'quality': 'scr'}, + 'Movie Name.2014.720p.WEBRip.x264.AC3-ReleaseGroup': {'size': 3000, 'quality': 'scr'}, } correct = 0 From fa054b6b3419595ecbe30560011fba96fb5f81fd Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 16 Sep 2014 13:55:44 +0200 Subject: [PATCH 104/202] Migration: Don't fail on missing release file --- couchpotato/core/database.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/database.py b/couchpotato/core/database.py index 10ae26c..09f2105 100644 --- a/couchpotato/core/database.py +++ b/couchpotato/core/database.py @@ -579,7 +579,10 @@ class Database(object): continue for f in release_files: - rfile = all_files[f.get('file_id')] + rfile = all_files.get(f.get('file_id')) + if not rfile: + continue + file_type = type_by_id.get(rfile.get('type_id')).get('identifier') if not release['files'].get(file_type): From 0358378cae45e76be1accfba32db8e37abc151bf Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 16 Sep 2014 15:16:32 +0200 Subject: [PATCH 105/202] Fix marshal data corrupted documents --- couchpotato/core/plugins/release/main.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 31c3e61..b8417c3 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -70,28 +70,45 @@ class Release(Plugin): db = get_db() # Get (and remove) parentless releases - releases = db.all('release', with_doc = True) + releases = db.all('release', with_doc = False) media_exist = [] + reindex = 0 for release in releases: if release.get('key') in media_exist: continue try: + + try: + doc = db.get('id', release.get('_id')) + except RecordDeleted: + reindex += 1 + continue + db.get('id', release.get('key')) media_exist.append(release.get('key')) try: - if release['doc'].get('status') == 'ignore': - release['doc']['status'] = 'ignored' - db.update(release['doc']) + if doc.get('status') == 'ignore': + doc['status'] = 'ignored' + db.update(doc) except: log.error('Failed fixing mis-status tag: %s', traceback.format_exc()) + except ValueError: + log.debug('Deleted corrupted document "%s": %s', (release.get('key'), traceback.format_exc(0))) + corrupted = db.get('id', release.get('key'), with_storage = False) + db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None) + reindex += 1 except RecordDeleted: - db.delete(release['doc']) - log.debug('Deleted orphaned release: %s', release['doc']) + db.delete(doc) + log.debug('Deleted orphaned release: %s', doc) + reindex += 1 except: log.debug('Failed cleaning up orphaned releases: %s', traceback.format_exc()) + if reindex > 0: + db.reindex() + del media_exist # get movies last_edit more than a week ago From 80df57f2b625231dfd76538f16428a303f7621ff Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 16 Sep 2014 22:00:54 +0200 Subject: [PATCH 106/202] Delete corrupted documents --- couchpotato/core/database.py | 12 ++++++++++++ couchpotato/core/media/_base/media/main.py | 4 +++- couchpotato/core/plugins/release/main.py | 6 ++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/database.py b/couchpotato/core/database.py index 09f2105..37841bf 100644 --- a/couchpotato/core/database.py +++ b/couchpotato/core/database.py @@ -32,6 +32,7 @@ class Database(object): addEvent('database.setup.after', self.startup_compact) addEvent('database.setup_index', self.setupIndex) + addEvent('database.delete_corrupted', self.deleteCorrupted) addEvent('app.migrate', self.migrate) addEvent('app.after_shutdown', self.close) @@ -147,6 +148,17 @@ class Database(object): return results + def deleteCorrupted(self, _id, traceback_error = ''): + + db = self.getDB() + + try: + log.debug('Deleted corrupted document "%s": %s', (_id, traceback_error)) + corrupted = db.get('id', _id, with_storage = False) + db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None) + except: + log.debug('Failed deleting corrupted: %s', traceback.format_exc()) + def reindex(self, **kwargs): success = True diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index d5e54d7..522d9fb 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -178,8 +178,10 @@ class MediaPlugin(MediaBase): continue yield doc - except RecordNotFound: + except (RecordDeleted, RecordNotFound): log.debug('Record not found, skipping: %s', ms['_id']) + except (ValueError, EOFError): + fireEvent('database.delete_corrupted', ms.get('_id'), traceback_error = traceback.format_exc(0)) else: yield ms diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index b8417c3..cc92b9f 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -95,9 +95,7 @@ class Release(Plugin): except: log.error('Failed fixing mis-status tag: %s', traceback.format_exc()) except ValueError: - log.debug('Deleted corrupted document "%s": %s', (release.get('key'), traceback.format_exc(0))) - corrupted = db.get('id', release.get('key'), with_storage = False) - db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None) + fireEvent('database.delete_corrupted', release.get('key'), traceback_error = traceback.format_exc(0)) reindex += 1 except RecordDeleted: db.delete(doc) @@ -112,7 +110,7 @@ class Release(Plugin): del media_exist # get movies last_edit more than a week ago - medias = fireEvent('media.with_status', ['done','active'], single = True) + medias = fireEvent('media.with_status', ['done', 'active'], single = True) for media in medias: if media.get('last_edit', 0) > (now - week): From 002ce4d4e1283e6a1782cec718dc75c8407048a9 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 16 Sep 2014 22:05:54 +0200 Subject: [PATCH 107/202] Ignore deleted media documents --- couchpotato/core/media/__init__.py | 12 ++++++------ couchpotato/core/media/_base/media/main.py | 4 ++++ couchpotato/core/media/movie/_base/main.py | 3 +++ couchpotato/core/media/movie/searcher.py | 8 +++++--- couchpotato/core/plugins/automation.py | 3 ++- couchpotato/core/plugins/dashboard.py | 7 ++++++- couchpotato/core/plugins/manage.py | 3 ++- 7 files changed, 28 insertions(+), 12 deletions(-) diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index 4e319fc..549ed0d 100755 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -26,9 +26,9 @@ class MediaBase(Plugin): def onComplete(): try: media = fireEvent('media.get', media_id, single = True) - event_name = '%s.searcher.single' % media.get('type') - - fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id), manual = True) + if media: + event_name = '%s.searcher.single' % media.get('type') + fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id), manual = True) except: log.error('Failed creating onComplete: %s', traceback.format_exc()) @@ -39,9 +39,9 @@ class MediaBase(Plugin): def notifyFront(): try: media = fireEvent('media.get', media_id, single = True) - event_name = '%s.update' % media.get('type') - - fireEvent('notify.frontend', type = event_name, data = media) + if media: + event_name = '%s.update' % media.get('type') + fireEvent('notify.frontend', type = event_name, data = media) except: log.error('Failed creating onComplete: %s', traceback.format_exc()) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 522d9fb..ccee1fe 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -282,6 +282,10 @@ class MediaPlugin(MediaBase): media = fireEvent('media.get', media_id, single = True) + # Skip if no media has been found + if not media: + continue + # Merge releases with movie dict medias.append(media) diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 0dc39e5..2757a92 100755 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -179,6 +179,9 @@ class MovieBase(MovieTypeBase): db.delete(rel) movie_dict = fireEvent('media.get', m['_id'], single = True) + if not movie_dict: + log.debug('Failed adding media, can\'t find it anymore') + return False if do_search and search_after: onComplete = self.createOnComplete(m['_id']) diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index 97181ae..7ebaaf5 100755 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -89,6 +89,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): for media_id in medias: media = fireEvent('media.get', media_id, single = True) + if not media: continue try: self.single(media, search_protocols, manual = manual) @@ -388,9 +389,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase): rel['status'] = 'ignored' db.update(rel) - movie_dict = fireEvent('media.get', media_id, single = True) - log.info('Trying next release for: %s', getTitle(movie_dict)) - self.single(movie_dict, manual = manual, force_download = force_download) + media = fireEvent('media.get', media_id, single = True) + if media: + log.info('Trying next release for: %s', getTitle(media)) + self.single(media, manual = manual, force_download = force_download) return True diff --git a/couchpotato/core/plugins/automation.py b/couchpotato/core/plugins/automation.py index 39d7c9e..e98a00a 100644 --- a/couchpotato/core/plugins/automation.py +++ b/couchpotato/core/plugins/automation.py @@ -46,7 +46,8 @@ class Automation(Plugin): break movie_dict = fireEvent('media.get', movie_id, single = True) - fireEvent('movie.searcher.single', movie_dict) + if movie_dict: + fireEvent('movie.searcher.single', movie_dict) return True diff --git a/couchpotato/core/plugins/dashboard.py b/couchpotato/core/plugins/dashboard.py index d4af7ad..afead44 100644 --- a/couchpotato/core/plugins/dashboard.py +++ b/couchpotato/core/plugins/dashboard.py @@ -1,5 +1,6 @@ import random as rndm import time +from CodernityDB.database import RecordDeleted from couchpotato import get_db from couchpotato.api import addApiView @@ -58,7 +59,11 @@ class Dashboard(Plugin): rndm.shuffle(active_ids) for media_id in active_ids: - media = db.get('id', media_id) + try: + media = db.get('id', media_id) + except RecordDeleted: + log.debug('Record already deleted: %s', media_id) + continue pp = profile_pre.get(media.get('profile_id')) if not pp: continue diff --git a/couchpotato/core/plugins/manage.py b/couchpotato/core/plugins/manage.py index 132f7ec..75c550b 100755 --- a/couchpotato/core/plugins/manage.py +++ b/couchpotato/core/plugins/manage.py @@ -234,7 +234,8 @@ class Manage(Plugin): total = self.in_progress[folder]['total'] movie_dict = fireEvent('media.get', identifier, single = True) - fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict)) + if movie_dict: + fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict)) return afterUpdate From f7eeaf3eda2b4144ebf3ec90c4b7488325519445 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 16 Sep 2014 22:44:33 +0200 Subject: [PATCH 108/202] Locking mechanism --- couchpotato/core/event.py | 2 +- couchpotato/core/media/movie/_base/main.py | 8 ++++++-- couchpotato/core/plugins/base.py | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/event.py b/couchpotato/core/event.py index 7246cde..35818e7 100644 --- a/couchpotato/core/event.py +++ b/couchpotato/core/event.py @@ -90,7 +90,7 @@ def fireEvent(name, *args, **kwargs): else: - e = Event(name = name, threads = 10, exc_info = True, traceback = True, lock = threading.RLock()) + e = Event(name = name, threads = 10, exc_info = True, traceback = True) for event in events[name]: e.handle(event['handler'], priority = event['priority']) diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 2757a92..1863d89 100755 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -271,6 +271,10 @@ class MovieBase(MovieTypeBase): if self.shuttingDown(): return + lock_key = 'media.get.%s' % media_id if media_id else identifier + self.acquireLock(lock_key) + + media = {} try: db = get_db() @@ -319,11 +323,11 @@ class MovieBase(MovieTypeBase): self.getPoster(media, image_urls) db.update(media) - return media except: log.error('Failed update media: %s', traceback.format_exc()) - return {} + self.releaseLock(lock_key) + return media def updateReleaseDate(self, media_id): """ diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 2c7f9e1..d9308f4 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -1,3 +1,4 @@ +import threading from urllib import quote from urlparse import urlparse import glob @@ -35,6 +36,8 @@ class Plugin(object): _needs_shutdown = False _running = None + _locks = {} + user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20130519 Firefox/24.0' http_last_use = {} http_time_between_calls = 0 @@ -393,3 +396,19 @@ class Plugin(object): def isEnabled(self): return self.conf(self.enabled_option) or self.conf(self.enabled_option) is None + + def acquireLock(self, key): + + lock = self._locks.get(key) + if not lock: + self._locks[key] = threading.RLock() + + log.debug('Acquiring lock: %s', key) + self._locks.get(key).acquire() + + def releaseLock(self, key): + + lock = self._locks.get(key) + if lock: + log.debug('Releasing lock: %s', key) + self._locks.get(key).release() From 4a9452672a77d14c6c0c8def313050b1d645b0eb Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 17 Sep 2014 12:59:24 +0200 Subject: [PATCH 109/202] Use status_code to stop requesting url --- couchpotato/core/media/_base/providers/nzb/newznab.py | 7 ++++--- couchpotato/core/plugins/base.py | 9 ++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/media/_base/providers/nzb/newznab.py b/couchpotato/core/media/_base/providers/nzb/newznab.py index 87ecc75..62b787d 100644 --- a/couchpotato/core/media/_base/providers/nzb/newznab.py +++ b/couchpotato/core/media/_base/providers/nzb/newznab.py @@ -187,11 +187,12 @@ class Base(NZBProvider, RSS): self.limits_reached[host] = False return data except HTTPError as e: - if e.response.status_code == 503: + sc = e.response.status_code + if sc in [503, 429]: response = e.read().lower() - if 'maximum api' in response or 'download limit' in response: + if sc == 429 or 'maximum api' in response or 'download limit' in response: if not self.limits_reached.get(host): - log.error('Limit reached for newznab provider: %s', host) + log.error('Limit reached / to many requests for newznab provider: %s', host) self.limits_reached[host] = time.time() return 'try_next' diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 2c7f9e1..ecc77b3 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -198,6 +198,7 @@ class Plugin(object): del self.http_failed_disabled[host] self.wait(host) + status_code = None try: kwargs = { @@ -212,6 +213,7 @@ class Plugin(object): log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.keys()] if isinstance(data, dict) else 'with data')) response = r.request(method, url, **kwargs) + status_code = response.status_code if response.status_code == requests.codes.ok: data = response.content else: @@ -224,6 +226,11 @@ class Plugin(object): # Save failed requests by hosts try: + + # To many requests + if status_code == 429: + self.http_failed_request[host] = time.time() + if not self.http_failed_request.get(host): self.http_failed_request[host] = 1 else: @@ -255,7 +262,7 @@ class Plugin(object): if wait > 0: log.debug('Waiting for %s, %d seconds', (self.getName(), wait)) - time.sleep(wait) + time.sleep(min(wait, 30)) def beforeCall(self, handler): self.isRunning('%s.%s' % (self.getName(), handler.__name__)) From 76322c0145335f2e46575b543f1220e026ae6e1c Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 17 Sep 2014 21:50:41 +0200 Subject: [PATCH 110/202] Don't save data in notification --- couchpotato/core/notifications/core/main.py | 9 +++++++-- couchpotato/core/notifications/core/static/notification.js | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/notifications/core/main.py b/couchpotato/core/notifications/core/main.py index fdc837a..c5f4793 100644 --- a/couchpotato/core/notifications/core/main.py +++ b/couchpotato/core/notifications/core/main.py @@ -155,9 +155,14 @@ class CoreNotifier(Notification): n = { '_t': 'notification', 'time': int(time.time()), - 'message': toUnicode(message), - 'data': data + 'message': toUnicode(message) } + + if data.get('sticky'): + n['sticky'] = True + if data.get('important'): + n['important'] = True + db.insert(n) self.frontend(type = listener, data = n) diff --git a/couchpotato/core/notifications/core/static/notification.js b/couchpotato/core/notifications/core/static/notification.js index 93bfa15..b388b28 100644 --- a/couchpotato/core/notifications/core/static/notification.js +++ b/couchpotato/core/notifications/core/static/notification.js @@ -50,7 +50,7 @@ var NotificationBase = new Class({ , 'top'); self.notifications.include(result); - if((result.data.important !== undefined || result.data.sticky !== undefined) && !result.read){ + if((result.important !== undefined || result.sticky !== undefined) && !result.read){ var sticky = true; App.trigger('message', [result.message, sticky, result]) } @@ -72,7 +72,7 @@ var NotificationBase = new Class({ if(!force_ids) { var rn = self.notifications.filter(function(n){ - return !n.read && n.data.important === undefined + return !n.read && n.important === undefined }); var ids = []; From 4c198f71168dd5e128a4cbbd09eb083858b65e26 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 17 Sep 2014 22:51:27 +0200 Subject: [PATCH 111/202] Ignore RecordDeleted on notification getter fix #3888 --- couchpotato/core/notifications/core/main.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/notifications/core/main.py b/couchpotato/core/notifications/core/main.py index c5f4793..fa8e9a7 100644 --- a/couchpotato/core/notifications/core/main.py +++ b/couchpotato/core/notifications/core/main.py @@ -3,6 +3,7 @@ import threading import time import traceback import uuid +from CodernityDB.database import RecordDeleted from couchpotato import get_db from couchpotato.api import addApiView, addNonBlockApiView @@ -270,11 +271,16 @@ class CoreNotifier(Notification): if init: db = get_db() - notifications = db.all('notification', with_doc = True) + notifications = db.all('notification') for n in notifications: - if n['doc'].get('time') > (time.time() - 604800): - messages.append(n['doc']) + + try: + doc = db.get('id', n.get('_id')) + if doc.get('time') > (time.time() - 604800): + messages.append(doc) + except RecordDeleted: + pass return { 'success': True, From f74b837faa072c4b0762e22ea107f317a8571b4c Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 17 Sep 2014 23:05:01 +0200 Subject: [PATCH 112/202] Ignore RecordDeleted in release for media call --- couchpotato/core/plugins/release/main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index cc92b9f..13ba5d4 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -543,11 +543,15 @@ class Release(Plugin): def forMedia(self, media_id): db = get_db() - raw_releases = list(db.get_many('release', media_id, with_doc = True)) + raw_releases = db.get_many('release', media_id) releases = [] for r in raw_releases: - releases.append(r['doc']) + try: + doc = db.get('id', r.get('_id')) + releases.append(doc) + except RecordDeleted: + pass releases = sorted(releases, key = lambda k: k.get('info', {}).get('score', 0), reverse = True) From 035b99bc8abe8cfc0f37e524cf1e806d9eb42718 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 17 Sep 2014 23:06:59 +0200 Subject: [PATCH 113/202] Don't use event when not needed --- couchpotato/core/plugins/release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 13ba5d4..0385e22 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -116,7 +116,7 @@ class Release(Plugin): if media.get('last_edit', 0) > (now - week): continue - for rel in fireEvent('release.for_media', media['_id'], single = True): + for rel in self.forMedia(media['_id']): # Remove all available releases if rel['status'] in ['available']: From c948216e33476339b2687a92362d76526049b514 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 14:17:00 +0200 Subject: [PATCH 114/202] I need to watch more Sesame Street.. 26 letters in the alphabet + # for numbers is 27.. --- couchpotato/core/media/_base/media/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index ccee1fe..4bcd820 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -379,7 +379,7 @@ class MediaPlugin(MediaBase): if x['_id'] in media_ids: chars.add(x['key']) - if len(chars) == 25: + if len(chars) == 27: break return list(chars) From 5fc9d7182c75dffe10afe1df47ed71ce5c8ad9a6 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 14:26:34 +0200 Subject: [PATCH 115/202] Hide urllib3 error closes #3887 --- couchpotato/runner.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 5d3f62b..1c6f577 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -19,6 +19,7 @@ from couchpotato.core.event import fireEventAsync, fireEvent from couchpotato.core.helpers.encoding import sp from couchpotato.core.helpers.variable import getDataDir, tryInt, getFreeSpace import requests +from requests.packages.urllib3 import disable_warnings from tornado.httpserver import HTTPServer from tornado.web import Application, StaticFileHandler, RedirectHandler @@ -174,6 +175,9 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En for logger_name in ['gntp']: logging.getLogger(logger_name).setLevel(logging.WARNING) + # Disable SSL warning + disable_warnings() + # Use reloader reloader = debug is True and development and not Env.get('desktop') and not options.daemon From 1b724b5606933daff50e0375137d9981a82ea4dd Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 16:04:21 +0200 Subject: [PATCH 116/202] Media got tagged with ignored, instead of release --- couchpotato/core/media/movie/searcher.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index 7ebaaf5..4940e7c 100755 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -382,12 +382,14 @@ class MovieSearcher(SearcherBase, MovieTypeBase): def tryNextRelease(self, media_id, manual = False, force_download = False): try: + db = get_db() - rels = fireEvent('media.with_status', ['snatched', 'done'], single = True) + rels = fireEvent('release.for_media', media_id, single = True) for rel in rels: - rel['status'] = 'ignored' - db.update(rel) + if rel.get('status') in ['snatched', 'done']: + rel['status'] = 'ignored' + db.update(rel) media = fireEvent('media.get', media_id, single = True) if media: From a3a8a820fedcf6d730b3b36b7aadd1c1dd61deb2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 16:49:44 +0200 Subject: [PATCH 117/202] release.update_status not triggered on frontend --- couchpotato/core/media/movie/_base/main.py | 3 +-- .../core/media/movie/_base/static/movie.actions.js | 15 +++++++++++---- couchpotato/core/media/movie/_base/static/movie.js | 14 +++++++++++--- couchpotato/core/media/movie/searcher.py | 4 +--- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 1863d89..75e10c7 100755 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -150,8 +150,7 @@ class MovieBase(MovieTypeBase): for release in fireEvent('release.for_media', m['_id'], single = True): if release.get('status') in ['downloaded', 'snatched', 'seeding', 'done']: if params.get('ignore_previous', False): - release['status'] = 'ignored' - db.update(release) + fireEvent('release.update_status', m['_id'], status = 'ignored') else: fireEvent('release.delete', release['_id'], single = True) diff --git a/couchpotato/core/media/movie/_base/static/movie.actions.js b/couchpotato/core/media/movie/_base/static/movie.actions.js index ff71f31..09a998f 100644 --- a/couchpotato/core/media/movie/_base/static/movie.actions.js +++ b/couchpotato/core/media/movie/_base/static/movie.actions.js @@ -115,8 +115,15 @@ MA.Release = new Class({ self.releases = null; if(self.options_container){ - self.options_container.destroy(); - self.options_container = null; + // Releases are currently displayed + if(self.options_container.isDisplayed()){ + self.options_container.destroy(); + self.createReleases(); + } + else { + self.options_container.destroy(); + self.options_container = null; + } } }); @@ -131,10 +138,10 @@ MA.Release = new Class({ }, - createReleases: function(){ + createReleases: function(refresh){ var self = this; - if(!self.options_container){ + if(!self.options_container || refresh){ self.options_container = new Element('div.options').grab( self.release_container = new Element('div.releases.table') ); diff --git a/couchpotato/core/media/movie/_base/static/movie.js b/couchpotato/core/media/movie/_base/static/movie.js index 669546b..e2977a1 100644 --- a/couchpotato/core/media/movie/_base/static/movie.js +++ b/couchpotato/core/media/movie/_base/static/movie.js @@ -54,13 +54,21 @@ var Movie = new Class({ // Reload when releases have updated self.global_events['release.update_status'] = function(notification){ var data = notification.data; - if(data && self.data._id == data.movie_id){ + if(data && self.data._id == data.media_id){ if(!self.data.releases) self.data.releases = []; - self.data.releases.push({'quality': data.quality, 'status': data.status}); - self.updateReleases(); + var updated = false; + self.data.releases.each(function(release){ + if(release._id == data._id){ + release['status'] = data.status; + updated = true; + } + }); + + if(updated) + self.updateReleases(); } }; diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index 4940e7c..50b2d44 100755 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -383,13 +383,11 @@ class MovieSearcher(SearcherBase, MovieTypeBase): try: - db = get_db() rels = fireEvent('release.for_media', media_id, single = True) for rel in rels: if rel.get('status') in ['snatched', 'done']: - rel['status'] = 'ignored' - db.update(rel) + fireEvent('release.update_status', rel.get('_id'), status = 'ignored') media = fireEvent('media.get', media_id, single = True) if media: From 052d64eb396e3d4caf000df5e722c1d89162433c Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 17:45:55 +0200 Subject: [PATCH 118/202] Force restatus on ignored movies --- couchpotato/core/media/_base/media/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 4bcd820..24b0124 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -77,6 +77,7 @@ class MediaPlugin(MediaBase): addEvent('app.load', self.addSingleListView, priority = 100) addEvent('app.load', self.addSingleCharView, priority = 100) addEvent('app.load', self.addSingleDeleteView, priority = 100) + addEvent('app.load', self.cleanupFaults) addEvent('media.get', self.get) addEvent('media.with_status', self.withStatus) @@ -87,6 +88,12 @@ class MediaPlugin(MediaBase): addEvent('media.tag', self.tag) addEvent('media.untag', self.unTag) + # Wrongly tagged media files + def cleanupFaults(self): + medias = fireEvent('media.with_status', 'ignored', with_doc = False, single = True) + for media in medias: + self.restatus(media.get('_id')) + def refresh(self, id = '', **kwargs): handlers = [] ids = splitString(id) From 387650d0404b4882f1f2b6a2faa9bbddd621faa8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 17:51:21 +0200 Subject: [PATCH 119/202] Don't tag recent for fixed ignored movies --- couchpotato/core/media/_base/media/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 24b0124..c11fe0a 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -92,7 +92,7 @@ class MediaPlugin(MediaBase): def cleanupFaults(self): medias = fireEvent('media.with_status', 'ignored', with_doc = False, single = True) for media in medias: - self.restatus(media.get('_id')) + self.restatus(media.get('_id'), restatus = False) def refresh(self, id = '', **kwargs): handlers = [] @@ -485,7 +485,7 @@ class MediaPlugin(MediaBase): } }) - def restatus(self, media_id): + def restatus(self, media_id, tag_recent = True): try: db = get_db() @@ -524,7 +524,8 @@ class MediaPlugin(MediaBase): db.update(m) # Tag media as recent - self.tag(media_id, 'recent', update_edited = True) + if tag_recent: + self.tag(media_id, 'recent', update_edited = True) return m['status'] except: From be30200a18eab601db4bbe5d439680c8967d1a03 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 17:57:10 +0200 Subject: [PATCH 120/202] Use correct arg --- couchpotato/core/media/_base/media/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index c11fe0a..e0a7886 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -92,7 +92,7 @@ class MediaPlugin(MediaBase): def cleanupFaults(self): medias = fireEvent('media.with_status', 'ignored', with_doc = False, single = True) for media in medias: - self.restatus(media.get('_id'), restatus = False) + self.restatus(media.get('_id'), tag_recent = False) def refresh(self, id = '', **kwargs): handlers = [] From 41e69aeac302da5aaf2cd5294b34004512e949ec Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 19:52:10 +0200 Subject: [PATCH 121/202] Delete nzbindex provider --- .../core/media/_base/providers/nzb/nzbindex.py | 126 --------------------- .../core/media/movie/providers/nzb/nzbindex.py | 30 ----- 2 files changed, 156 deletions(-) delete mode 100644 couchpotato/core/media/_base/providers/nzb/nzbindex.py delete mode 100644 couchpotato/core/media/movie/providers/nzb/nzbindex.py diff --git a/couchpotato/core/media/_base/providers/nzb/nzbindex.py b/couchpotato/core/media/_base/providers/nzb/nzbindex.py deleted file mode 100644 index 58f4b23..0000000 --- a/couchpotato/core/media/_base/providers/nzb/nzbindex.py +++ /dev/null @@ -1,126 +0,0 @@ -import re -import time - -from bs4 import BeautifulSoup -from couchpotato.core.helpers.encoding import toUnicode -from couchpotato.core.helpers.rss import RSS -from couchpotato.core.helpers.variable import tryInt -from couchpotato.core.logger import CPLog -from couchpotato.core.event import fireEvent -from couchpotato.core.media._base.providers.nzb.base import NZBProvider -from dateutil.parser import parse - - -log = CPLog(__name__) - - -class Base(NZBProvider, RSS): - - urls = { - 'download': 'https://www.nzbindex.com/download/', - 'search': 'https://www.nzbindex.com/rss/?%s', - } - - http_time_between_calls = 1 # Seconds - - def _search(self, media, quality, results): - - nzbs = self.getRSSData(self.urls['search'] % self.buildUrl(media, quality)) - - for nzb in nzbs: - - enclosure = self.getElement(nzb, 'enclosure').attrib - nzbindex_id = int(self.getTextElement(nzb, "link").split('/')[4]) - - title = self.getTextElement(nzb, "title") - - match = fireEvent('matcher.parse', title, parser='usenet', single = True) - if not match.chains: - log.info('Unable to parse release with title "%s"', title) - continue - - # TODO should we consider other lower-weight chains here? - info = fireEvent('matcher.flatten_info', match.chains[0].info, single = True) - - release_name = fireEvent('matcher.construct_from_raw', info.get('release_name'), single = True) - - file_name = info.get('detail', {}).get('file_name') - file_name = file_name[0] if file_name else None - - title = release_name or file_name - - # Strip extension from parsed title (if one exists) - ext_pos = title.rfind('.') - - # Assume extension if smaller than 4 characters - # TODO this should probably be done a better way - if len(title[ext_pos + 1:]) <= 4: - title = title[:ext_pos] - - if not title: - log.info('Unable to find release name from match') - continue - - try: - description = self.getTextElement(nzb, "description") - except: - description = '' - - def extra_check(item): - if '#c20000' in item['description'].lower(): - log.info('Wrong: Seems to be passworded: %s', item['name']) - return False - - return True - - results.append({ - 'id': nzbindex_id, - 'name': title, - 'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, "pubDate")).timetuple()))), - 'size': tryInt(enclosure['length']) / 1024 / 1024, - 'url': enclosure['url'], - 'detail_url': enclosure['url'].replace('/download/', '/release/'), - 'description': description, - 'get_more_info': self.getMoreInfo, - 'extra_check': extra_check, - }) - - def getMoreInfo(self, item): - try: - if '/nfo/' in item['description'].lower(): - nfo_url = re.search('href=\"(?P.+)\" ', item['description']).group('nfo') - full_description = self.getCache('nzbindex.%s' % item['id'], url = nfo_url, cache_timeout = 25920000) - html = BeautifulSoup(full_description) - item['description'] = toUnicode(html.find('pre', attrs = {'id': 'nfo0'}).text) - except: - pass - - -config = [{ - 'name': 'nzbindex', - 'groups': [ - { - 'tab': 'searcher', - 'list': 'nzb_providers', - 'name': 'nzbindex', - 'description': 'Free provider, less accurate. See NZBIndex', - 'wizard': True, - 'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAo0lEQVR42t2SQQ2AMBAEcUCwUAv94QMLfHliAQtYqIVawEItYAG6yZFMLkUANNlk79Kbbtp2P1j9uKxVV9VWFeStl+Wh3fWK9hNwEoADZkJtMD49AqS5AUjWGx6A+m+ARICGrM5W+wSTB0gETKzdHZwCEZAJ8PGZQN4AiQAmkR9s06EBAugJiBoAAPFfAQcBgZcIHzwA6TYP4JsXeSg3P9L31w3eksbH3zMb/wAAAABJRU5ErkJggg==', - 'options': [ - { - 'name': 'enabled', - 'type': 'enabler', - 'default': True, - }, - { - 'name': 'extra_score', - 'advanced': True, - 'label': 'Extra Score', - 'type': 'int', - 'default': 0, - 'description': 'Starting score for each release found via this provider.', - } - ], - }, - ], -}] diff --git a/couchpotato/core/media/movie/providers/nzb/nzbindex.py b/couchpotato/core/media/movie/providers/nzb/nzbindex.py deleted file mode 100644 index 70e939d..0000000 --- a/couchpotato/core/media/movie/providers/nzb/nzbindex.py +++ /dev/null @@ -1,30 +0,0 @@ -from couchpotato.core.helpers.encoding import tryUrlencode -from couchpotato.core.logger import CPLog -from couchpotato.core.event import fireEvent -from couchpotato.core.media._base.providers.nzb.nzbindex import Base -from couchpotato.core.media.movie.providers.base import MovieProvider -from couchpotato.environment import Env - -log = CPLog(__name__) - -autoload = 'NzbIndex' - - -class NzbIndex(MovieProvider, Base): - - def buildUrl(self, media, quality): - title = fireEvent('library.query', media, include_year = False, single = True) - year = media['info']['year'] - - query = tryUrlencode({ - 'q': '"%s %s" | "%s (%s)"' % (title, year, title, year), - 'age': Env.setting('retention', 'nzb'), - 'sort': 'agedesc', - 'minsize': quality.get('size_min'), - 'maxsize': quality.get('size_max'), - 'rating': 1, - 'max': 250, - 'more': 1, - 'complete': 1, - }) - return query From 8e23b02653f6f6805dc30ed23dd11ad7aedeba84 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 20:19:49 +0200 Subject: [PATCH 122/202] Stop on 429 code --- couchpotato/core/plugins/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 73bc935..8a4cada 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -231,8 +231,9 @@ class Plugin(object): try: # To many requests - if status_code == 429: - self.http_failed_request[host] = time.time() + if status_code in [429]: + self.http_failed_request[host] = 1 + self.http_failed_disabled[host] = time.time() if not self.http_failed_request.get(host): self.http_failed_request[host] = 1 From 160bc1a5c472d8f1b4464a85d97b44c0bb8a124f Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 20:57:46 +0200 Subject: [PATCH 123/202] Always release lock --- libs/axl/axel.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/axl/axel.py b/libs/axl/axel.py index 64d2977..2abf12a 100644 --- a/libs/axl/axel.py +++ b/libs/axl/axel.py @@ -235,12 +235,12 @@ class Event(object): self.error_handler(sys.exc_info()) finally: - if not self.asynchronous: - self.queue.task_done() - if order_lock: order_lock.release() + if not self.asynchronous: + self.queue.task_done() + if self.queue.empty(): raise Empty From db4958581854ac70c942078629263699cedf7dc4 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 21:32:31 +0200 Subject: [PATCH 124/202] Renamer doesn't loop over all movies properly --- couchpotato/core/plugins/renamer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index fdc4329..9decf08 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -526,7 +526,7 @@ class Renamer(Plugin): # Remove leftover files if not remove_leftovers: # Don't remove anything - break + continue log.debug('Removing leftover files') for current_file in group['files']['leftover']: From 7fe5a271dc812b7db16f6d4816fd770b213dd3dc Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 22:00:59 +0200 Subject: [PATCH 125/202] Make sure original_folder isn't empty fix #3747 --- couchpotato/core/plugins/renamer.py | 3 +++ couchpotato/core/plugins/scanner.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 9decf08..4c7aa3b 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -344,6 +344,9 @@ class Renamer(Plugin): replacements['original'] = os.path.splitext(os.path.basename(current_file))[0] replacements['original_folder'] = fireEvent('scanner.remove_cptag', group['dirname'], single = True) + if not replacements['original_folder'] or len(replacements['original_folder']) == 0: + replacements['original_folder'] = replacements['original'] + # Extension replacements['ext'] = getExt(current_file) diff --git a/couchpotato/core/plugins/scanner.py b/couchpotato/core/plugins/scanner.py index 8e835b5..a1b5cf8 100644 --- a/couchpotato/core/plugins/scanner.py +++ b/couchpotato/core/plugins/scanner.py @@ -694,7 +694,7 @@ class Scanner(Plugin): def removeCPTag(self, name): try: - return re.sub(self.cp_imdb, '', name) + return re.sub(self.cp_imdb, '', name).strip() except: pass return name From 55e489cc51c134f2b373dcb44cfd21aa8715dc83 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Sep 2014 00:14:52 +0200 Subject: [PATCH 126/202] Mark faulty movies done --- couchpotato/core/media/_base/media/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index e0a7886..aa1ee38 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -90,9 +90,15 @@ class MediaPlugin(MediaBase): # Wrongly tagged media files def cleanupFaults(self): - medias = fireEvent('media.with_status', 'ignored', with_doc = False, single = True) + medias = fireEvent('media.with_status', 'ignored', single = True) or [] + + db = get_db() for media in medias: - self.restatus(media.get('_id'), tag_recent = False) + try: + media['status'] = 'done' + db.update(media) + except: + pass def refresh(self, id = '', **kwargs): handlers = [] From 2f0e197320e8c3f8dcbf3bfeb5340d74b250a762 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Sep 2014 00:14:52 +0200 Subject: [PATCH 127/202] Mark faulty movies done --- couchpotato/core/media/_base/media/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index e0a7886..aa1ee38 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -90,9 +90,15 @@ class MediaPlugin(MediaBase): # Wrongly tagged media files def cleanupFaults(self): - medias = fireEvent('media.with_status', 'ignored', with_doc = False, single = True) + medias = fireEvent('media.with_status', 'ignored', single = True) or [] + + db = get_db() for media in medias: - self.restatus(media.get('_id'), tag_recent = False) + try: + media['status'] = 'done' + db.update(media) + except: + pass def refresh(self, id = '', **kwargs): handlers = [] From 98540f2fcd8a91b14ff4abe4624f39e21b7b5540 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 22:00:59 +0200 Subject: [PATCH 128/202] Make sure original_folder isn't empty fix #3747 --- couchpotato/core/plugins/renamer.py | 3 +++ couchpotato/core/plugins/scanner.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 9decf08..4c7aa3b 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -344,6 +344,9 @@ class Renamer(Plugin): replacements['original'] = os.path.splitext(os.path.basename(current_file))[0] replacements['original_folder'] = fireEvent('scanner.remove_cptag', group['dirname'], single = True) + if not replacements['original_folder'] or len(replacements['original_folder']) == 0: + replacements['original_folder'] = replacements['original'] + # Extension replacements['ext'] = getExt(current_file) diff --git a/couchpotato/core/plugins/scanner.py b/couchpotato/core/plugins/scanner.py index 8e835b5..a1b5cf8 100644 --- a/couchpotato/core/plugins/scanner.py +++ b/couchpotato/core/plugins/scanner.py @@ -694,7 +694,7 @@ class Scanner(Plugin): def removeCPTag(self, name): try: - return re.sub(self.cp_imdb, '', name) + return re.sub(self.cp_imdb, '', name).strip() except: pass return name From 7c674b3aab59bdf0c04b3d27f98c08f8fc47dc35 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Sep 2014 10:03:02 +0200 Subject: [PATCH 129/202] Re-add movie giving rev conflict fix #3939 --- couchpotato/core/media/movie/_base/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 75e10c7..c5ed462 100755 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -150,7 +150,7 @@ class MovieBase(MovieTypeBase): for release in fireEvent('release.for_media', m['_id'], single = True): if release.get('status') in ['downloaded', 'snatched', 'seeding', 'done']: if params.get('ignore_previous', False): - fireEvent('release.update_status', m['_id'], status = 'ignored') + fireEvent('release.update_status', release['_id'], status = 'ignored') else: fireEvent('release.delete', release['_id'], single = True) From d70da1edcee260429f156ccc1365831fa71b40d8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Sep 2014 10:20:02 +0200 Subject: [PATCH 130/202] TorrentShack use correct columns fix #3940 --- couchpotato/core/media/_base/providers/torrent/torrentshack.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentshack.py b/couchpotato/core/media/_base/providers/torrent/torrentshack.py index 226993c..f56017f 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentshack.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentshack.py @@ -42,6 +42,7 @@ class Base(TorrentProvider): link = result.find('span', attrs = {'class': 'torrent_name_link'}).parent url = result.find('td', attrs = {'class': 'torrent_td'}).find('a') + tds = result.find_all('td') results.append({ 'id': link['href'].replace('torrents.php?torrentid=', ''), @@ -49,8 +50,8 @@ class Base(TorrentProvider): 'url': self.urls['download'] % url['href'], 'detail_url': self.urls['download'] % link['href'], 'size': self.parseSize(result.find_all('td')[5].string), - 'seeders': tryInt(result.find_all('td')[7].string), - 'leechers': tryInt(result.find_all('td')[8].string), + 'seeders': tryInt(tds[len(tds)-2].string), + 'leechers': tryInt(tds[len(tds)-1].string), }) except: From a0b3ee818641675f7d048aa196314c7157de2196 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Sep 2014 10:39:54 +0200 Subject: [PATCH 131/202] Safe encode path names in renamer fix #3425 --- couchpotato/core/plugins/renamer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 4c7aa3b..ad2199d 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -266,7 +266,7 @@ class Renamer(Plugin): category_label = category['label'] if category['destination'] and len(category['destination']) > 0 and category['destination'] != 'None': - destination = category['destination'] + destination = sp(category['destination']) log.debug('Setting category destination for "%s": %s' % (media_title, destination)) else: log.debug('No category destination found for "%s"' % media_title) @@ -369,6 +369,9 @@ class Renamer(Plugin): if separator: final_file_name = final_file_name.replace(' ', separator) + final_folder_name = ss(final_folder_name) + final_file_name = ss(final_file_name) + # Move DVD files (no structure renaming) if group['is_dvd'] and file_type is 'movie': found = False From 9b62e32da82d5e0d5c9532fbc95e89f0e8e2aaf5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Sep 2014 10:42:48 +0200 Subject: [PATCH 132/202] Symlink failing on encode fix #3371 --- couchpotato/core/plugins/renamer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index ad2199d..34ac555 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -802,9 +802,10 @@ Remove it if you want it to be renamed (again, or at least let it try again) log.debug('Couldn\'t hardlink file "%s" to "%s". Symlinking instead. Error: %s.', (old, dest, traceback.format_exc())) shutil.copy(old, dest) try: - symlink(dest, old + '.link') + old_link = '%s.link' % sp(old) + symlink(dest, old_link) os.unlink(old) - os.rename(old + '.link', old) + os.rename(old_link, old) except: log.error('Couldn\'t symlink file "%s" to "%s". Copied instead. Error: %s. ', (old, dest, traceback.format_exc())) From 3093b21555b3141986019352eefc217468ab0159 Mon Sep 17 00:00:00 2001 From: softcat Date: Sat, 20 Sep 2014 17:57:51 +0200 Subject: [PATCH 133/202] Fixed filmstarts.de provider --- couchpotato/core/media/movie/providers/userscript/filmstarts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/movie/providers/userscript/filmstarts.py b/couchpotato/core/media/movie/providers/userscript/filmstarts.py index 59027e0..4e61f29 100644 --- a/couchpotato/core/media/movie/providers/userscript/filmstarts.py +++ b/couchpotato/core/media/movie/providers/userscript/filmstarts.py @@ -25,6 +25,6 @@ class Filmstarts(UserscriptBase): name = html.find("meta", {"property":"og:title"})['content'] # Year of production is not available in the meta data, so get it from the table - year = table.find("tr", text="Produktionsjahr").parent.parent.parent.td.text + year = table.find(text="Produktionsjahr").parent.parent.next_sibling.text - return self.search(name, year) \ No newline at end of file + return self.search(name, year) From db367a80d121be54f014adce07b4aa0799b92653 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 20 Sep 2014 23:58:54 +0200 Subject: [PATCH 134/202] Do proper cleanup after rename --- couchpotato/core/plugins/base.py | 18 +++++++----------- couchpotato/core/plugins/renamer.py | 10 ++++++---- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 8a4cada..39b17a8 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -146,21 +146,17 @@ class Plugin(object): folder = sp(folder) for item in os.listdir(folder): - full_folder = os.path.join(folder, item) + full_folder = sp(os.path.join(folder, item)) if not only_clean or (item in only_clean and os.path.isdir(full_folder)): - for root, dirs, files in os.walk(full_folder): + for subfolder, dirs, files in os.walk(full_folder, topdown = False): - for dir_name in dirs: - full_path = os.path.join(root, dir_name) - - if len(os.listdir(full_path)) == 0: - try: - os.rmdir(full_path) - except: - if show_error: - log.info2('Couldn\'t remove directory %s: %s', (full_path, traceback.format_exc())) + try: + os.rmdir(subfolder) + except: + if show_error: + log.info2('Couldn\'t remove directory %s: %s', (subfolder, traceback.format_exc())) try: os.rmdir(folder) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 34ac555..3c506a4 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -555,9 +555,9 @@ class Renamer(Plugin): os.remove(src) parent_dir = os.path.dirname(src) - if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and \ + if parent_dir not in delete_folders and os.path.isdir(parent_dir) and \ not isSubFolder(destination, parent_dir) and not isSubFolder(media_folder, parent_dir) and \ - not isSubFolder(parent_dir, base_folder): + isSubFolder(parent_dir, base_folder): delete_folders.append(parent_dir) @@ -566,6 +566,7 @@ class Renamer(Plugin): self.tagRelease(group = group, tag = 'failed_remove') # Delete leftover folder from older releases + delete_folders = sorted(delete_folders, key = len, reverse = True) for delete_folder in delete_folders: try: self.deleteEmptyFolder(delete_folder, show_error = False) @@ -620,8 +621,9 @@ class Renamer(Plugin): group_folder = sp(os.path.join(base_folder, os.path.relpath(group['parentdir'], base_folder).split(os.path.sep)[0])) try: - log.info('Deleting folder: %s', group_folder) - self.deleteEmptyFolder(group_folder) + if self.conf('cleanup') or self.conf('move_leftover'): + log.info('Deleting folder: %s', group_folder) + self.deleteEmptyFolder(group_folder) except: log.error('Failed removing %s: %s', (group_folder, traceback.format_exc())) From c4db4ace132f8878e764f8f381eeca27f6942fe6 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 00:09:56 +0200 Subject: [PATCH 135/202] Log move, copy, link --- couchpotato/core/plugins/renamer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 3c506a4..af39aa3 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -579,7 +579,6 @@ class Renamer(Plugin): for src in rename_files: if rename_files[src]: dst = rename_files[src] - log.info('Renaming "%s" to "%s"', (src, dst)) # Create dir self.makeDir(os.path.dirname(dst)) @@ -785,6 +784,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) if move_type not in ['copy', 'link']: try: + log.info('Moving "%s" to "%s"', (old, dest)) shutil.move(old, dest) except: if os.path.exists(dest): @@ -793,8 +793,10 @@ Remove it if you want it to be renamed (again, or at least let it try again) else: raise elif move_type == 'copy': + log.info('Copying "%s" to "%s"', (old, dest)) shutil.copy(old, dest) else: + log.info('Linking "%s" to "%s"', (old, dest)) # First try to hardlink try: log.debug('Hardlinking file "%s" to "%s"...', (old, dest)) From 2639c5e9ad23004d5f1506d65d83884d093b9af4 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 10:40:29 +0200 Subject: [PATCH 136/202] Force add if not set fix #1811 --- couchpotato/core/plugins/renamer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index af39aa3..a2d993e 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -219,6 +219,12 @@ class Renamer(Plugin): nfo_name = self.conf('nfo_name') separator = self.conf('separator') + cd_keys = ['',''] + if not any(x in folder_name for x in cd_keys) and not any(x in file_name for x in cd_keys): + log.error('Missing `cd` or `cd_nr` in the renamer. This will cause multi-file releases of being renamed to the same file.' + 'Force adding it') + file_name = '%s %s' % ('', file_name) + # Tag release folder as failed_rename in case no groups were found. This prevents check_snatched from removing the release from the downloader. if not groups and self.statusInfoComplete(release_download): self.tagRelease(release_download = release_download, tag = 'failed_rename') From 7861416dc535324af22290a883d098a92d8a5e13 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 10:43:53 +0200 Subject: [PATCH 137/202] Don't write over files already renamed --- couchpotato/core/plugins/renamer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index a2d993e..7784053 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -10,7 +10,7 @@ from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, ss, sp from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ - getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder, getIdentifier + getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder, getIdentifier, randomString from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env @@ -586,6 +586,10 @@ class Renamer(Plugin): if rename_files[src]: dst = rename_files[src] + if dst in group['renamed_files']: + log.error('File "%s" already exists, adding random string at the end to prevent data loss', dst) + dst = '%s.random-%s' % (dst, randomString()) + # Create dir self.makeDir(os.path.dirname(dst)) From 0d166025d003f2d82e822481b2a59bbd5ef162dc Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 15:13:50 +0200 Subject: [PATCH 138/202] Safestring before base encode --- couchpotato/core/media/movie/providers/info/couchpotatoapi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/movie/providers/info/couchpotatoapi.py b/couchpotato/core/media/movie/providers/info/couchpotatoapi.py index 4c65bf8..51afbae 100644 --- a/couchpotato/core/media/movie/providers/info/couchpotatoapi.py +++ b/couchpotato/core/media/movie/providers/info/couchpotatoapi.py @@ -2,7 +2,7 @@ import base64 import time from couchpotato.core.event import addEvent, fireEvent -from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.helpers.encoding import tryUrlencode, ss from couchpotato.core.logger import CPLog from couchpotato.core.media.movie.providers.base import MovieProvider from couchpotato.environment import Env @@ -66,7 +66,7 @@ class CouchPotatoApi(MovieProvider): if not name: return - name_enc = base64.b64encode(name) + name_enc = base64.b64encode(ss(name)) return self.getJsonData(self.urls['validate'] % name_enc, headers = self.getRequestHeaders()) def isMovie(self, identifier = None): From 2deb6ee6a79368f07c9b0b96fe81656582ec14da Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 15:57:29 +0200 Subject: [PATCH 139/202] Trakt not moving movie to collection fix #3018 --- couchpotato/core/notifications/trakt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/notifications/trakt.py b/couchpotato/core/notifications/trakt.py index 8f35dea..fe170be 100644 --- a/couchpotato/core/notifications/trakt.py +++ b/couchpotato/core/notifications/trakt.py @@ -1,4 +1,4 @@ -from couchpotato.core.helpers.variable import getTitle +from couchpotato.core.helpers.variable import getTitle, getIdentifier from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification @@ -16,7 +16,7 @@ class Trakt(Notification): 'test': 'account/test/%s', } - listen_to = ['movie.downloaded'] + listen_to = ['movie.snatched'] def notify(self, message = '', data = None, listener = None): if not data: data = {} @@ -38,7 +38,7 @@ class Trakt(Notification): 'username': self.conf('automation_username'), 'password': self.conf('automation_password'), 'movies': [{ - 'imdb_id': data['identifier'], + 'imdb_id': getIdentifier(data), 'title': getTitle(data), 'year': data['info']['year'] }] if data else [] From 2fa7834e6ef0cab77abd829508862f284ec9fa07 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 22:17:09 +0200 Subject: [PATCH 140/202] Allow typing in directory setting closes #479 --- couchpotato/static/scripts/page/settings.js | 122 +++++++++++++++++++++++----- couchpotato/static/style/settings.css | 13 ++- 2 files changed, 112 insertions(+), 23 deletions(-) diff --git a/couchpotato/static/scripts/page/settings.js b/couchpotato/static/scripts/page/settings.js index b9f72ab..c8e0a5e 100644 --- a/couchpotato/static/scripts/page/settings.js +++ b/couchpotato/static/scripts/page/settings.js @@ -634,6 +634,7 @@ Option.Directory = new Class({ browser: null, save_on_change: false, use_cache: false, + current_dir: '', create: function(){ var self = this; @@ -645,8 +646,17 @@ Option.Directory = new Class({ 'click': self.showBrowser.bind(self) } }).adopt( - self.input = new Element('span', { - 'text': self.getSettingValue() + self.input = new Element('input', { + 'value': self.getSettingValue(), + 'events': { + 'change': self.filterDirectory.bind(self), + 'keydown': function(e){ + if(e.key == 'enter' || e.key == 'tab') + (e).stop(); + }, + 'keyup': self.filterDirectory.bind(self), + 'paste': self.filterDirectory.bind(self) + } }) ) ); @@ -654,10 +664,55 @@ Option.Directory = new Class({ self.cached = {}; }, + filterDirectory: function(e){ + var self = this, + value = self.getValue(), + path_sep = Api.getOption('path_sep'), + active_selector = 'li:not(.blur):not(.empty)'; + + if(e.key == 'enter' || e.key == 'tab'){ + (e).stop(); + + var first = self.dir_list.getElement(active_selector); + if(first){ + self.selectDirectory(first.get('data-value')); + } + } + else { + + // New folder + if(value.substr(-1) == path_sep){ + if(self.current_dir != value) + self.selectDirectory(value) + } + else { + var pd = self.getParentDir(value); + if(self.current_dir != pd) + self.getDirs(pd); + + var folder_filter = value.split(path_sep).getLast() + self.dir_list.getElements('li').each(function(li){ + var valid = li.get('text').substr(0, folder_filter.length).toLowerCase() != folder_filter.toLowerCase() + li[valid ? 'addClass' : 'removeClass']('blur') + }); + + var first = self.dir_list.getElement(active_selector); + if(first){ + if(!self.dir_list_scroll) + self.dir_list_scroll = new Fx.Scroll(self.dir_list, { + 'transition': 'quint:in:out' + }); + + self.dir_list_scroll.toElement(first); + } + } + } + }, + selectDirectory: function(dir){ var self = this; - self.input.set('text', dir); + self.input.set('value', dir); self.getDirs() }, @@ -668,9 +723,28 @@ Option.Directory = new Class({ self.selectDirectory(self.getParentDir()) }, + caretAtEnd: function(){ + var self = this; + + self.input.focus(); + + if (typeof self.input.selectionStart == "number") { + self.input.selectionStart = self.input.selectionEnd = self.input.get('value').length; + } else if (typeof el.createTextRange != "undefined") { + self.input.focus(); + var range = self.input.createTextRange(); + range.collapse(false); + range.select(); + } + }, + showBrowser: function(){ var self = this; + // Move caret to back of the input + if(!self.browser || self.browser && !self.browser.isVisible()) + self.caretAtEnd() + if(!self.browser){ self.browser = new Element('div.directory_list').adopt( new Element('div.pointer'), @@ -686,7 +760,9 @@ Option.Directory = new Class({ }).adopt( self.show_hidden = new Element('input[type=checkbox].inlay', { 'events': { - 'change': self.getDirs.bind(self) + 'change': function(){ + self.getDirs() + } } }) ) @@ -707,7 +783,7 @@ Option.Directory = new Class({ 'text': 'Clear', 'events': { 'click': function(e){ - self.input.set('text', ''); + self.input.set('value', ''); self.hideBrowser(e, true); } } @@ -735,7 +811,7 @@ Option.Directory = new Class({ new Form.Check(self.show_hidden); } - self.initial_directory = self.input.get('text'); + self.initial_directory = self.input.get('value'); self.getDirs(); self.browser.show(); @@ -749,7 +825,7 @@ Option.Directory = new Class({ if(save) self.save(); else - self.input.set('text', self.initial_directory); + self.input.set('value', self.initial_directory); self.browser.hide(); self.el.removeEvents('outerClick') @@ -757,21 +833,21 @@ Option.Directory = new Class({ }, fillBrowser: function(json){ - var self = this; + var self = this, + v = self.getValue(); self.data = json; - var v = self.getValue(); - var previous_dir = self.getParentDir(); + var previous_dir = json.parent; if(v == '') - self.input.set('text', json.home); + self.input.set('value', json.home); - if(previous_dir != v && previous_dir.length >= 1 && !json.is_root){ + if(previous_dir.length >= 1 && !json.is_root){ var prev_dirname = self.getCurrentDirname(previous_dir); if(previous_dir == json.home) - prev_dirname = 'Home'; + prev_dirname = 'Home Folder'; else if(previous_dir == '/' && json.platform == 'nt') prev_dirname = 'Computer'; @@ -801,12 +877,13 @@ Option.Directory = new Class({ new Element('li.empty', { 'text': 'Selected folder is empty' }).inject(self.dir_list) - }, - getDirs: function(){ - var self = this; + self.caretAtEnd(); + }, - var c = self.getValue(); + getDirs: function(dir){ + var self = this, + c = dir || self.getValue(); if(self.cached[c] && self.use_cache){ self.fillBrowser() @@ -817,7 +894,10 @@ Option.Directory = new Class({ 'path': c, 'show_hidden': +self.show_hidden.checked }, - 'onComplete': self.fillBrowser.bind(self) + 'onComplete': function(json){ + self.current_dir = c; + self.fillBrowser(json); + } }) } }, @@ -831,8 +911,8 @@ Option.Directory = new Class({ var v = dir || self.getValue(); var sep = Api.getOption('path_sep'); var dirs = v.split(sep); - if(dirs.pop() == '') - dirs.pop(); + if(dirs.pop() == '') + dirs.pop(); return dirs.join(sep) + sep }, @@ -845,7 +925,7 @@ Option.Directory = new Class({ getValue: function(){ var self = this; - return self.input.get('text'); + return self.input.get('value'); } }); diff --git a/couchpotato/static/style/settings.css b/couchpotato/static/style/settings.css index 7fb1df2..e84d381 100644 --- a/couchpotato/static/style/settings.css +++ b/couchpotato/static/style/settings.css @@ -302,15 +302,19 @@ font-family: 'Elusive-Icons'; color: #f5e39c; } - .page form .directory > span { + .page form .directory > input { height: 25px; display: inline-block; float: right; text-align: right; white-space: nowrap; cursor: pointer; + background: none; + border: 0; + color: #FFF; + width: 100%; } - .page form .directory span:empty:before { + .page form .directory input:empty:before { content: 'No folder selected'; font-style: italic; opacity: .3; @@ -353,6 +357,11 @@ white-space: nowrap; text-overflow: ellipsis; } + + .page .directory_list li.blur { + opacity: .3; + } + .page .directory_list li:last-child { border-bottom: 1px solid rgba(255,255,255,0.1); } From 0bae50931154172a6ae0915aec29cfcceace5d74 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 22:29:52 +0200 Subject: [PATCH 141/202] Use github img url for Growl notification fix #1363 --- couchpotato/core/notifications/growl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/notifications/growl.py b/couchpotato/core/notifications/growl.py index e60e7ef..a0081a2 100644 --- a/couchpotato/core/notifications/growl.py +++ b/couchpotato/core/notifications/growl.py @@ -34,9 +34,9 @@ class Growl(Notification): self.growl = notifier.GrowlNotifier( applicationName = Env.get('appname'), - notifications = ["Updates"], - defaultNotifications = ["Updates"], - applicationIcon = '%s/static/images/couch.png' % fireEvent('app.api_url', single = True), + notifications = ['Updates'], + defaultNotifications = ['Updates'], + applicationIcon = self.getNotificationImage('medium'), hostname = hostname if hostname else 'localhost', password = password if password else None, port = port if port else 23053 @@ -56,7 +56,7 @@ class Growl(Notification): try: self.growl.notify( - noteType = "Updates", + noteType = 'Updates', title = self.default_title, description = message, sticky = False, From da9d2b5ed8ca45de5a5e19fced5528abd8fc7bbf Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 22 Sep 2014 21:03:13 +0200 Subject: [PATCH 142/202] Check free diskspace before starting moving files fix #3893 --- couchpotato/core/helpers/variable.py | 22 ++++++++++++++++++++++ couchpotato/core/plugins/renamer.py | 11 ++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index db68da2..9e064b7 100755 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -382,6 +382,28 @@ def getFreeSpace(directories): return free_space +def getSize(paths): + + single = not isinstance(paths, (tuple, list)) + if single: + paths = [paths] + + total_size = 0 + for path in paths: + path = sp(path) + + if os.path.isdir(path): + total_size = 0 + for dirpath, _, filenames in os.walk(path): + for f in filenames: + total_size += os.path.getsize(sp(os.path.join(dirpath, f))) + + elif os.path.isfile(path): + total_size += os.path.getsize(path) + + return total_size / 1048576 # MB + + def find(func, iterable): for item in iterable: if func(item): diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 7784053..01e78f0 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -10,7 +10,8 @@ from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, ss, sp from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ - getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder, getIdentifier, randomString + getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder, \ + getIdentifier, randomString, getFreeSpace, getSize from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env @@ -279,6 +280,7 @@ class Renamer(Plugin): except: log.error('Failed getting category label: %s', traceback.format_exc()) + # Find subtitle for renaming group['before_rename'] = [] fireEvent('renamer.before', group) @@ -546,6 +548,13 @@ class Renamer(Plugin): (not keep_original or self.fileIsAdded(current_file, group)): remove_files.append(current_file) + total_space, available_space = getFreeSpace(destination) + renaming_size = getSize(rename_files.keys()) + if renaming_size > available_space: + log.error('Not enough space left, need %s MB but only %s MB available', (renaming_size, available_space)) + self.tagRelease(group = group, tag = 'not_enough_space') + continue + # Remove files delete_folders = [] for src in remove_files: From 11e7fb23cab01d909e658a29b5ecddb2eeb30c0c Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 22 Sep 2014 21:38:53 +0200 Subject: [PATCH 143/202] Stream larger file download fix #2488 --- couchpotato/core/plugins/base.py | 39 ++++++++++++++++++++++++++++----------- couchpotato/core/plugins/file.py | 5 ++++- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 39b17a8..29b3432 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -121,15 +121,31 @@ class Plugin(object): if os.path.exists(path): log.debug('%s already exists, overwriting file with new version', path) - try: - f = open(path, 'w+' if not binary else 'w+b') - f.write(content) - f.close() - os.chmod(path, Env.getPermission('file')) - except: - log.error('Unable writing to file "%s": %s', (path, traceback.format_exc())) - if os.path.isfile(path): - os.remove(path) + write_type = 'w+' if not binary else 'w+b' + + # Stream file using response object + if isinstance(content, requests.models.Response): + + # Write file to temp + with open('%s.tmp' % path, write_type) as f: + for chunk in content.iter_content(chunk_size = 1048576): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + f.flush() + + # Rename to destination + os.rename('%s.tmp' % path, path) + + else: + try: + f = open(path, write_type) + f.write(content) + f.close() + os.chmod(path, Env.getPermission('file')) + except: + log.error('Unable writing to file "%s": %s', (path, traceback.format_exc())) + if os.path.isfile(path): + os.remove(path) def makeDir(self, path): path = sp(path) @@ -165,7 +181,7 @@ class Plugin(object): log.error('Couldn\'t remove empty directory %s: %s', (folder, traceback.format_exc())) # http request - def urlopen(self, url, timeout = 30, data = None, headers = None, files = None, show_error = True): + def urlopen(self, url, timeout = 30, data = None, headers = None, files = None, show_error = True, stream = False): url = quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]") if not headers: headers = {} @@ -206,6 +222,7 @@ class Plugin(object): 'timeout': timeout, 'files': files, 'verify': False, #verify_ssl, Disable for now as to many wrongly implemented certificates.. + 'stream': stream } method = 'post' if len(data) > 0 or files else 'get' @@ -214,7 +231,7 @@ class Plugin(object): status_code = response.status_code if response.status_code == requests.codes.ok: - data = response.content + data = response if stream else response.content else: response.raise_for_status() diff --git a/couchpotato/core/plugins/file.py b/couchpotato/core/plugins/file.py index 80c073f..db6787d 100644 --- a/couchpotato/core/plugins/file.py +++ b/couchpotato/core/plugins/file.py @@ -64,6 +64,9 @@ class FileManager(Plugin): def download(self, url = '', dest = None, overwrite = False, urlopen_kwargs = None): if not urlopen_kwargs: urlopen_kwargs = {} + # Return response object to stream download + urlopen_kwargs['stream'] = True + if not dest: # to Cache dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) @@ -107,4 +110,4 @@ class FileManager(Plugin): else: log.info('Subfolder test succeeded') - return failed == 0 \ No newline at end of file + return failed == 0 From 6fa6d530ec1f45aeb73614b528433de84c31b462 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 22 Sep 2014 21:45:43 +0200 Subject: [PATCH 144/202] Remove older backup folders --- couchpotato/runner.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 1c6f577..bc46518 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -9,6 +9,7 @@ import traceback import warnings import re import tarfile +import shutil from CodernityDB.database_super_thread_safe import SuperThreadSafeDatabase from argparse import ArgumentParser @@ -108,14 +109,19 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En if not os.path.isdir(backup_path): os.makedirs(backup_path) for root, dirs, files in os.walk(backup_path): - for backup_file in sorted(files): - ints = re.findall('\d+', backup_file) - - # Delete non zip files - if len(ints) != 1: - os.remove(os.path.join(backup_path, backup_file)) - else: - existing_backups.append((int(ints[0]), backup_file)) + # Only consider files being a direct child of the backup_path + if root == backup_path: + for backup_file in sorted(files): + ints = re.findall('\d+', backup_file) + + # Delete non zip files + if len(ints) != 1: + os.remove(os.path.join(root, backup_file)) + else: + existing_backups.append((int(ints[0]), backup_file)) + else: + # Delete stray directories. + shutil.rmtree(root) # Remove all but the last 5 for eb in existing_backups[:-backup_count]: From b1f88c1c48266cae5f41dfd3908b1ff9faeb63a7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 22 Sep 2014 21:53:27 +0200 Subject: [PATCH 145/202] Allow https for Transmission close #3880 --- couchpotato/core/downloaders/transmission.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/downloaders/transmission.py b/couchpotato/core/downloaders/transmission.py index 409efaa..d6112a9 100644 --- a/couchpotato/core/downloaders/transmission.py +++ b/couchpotato/core/downloaders/transmission.py @@ -25,7 +25,7 @@ class Transmission(DownloaderBase): def connect(self): # Load host from config and split out port. - host = cleanHost(self.conf('host'), protocol = False).split(':') + host = cleanHost(self.conf('host')).rstrip('/').rsplit(':', 1) if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False @@ -162,11 +162,11 @@ class Transmission(DownloaderBase): class TransmissionRPC(object): """TransmissionRPC lite library""" - def __init__(self, host = 'localhost', port = 9091, rpc_url = 'transmission', username = None, password = None): + def __init__(self, host = 'http://localhost', port = 9091, rpc_url = 'transmission', username = None, password = None): super(TransmissionRPC, self).__init__() - self.url = 'http://' + host + ':' + str(port) + '/' + rpc_url + '/rpc' + self.url = host + ':' + str(port) + '/' + rpc_url + '/rpc' self.tag = 0 self.session_id = 0 self.session = {} @@ -274,8 +274,8 @@ config = [{ }, { 'name': 'host', - 'default': 'localhost:9091', - 'description': 'Hostname with port. Usually localhost:9091', + 'default': 'http://localhost:9091', + 'description': 'Hostname with port. Usually http://localhost:9091', }, { 'name': 'rpc_url', From d7f43c2cf8fb528d86929847a0761cad61ff37a6 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 22 Sep 2014 22:15:13 +0200 Subject: [PATCH 146/202] Make minimum seeders configurable fix #3202 --- couchpotato/core/media/_base/searcher/__init__.py | 20 ++++++++++++++++++++ couchpotato/core/plugins/release/main.py | 7 ++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/media/_base/searcher/__init__.py b/couchpotato/core/media/_base/searcher/__init__.py index bf69b95..0e3655e 100644 --- a/couchpotato/core/media/_base/searcher/__init__.py +++ b/couchpotato/core/media/_base/searcher/__init__.py @@ -73,4 +73,24 @@ config = [{ ], }, ], +}, { + 'name': 'torrent', + 'groups': [ + { + 'tab': 'searcher', + 'name': 'searcher', + 'wizard': True, + 'options': [ + { + 'name': 'minimum_seeders', + 'advanced': True, + 'label': 'Minimum seeders', + 'description': 'Ignore torrents with seeders below this number', + 'default': 1, + 'type': 'int', + 'unit': 'seeders' + }, + ], + }, + ], }] diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 0385e22..815cb40 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -8,7 +8,7 @@ from couchpotato import md5, get_db from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, addEvent from couchpotato.core.helpers.encoding import toUnicode, sp -from couchpotato.core.helpers.variable import getTitle +from couchpotato.core.helpers.variable import getTitle, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from .index import ReleaseIndex, ReleaseStatusIndex, ReleaseIDIndex, ReleaseDownloadIndex @@ -380,6 +380,7 @@ class Release(Plugin): wait_for = False let_through = False filtered_results = [] + minimum_seeders = tryInt(Env.setting('minimum_seeders', section = 'torrent', default = 1)) # Filter out ignored and other releases we don't want for rel in results: @@ -396,8 +397,8 @@ class Release(Plugin): log.info('Ignored, size "%sMB" to low: %s', (rel['size'], rel['name'])) continue - if 'seeders' in rel and rel.get('seeders') <= 0: - log.info('Ignored, no seeders: %s', (rel['name'])) + if 'seeders' in rel and rel.get('seeders') < minimum_seeders: + log.info('Ignored, not enough seeders, has %s needs %s: %s', (rel.get('seeders'), minimum_seeders, rel['name'])) continue # If a single release comes through the "wait for", let through all From 70ca31a265ed6d26fb96fca301081a9e67b18d35 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 00:29:48 +0200 Subject: [PATCH 147/202] Only allow single redirect for now fix #3931 --- couchpotato/core/plugins/base.py | 2 +- couchpotato/runner.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 29b3432..5054b32 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -222,7 +222,7 @@ class Plugin(object): 'timeout': timeout, 'files': files, 'verify': False, #verify_ssl, Disable for now as to many wrongly implemented certificates.. - 'stream': stream + 'stream': stream, } method = 'post' if len(data) > 0 or files else 'get' diff --git a/couchpotato/runner.py b/couchpotato/runner.py index bc46518..8718355 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -151,12 +151,15 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En if not os.path.exists(python_cache): os.mkdir(python_cache) + session = requests.Session() + session.max_redirects = 1 + # Register environment settings Env.set('app_dir', sp(base_path)) Env.set('data_dir', sp(data_dir)) Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log'))) Env.set('db', db) - Env.set('http_opener', requests.Session()) + Env.set('http_opener', session) Env.set('cache_dir', cache_dir) Env.set('cache', FileSystemCache(python_cache)) Env.set('console_log', options.console_log) From 3338b72d1f6845f6a44e6407d62d4ba27deee266 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 00:38:36 +0200 Subject: [PATCH 148/202] Stop endless redirect loop fix #3931 --- libs/requests/sessions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/requests/sessions.py b/libs/requests/sessions.py index 508b0ef..02c9fb2 100644 --- a/libs/requests/sessions.py +++ b/libs/requests/sessions.py @@ -532,7 +532,11 @@ class Session(SessionRedirectMixin): if not isinstance(request, PreparedRequest): raise ValueError('You can only send PreparedRequests.') + redirect_count = 0 while request.url in self.redirect_cache: + redirect_count += 1 + if redirect_count > self.max_redirects: + raise TooManyRedirects request.url = self.redirect_cache.get(request.url) # Set up variables needed for resolve_redirects and dispatching of hooks From 17b940a271318cd94107f77cc8f1181eae63280b Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 00:38:44 +0200 Subject: [PATCH 149/202] Allow 5 redirects --- couchpotato/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 8718355..36fc366 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -152,7 +152,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En os.mkdir(python_cache) session = requests.Session() - session.max_redirects = 1 + session.max_redirects = 5 # Register environment settings Env.set('app_dir', sp(base_path)) From b3d75cb48538743b73098cc14e12f735c5ed3e9c Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 10:02:36 +0200 Subject: [PATCH 150/202] Check if file got moved successful on move/copy close #3893 --- couchpotato/core/plugins/renamer.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 01e78f0..aeeb354 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -596,7 +596,7 @@ class Renamer(Plugin): dst = rename_files[src] if dst in group['renamed_files']: - log.error('File "%s" already exists, adding random string at the end to prevent data loss', dst) + log.error('File "%s" already renamed once, adding random string at the end to prevent data loss', dst) dst = '%s.random-%s' % (dst, randomString()) # Create dir @@ -797,6 +797,9 @@ Remove it if you want it to be renamed (again, or at least let it try again) dest = sp(dest) try: + if os.path.exists(dest): + raise Exception('Destination "%s" already exists' % dest) + move_type = self.conf('file_action') if use_default: move_type = self.conf('default_file_action') @@ -806,10 +809,14 @@ Remove it if you want it to be renamed (again, or at least let it try again) log.info('Moving "%s" to "%s"', (old, dest)) shutil.move(old, dest) except: - if os.path.exists(dest): + exists = os.path.exists(dest) + if exists and os.path.getsize(old) == os.path.getsize(dest): log.error('Successfully moved file "%s", but something went wrong: %s', (dest, traceback.format_exc())) os.unlink(old) else: + # remove faultly copied file + if exists: + os.unlink(dest) raise elif move_type == 'copy': log.info('Copying "%s" to "%s"', (old, dest)) @@ -1219,7 +1226,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) except Exception as e: log.error('Failed moving left over file %s to %s: %s %s', (leftoverfile, move_to, e, traceback.format_exc())) # As we probably tried to overwrite the nfo file, check if it exists and then remove the original - if os.path.isfile(move_to): + if os.path.isfile(move_to) and os.path.getsize(leftoverfile) == os.path.getsize(move_to): if cleanup: log.info('Deleting left over file %s instead...', leftoverfile) os.unlink(leftoverfile) From 39d0f91de251c75a97cee1f0f4591407175cc0f7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 12:27:09 +0200 Subject: [PATCH 151/202] Add permission calculator link #3953 --- couchpotato/core/_base/_core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/_base/_core.py b/couchpotato/core/_base/_core.py index 852c42c..6bfb441 100644 --- a/couchpotato/core/_base/_core.py +++ b/couchpotato/core/_base/_core.py @@ -286,13 +286,13 @@ config = [{ 'name': 'permission_folder', 'default': '0755', 'label': 'Folder CHMOD', - 'description': 'Can be either decimal (493) or octal (leading zero: 0755)', + 'description': 'Can be either decimal (493) or octal (leading zero: 0755). Calculate the correct value', }, { 'name': 'permission_file', 'default': '0755', 'label': 'File CHMOD', - 'description': 'Same as Folder CHMOD but for files', + 'description': 'See Folder CHMOD description, but for files', }, ], }, From 8f02b0eea0cd3a2c98650cbef2e7c643fef96ac3 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 12:36:46 +0200 Subject: [PATCH 152/202] Api documention updates close #3955 --- couchpotato/core/media/movie/_base/main.py | 4 ++++ couchpotato/core/plugins/category/main.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index c5ed462..1b0881b 100755 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -27,6 +27,10 @@ class MovieBase(MovieTypeBase): addApiView('movie.add', self.addView, docs = { 'desc': 'Add new movie to the wanted list', + 'return': {'type': 'object', 'example': """{ + 'success': True, + 'movie': object +}"""}, 'params': { 'identifier': {'desc': 'IMDB id of the movie your want to add.'}, 'profile_id': {'desc': 'ID of quality profile you want the add the movie in. If empty will use the default profile.'}, diff --git a/couchpotato/core/plugins/category/main.py b/couchpotato/core/plugins/category/main.py index a0852cc..4abc94c 100644 --- a/couchpotato/core/plugins/category/main.py +++ b/couchpotato/core/plugins/category/main.py @@ -27,7 +27,7 @@ class CategoryPlugin(Plugin): 'desc': 'List all available categories', 'return': {'type': 'object', 'example': """{ 'success': True, - 'list': array, categories + 'categories': array, categories }"""} }) From faefd7a5b5cd6cb760eacc9a5595ccab91bb64de Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 16:54:24 +0200 Subject: [PATCH 153/202] Traks notifier always enabled --- couchpotato/core/notifications/trakt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/couchpotato/core/notifications/trakt.py b/couchpotato/core/notifications/trakt.py index fe170be..91c6ae1 100644 --- a/couchpotato/core/notifications/trakt.py +++ b/couchpotato/core/notifications/trakt.py @@ -17,6 +17,7 @@ class Trakt(Notification): } listen_to = ['movie.snatched'] + enabled_option = 'notification_enabled' def notify(self, message = '', data = None, listener = None): if not data: data = {} From f6d4ddbe80601aee894a7fc0e195fc581ee2e4f6 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 17:40:21 +0200 Subject: [PATCH 154/202] NZBVortex, create unique ID --- couchpotato/core/downloaders/nzbvortex.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/couchpotato/core/downloaders/nzbvortex.py b/couchpotato/core/downloaders/nzbvortex.py index 9094055..af95628 100644 --- a/couchpotato/core/downloaders/nzbvortex.py +++ b/couchpotato/core/downloaders/nzbvortex.py @@ -1,4 +1,5 @@ from base64 import b64encode +import re from urllib2 import URLError from uuid import uuid4 import hashlib @@ -14,7 +15,7 @@ import urllib2 from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList from couchpotato.core.helpers.encoding import tryUrlencode, sp -from couchpotato.core.helpers.variable import cleanHost +from couchpotato.core.helpers.variable import cleanHost, randomString from couchpotato.core.logger import CPLog @@ -35,13 +36,17 @@ class NZBVortex(DownloaderBase): # Send the nzb try: + nzb_id = '%s-%s' % (self.cpTag(media), randomString()) nzb_filename = self.createFileName(data, filedata, media) - self.call('nzb/add', files = {'file': (nzb_filename, filedata)}) + nzb_filename = re.sub('(.cp\(tt[0-9{7}]+\))', '', nzb_filename) + nzb_filename = '%s%s.nzb' % (nzb_filename[0:-4], nzb_id) + response = self.call('nzb/add', files = {'file': (nzb_filename, filedata, 'application/octet-stream')}) - time.sleep(10) - raw_statuses = self.call('nzb') - nzb_id = [nzb['id'] for nzb in raw_statuses.get('nzbs', []) if os.path.basename(nzb['nzbFileName']) == nzb_filename][0] - return self.downloadReturnId(nzb_id) + if response and response.get('result', '').lower() == 'ok': + return self.downloadReturnId(nzb_id) + + log.error('Something went wrong sending the NZB file. Response: %s', response) + return False except: log.error('Something went wrong sending the NZB file: %s', traceback.format_exc()) return False @@ -114,7 +119,7 @@ class NZBVortex(DownloaderBase): log.error('Login failed, please check you api-key') return False - def call(self, call, parameters = None, repeat = False, auth = True, *args, **kwargs): + def call(self, call, parameters = None, is_repeat = False, auth = True, *args, **kwargs): # Login first if not parameters: parameters = {} @@ -130,16 +135,16 @@ class NZBVortex(DownloaderBase): url = cleanHost(self.conf('host'), ssl = self.conf('ssl')) + 'api/' + call try: - data = self.urlopen('%s?%s' % (url, params), *args, **kwargs) + data = self.getJsonData('%s?%s' % (url, params), *args, **kwargs) if data: - return json.loads(data) + return data except URLError as e: if hasattr(e, 'code') and e.code == 403: # Try login and do again - if not repeat: + if not is_repeat: self.login() - return self.call(call, parameters = parameters, repeat = True, **kwargs) + return self.call(call, parameters = parameters, is_repeat = True, **kwargs) log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) except: From 7cb214d8a2dcf31bef04f6c47bcf5c659c8e6c5e Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 20:33:26 +0200 Subject: [PATCH 155/202] Don't force send host with every request --- couchpotato/core/plugins/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 5054b32..3c655bc 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -192,7 +192,7 @@ class Plugin(object): host = '%s%s' % (parsed_url.hostname, (':' + str(parsed_url.port) if parsed_url.port else '')) headers['Referer'] = headers.get('Referer', '%s://%s' % (parsed_url.scheme, host)) - headers['Host'] = headers.get('Host', host) + headers['Host'] = headers.get('Host', None) headers['User-Agent'] = headers.get('User-Agent', self.user_agent) headers['Accept-encoding'] = headers.get('Accept-encoding', 'gzip') headers['Connection'] = headers.get('Connection', 'keep-alive') From b9dbadda0b1bdcceb3f25f2cf4f79d25f4d4268f Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 22:00:27 +0200 Subject: [PATCH 156/202] Add randomstring support to cptag --- couchpotato/core/plugins/base.py | 27 +++++++++++++++++---------- couchpotato/core/plugins/scanner.py | 4 ++-- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 3c655bc..2e99371 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -11,7 +11,8 @@ import traceback from couchpotato.core.event import fireEvent, addEvent from couchpotato.core.helpers.encoding import ss, toSafeString, \ toUnicode, sp -from couchpotato.core.helpers.variable import getExt, md5, isLocalIP, scanForPassword, tryInt, getIdentifier +from couchpotato.core.helpers.variable import getExt, md5, isLocalIP, scanForPassword, tryInt, getIdentifier, \ + randomString from couchpotato.core.logger import CPLog from couchpotato.environment import Env import requests @@ -346,9 +347,9 @@ class Plugin(object): Env.get('cache').set(cache_key_md5, value, timeout) return value - def createNzbName(self, data, media): + def createNzbName(self, data, media, unique_tag = False): release_name = data.get('name') - tag = self.cpTag(media) + tag = self.cpTag(media, unique_tag = unique_tag) # Check if password is filename name_password = scanForPassword(data.get('name')) @@ -361,18 +362,24 @@ class Plugin(object): max_length = 127 - len(tag) # Some filesystems don't support 128+ long filenames return '%s%s' % (toSafeString(toUnicode(release_name)[:max_length]), tag) - def createFileName(self, data, filedata, media): - name = self.createNzbName(data, media) + def createFileName(self, data, filedata, media, unique_tag = False): + name = self.createNzbName(data, media, unique_tag = unique_tag) if data.get('protocol') == 'nzb' and 'DOCTYPE nzb' not in filedata and '' not in filedata: return '%s.%s' % (name, 'rar') return '%s.%s' % (name, data.get('protocol')) - def cpTag(self, media): - if Env.setting('enabled', 'renamer'): - identifier = getIdentifier(media) - return '.cp(' + identifier + ')' if identifier else '' + def cpTag(self, media, unique_tag = False): - return '' + identifier = getIdentifier(media) or '' + unique_tag = ', ' + randomString() if unique_tag else '' + + tag = '.cp(' + tag += identifier + tag += ', ' if unique_tag and identifier else '' + tag += randomString() if unique_tag else '' + tag += ')' + + return tag if len(tag) > 7 else '' def checkFilesChanged(self, files, unchanged_for = 60): now = time.time() diff --git a/couchpotato/core/plugins/scanner.py b/couchpotato/core/plugins/scanner.py index a1b5cf8..a7a5e88 100644 --- a/couchpotato/core/plugins/scanner.py +++ b/couchpotato/core/plugins/scanner.py @@ -120,7 +120,7 @@ class Scanner(Plugin): '()([ab])(\.....?)$' #*a.mkv ] - cp_imdb = '(.cp.(?Ptt[0-9{7}]+).)' + cp_imdb = '\.cp\((?Ptt[0-9]+),?\s?(?P[A-Za-z0-9]+)?\)' def __init__(self): @@ -492,7 +492,7 @@ class Scanner(Plugin): data['quality_type'] = 'HD' if data.get('resolution_width', 0) >= 1280 or data['quality'].get('hd') else 'SD' - filename = re.sub('(.cp\(tt[0-9{7}]+\))', '', files[0]) + filename = re.sub(self.cp_imdb, '', files[0]) data['group'] = self.getGroup(filename[len(folder):]) data['source'] = self.getSourceMedia(filename) if data['quality'].get('is_3d', 0): From 543226450c40e33db30d0efcf2eb4d336fdc6409 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 22:22:27 +0200 Subject: [PATCH 157/202] NZBVortex status checking --- couchpotato/core/downloaders/nzbvortex.py | 84 +++++++++---------------------- 1 file changed, 23 insertions(+), 61 deletions(-) diff --git a/couchpotato/core/downloaders/nzbvortex.py b/couchpotato/core/downloaders/nzbvortex.py index af95628..aa2ac1e 100644 --- a/couchpotato/core/downloaders/nzbvortex.py +++ b/couchpotato/core/downloaders/nzbvortex.py @@ -1,21 +1,14 @@ from base64 import b64encode -import re -from urllib2 import URLError +import os from uuid import uuid4 import hashlib -import httplib -import json -import os -import socket -import ssl -import sys -import time import traceback -import urllib2 + +from requests import HTTPError from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList from couchpotato.core.helpers.encoding import tryUrlencode, sp -from couchpotato.core.helpers.variable import cleanHost, randomString +from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog @@ -36,14 +29,11 @@ class NZBVortex(DownloaderBase): # Send the nzb try: - nzb_id = '%s-%s' % (self.cpTag(media), randomString()) - nzb_filename = self.createFileName(data, filedata, media) - nzb_filename = re.sub('(.cp\(tt[0-9{7}]+\))', '', nzb_filename) - nzb_filename = '%s%s.nzb' % (nzb_filename[0:-4], nzb_id) + nzb_filename = self.createFileName(data, filedata, media, unique_tag = True) response = self.call('nzb/add', files = {'file': (nzb_filename, filedata, 'application/octet-stream')}) if response and response.get('result', '').lower() == 'ok': - return self.downloadReturnId(nzb_id) + return self.downloadReturnId(nzb_filename) log.error('Something went wrong sending the NZB file. Response: %s', response) return False @@ -65,7 +55,8 @@ class NZBVortex(DownloaderBase): release_downloads = ReleaseDownloadList(self) for nzb in raw_statuses.get('nzbs', []): - if nzb['id'] in ids: + nzb_id = os.path.basename(nzb['nzbFileName']) + if nzb_id in ids: # Check status status = 'busy' @@ -75,7 +66,8 @@ class NZBVortex(DownloaderBase): status = 'failed' release_downloads.append({ - 'id': nzb['id'], + 'temp_id': nzb['id'], + 'id': nzb_id, 'name': nzb['uiTitle'], 'status': status, 'original_status': nzb['state'], @@ -90,7 +82,7 @@ class NZBVortex(DownloaderBase): log.info('%s failed downloading, deleting...', release_download['name']) try: - self.call('nzb/%s/cancel' % release_download['id']) + self.call('nzb/%s/cancel' % release_download['temp_id']) except: log.error('Failed deleting: %s', traceback.format_exc(0)) return False @@ -132,15 +124,16 @@ class NZBVortex(DownloaderBase): params = tryUrlencode(parameters) - url = cleanHost(self.conf('host'), ssl = self.conf('ssl')) + 'api/' + call + url = cleanHost(self.conf('host')) + 'api/' + call try: - data = self.getJsonData('%s?%s' % (url, params), *args, **kwargs) + data = self.getJsonData('%s%s' % (url, '?' + params if params else ''), *args, cache_timeout = 0, show_error = False, **kwargs) if data: return data - except URLError as e: - if hasattr(e, 'code') and e.code == 403: + except HTTPError as e: + sc = e.response.status_code + if sc == 403: # Try login and do again if not is_repeat: self.login() @@ -156,13 +149,12 @@ class NZBVortex(DownloaderBase): if not self.api_level: - url = cleanHost(self.conf('host')) + 'api/app/apilevel' - try: - data = self.urlopen(url, show_error = False) - self.api_level = float(json.loads(data).get('apilevel')) - except URLError as e: - if hasattr(e, 'code') and e.code == 403: + data = self.call('app/apilevel', auth = False) + self.api_level = float(data.get('apilevel')) + except HTTPError as e: + sc = e.response.status_code + if sc == 403: log.error('This version of NZBVortex isn\'t supported. Please update to 2.8.6 or higher') else: log.error('NZBVortex doesn\'t seem to be running or maybe the remote option isn\'t enabled yet: %s', traceback.format_exc(1)) @@ -174,29 +166,6 @@ class NZBVortex(DownloaderBase): return super(NZBVortex, self).isEnabled(manual, data) and self.getApiLevel() -class HTTPSConnection(httplib.HTTPSConnection): - def __init__(self, *args, **kwargs): - httplib.HTTPSConnection.__init__(self, *args, **kwargs) - - def connect(self): - sock = socket.create_connection((self.host, self.port), self.timeout) - if sys.version_info < (2, 6, 7): - if hasattr(self, '_tunnel_host'): - self.sock = sock - self._tunnel() - else: - if self._tunnel_host: - self.sock = sock - self._tunnel() - - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version = ssl.PROTOCOL_TLSv1) - - -class HTTPSHandler(urllib2.HTTPSHandler): - def https_open(self, req): - return self.do_open(HTTPSConnection, req) - - config = [{ 'name': 'nzbvortex', 'groups': [ @@ -216,15 +185,8 @@ config = [{ }, { 'name': 'host', - 'default': 'localhost:4321', - 'description': 'Hostname with port. Usually localhost:4321', - }, - { - 'name': 'ssl', - 'default': 1, - 'type': 'bool', - 'advanced': True, - 'description': 'Use HyperText Transfer Protocol Secure, or https', + 'default': 'https://localhost:4321', + 'description': 'Hostname with port. Usually https://localhost:4321', }, { 'name': 'api_key', From 4b66b0ea0761707e8f8e1a36d44fa1649230c9a9 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 22:36:01 +0200 Subject: [PATCH 158/202] Add NZBVortex group support fix #1279 --- couchpotato/core/downloaders/nzbvortex.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/nzbvortex.py b/couchpotato/core/downloaders/nzbvortex.py index aa2ac1e..4f28ed4 100644 --- a/couchpotato/core/downloaders/nzbvortex.py +++ b/couchpotato/core/downloaders/nzbvortex.py @@ -30,7 +30,10 @@ class NZBVortex(DownloaderBase): # Send the nzb try: nzb_filename = self.createFileName(data, filedata, media, unique_tag = True) - response = self.call('nzb/add', files = {'file': (nzb_filename, filedata, 'application/octet-stream')}) + response = self.call('nzb/add', files = {'file': (nzb_filename, filedata, 'application/octet-stream')}, parameters = { + 'name': nzb_filename, + 'groupname': self.conf('group') + }) if response and response.get('result', '').lower() == 'ok': return self.downloadReturnId(nzb_filename) @@ -193,6 +196,11 @@ config = [{ 'label': 'Api Key', }, { + 'name': 'group', + 'label': 'Group', + 'description': 'The group CP places the nzb in. Make sure to create it in NZBVortex.', + }, + { 'name': 'manual', 'default': False, 'type': 'bool', From 910393d00ec083b250672bcff2f530ab6053a315 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 26 Sep 2014 15:33:38 +0200 Subject: [PATCH 159/202] Allow original without cd name --- couchpotato/core/plugins/renamer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index aeeb354..a4abeb4 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -220,7 +220,7 @@ class Renamer(Plugin): nfo_name = self.conf('nfo_name') separator = self.conf('separator') - cd_keys = ['',''] + cd_keys = ['','', ''] if not any(x in folder_name for x in cd_keys) and not any(x in file_name for x in cd_keys): log.error('Missing `cd` or `cd_nr` in the renamer. This will cause multi-file releases of being renamed to the same file.' 'Force adding it') From 5bea9dd04f2f576f633a2381688b58fae23669de Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 29 Sep 2014 16:24:57 +0200 Subject: [PATCH 160/202] Always return safestring on renamer replace --- couchpotato/core/plugins/renamer.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index a4abeb4..f6e4248 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -377,9 +377,6 @@ class Renamer(Plugin): if separator: final_file_name = final_file_name.replace(' ', separator) - final_folder_name = ss(final_folder_name) - final_file_name = ss(final_file_name) - # Move DVD files (no structure renaming) if group['is_dvd'] and file_type is 'movie': found = False @@ -878,7 +875,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) replaced = re.sub(r"[\x00:\*\?\"<>\|]", '', replaced) sep = self.conf('foldersep') if folder else self.conf('separator') - return replaced.replace(' ', ' ' if not sep else sep) + return ss(replaced.replace(' ', ' ' if not sep else sep)) def replaceDoubles(self, string): From 158f638fb9559116dfdbdad27cb6f33bfc5a3042 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 4 Oct 2014 14:07:38 +0200 Subject: [PATCH 161/202] No need for double replace --- couchpotato/core/plugins/renamer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index f6e4248..cd6255c 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -373,10 +373,6 @@ class Renamer(Plugin): elif file_type is 'nfo': final_file_name = self.doReplace(nfo_name, replacements, remove_multiple = True) - # Seperator replace - if separator: - final_file_name = final_file_name.replace(' ', separator) - # Move DVD files (no structure renaming) if group['is_dvd'] and file_type is 'movie': found = False From 78ba855c68995b0992a1742dc3a12a8c52bd5689 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 4 Oct 2014 14:33:51 +0200 Subject: [PATCH 162/202] Add CP tag only when renamer or unique tag is enabled. --- couchpotato/core/plugins/base.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 2e99371..704aa62 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -370,14 +370,16 @@ class Plugin(object): def cpTag(self, media, unique_tag = False): - identifier = getIdentifier(media) or '' - unique_tag = ', ' + randomString() if unique_tag else '' - - tag = '.cp(' - tag += identifier - tag += ', ' if unique_tag and identifier else '' - tag += randomString() if unique_tag else '' - tag += ')' + tag = '' + if Env.setting('enabled', 'renamer') or unique_tag: + identifier = getIdentifier(media) or '' + unique_tag = ', ' + randomString() if unique_tag else '' + + tag = '.cp(' + tag += identifier + tag += ', ' if unique_tag and identifier else '' + tag += randomString() if unique_tag else '' + tag += ')' return tag if len(tag) > 7 else '' From e595722139fc71f94fecef07eacc986094e83f0b Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 4 Oct 2014 15:37:18 +0200 Subject: [PATCH 163/202] Safari hanging on password input creation Fix #3997 --- couchpotato/static/scripts/page/settings.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/couchpotato/static/scripts/page/settings.js b/couchpotato/static/scripts/page/settings.js index c8e0a5e..b5aae3d 100644 --- a/couchpotato/static/scripts/page/settings.js +++ b/couchpotato/static/scripts/page/settings.js @@ -560,11 +560,19 @@ Option.Password = new Class({ create: function(){ var self = this; - self.parent(); - self.input.set('type', 'password'); + self.el.adopt( + self.createLabel(), + self.input = new Element('input.inlay', { + 'type': 'text', + 'name': self.postName(), + 'value': self.getSettingValue() ? '********' : '', + 'placeholder': self.getPlaceholder() + }) + ); self.input.addEvent('focus', function(){ - self.input.set('value', '') + self.input.set('value', ''); + self.input.set('type', 'password'); }) } From b5a0418a368c0df888b0373b79da166de987b1a3 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 4 Oct 2014 16:26:33 +0200 Subject: [PATCH 164/202] Make available space check optional fix #3973 --- couchpotato/core/plugins/renamer.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index cd6255c..d6deba6 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -541,12 +541,13 @@ class Renamer(Plugin): (not keep_original or self.fileIsAdded(current_file, group)): remove_files.append(current_file) - total_space, available_space = getFreeSpace(destination) - renaming_size = getSize(rename_files.keys()) - if renaming_size > available_space: - log.error('Not enough space left, need %s MB but only %s MB available', (renaming_size, available_space)) - self.tagRelease(group = group, tag = 'not_enough_space') - continue + if self.conf('check_space'): + total_space, available_space = getFreeSpace(destination) + renaming_size = getSize(rename_files.keys()) + if renaming_size > available_space: + log.error('Not enough space left, need %s MB but only %s MB available', (renaming_size, available_space)) + self.tagRelease(group = group, tag = 'not_enough_space') + continue # Remove files delete_folders = [] @@ -1390,6 +1391,14 @@ config = [{ 'description': ('Replace all the spaces with a character.', 'Example: ".", "-" (without quotes). Leave empty to use spaces.'), }, { + 'name': 'check_space', + 'label': 'Check space', + 'default': True, + 'type': 'bool', + 'description': ('Check if there\'s enough available space to rename the files', 'Disable when the filesystem doesn\'t return the proper value'), + 'advanced': True, + }, + { 'name': 'default_file_action', 'label': 'Default File Action', 'default': 'move', From 2270b2a28bbaddb5b52cadbcf2b482ac425eff8c Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 4 Oct 2014 16:50:01 +0200 Subject: [PATCH 165/202] Don't force parser for trailer searching --- couchpotato/core/media/movie/providers/trailer/hdtrailers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/media/movie/providers/trailer/hdtrailers.py b/couchpotato/core/media/movie/providers/trailer/hdtrailers.py index 828f017..4cbb64d 100644 --- a/couchpotato/core/media/movie/providers/trailer/hdtrailers.py +++ b/couchpotato/core/media/movie/providers/trailer/hdtrailers.py @@ -3,7 +3,7 @@ import re from bs4 import SoupStrainer, BeautifulSoup from couchpotato.core.helpers.encoding import tryUrlencode -from couchpotato.core.helpers.variable import mergeDicts, getTitle +from couchpotato.core.helpers.variable import mergeDicts, getTitle, getIdentifier from couchpotato.core.logger import CPLog from couchpotato.core.media.movie.providers.trailer.base import TrailerProvider from requests import HTTPError @@ -29,7 +29,7 @@ class HDTrailers(TrailerProvider): url = self.urls['api'] % self.movieUrlName(movie_name) try: - data = self.getCache('hdtrailers.%s' % group['identifier'], url, show_error = False) + data = self.getCache('hdtrailers.%s' % getIdentifier(group), url, show_error = False) except HTTPError: log.debug('No page found for: %s', movie_name) data = None @@ -59,7 +59,7 @@ class HDTrailers(TrailerProvider): url = "%s?%s" % (self.urls['backup'], tryUrlencode({'s':movie_name})) try: - data = self.getCache('hdtrailers.alt.%s' % group['identifier'], url, show_error = False) + data = self.getCache('hdtrailers.alt.%s' % getIdentifier(group), url, show_error = False) except HTTPError: log.debug('No alternative page found for: %s', movie_name) data = None @@ -68,7 +68,7 @@ class HDTrailers(TrailerProvider): return results try: - html = BeautifulSoup(data, 'html.parser', parse_only = self.only_tables_tags) + html = BeautifulSoup(data, parse_only = self.only_tables_tags) result_table = html.find_all('h2', text = re.compile(movie_name)) for h2 in result_table: @@ -90,7 +90,7 @@ class HDTrailers(TrailerProvider): results = {'480p':[], '720p':[], '1080p':[]} try: - html = BeautifulSoup(data, 'html.parser', parse_only = self.only_tables_tags) + html = BeautifulSoup(data, parse_only = self.only_tables_tags) result_table = html.find('table', attrs = {'class':'bottomTable'}) for tr in result_table.find_all('tr'): From 492f69b14914fd042e716a20c3d830633d85c64f Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 4 Oct 2014 16:53:40 +0200 Subject: [PATCH 166/202] Actually use the smtp port from settings fix #4003 --- couchpotato/core/notifications/email_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/email_.py b/couchpotato/core/notifications/email_.py index a63eb3d..d0c6516 100644 --- a/couchpotato/core/notifications/email_.py +++ b/couchpotato/core/notifications/email_.py @@ -42,7 +42,7 @@ class Email(Notification): # Open the SMTP connection, via SSL if requested log.debug("Connecting to host %s on port %s" % (smtp_server, smtp_port)) log.debug("SMTP over SSL %s", ("enabled" if ssl == 1 else "disabled")) - mailserver = smtplib.SMTP_SSL(smtp_server) if ssl == 1 else smtplib.SMTP(smtp_server) + mailserver = smtplib.SMTP_SSL(smtp_server, smtp_port) if ssl == 1 else smtplib.SMTP(smtp_server, smtp_port) if starttls: log.debug("Using StartTLS to initiate the connection with the SMTP server") From 01f70051f8465da6e1601b860b937b5af1408c18 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 4 Oct 2014 17:37:25 +0200 Subject: [PATCH 167/202] For people who can't read good --- couchpotato/core/plugins/log/static/log.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/log/static/log.js b/couchpotato/core/plugins/log/static/log.js index 6cb683b..71a65d0 100644 --- a/couchpotato/core/plugins/log/static/log.js +++ b/couchpotato/core/plugins/log/static/log.js @@ -241,7 +241,7 @@ Running on: ...\n\ 'href': 'https://github.com/RuudBurger/CouchPotatoServer/blob/develop/contributing.md' }), new Element('span', { - 'text': ' before posting (kittens die if you don\'t), then copy the text below.' + 'html': ' before posting, then copy the text below and FILL IN the dots.' }) ), textarea = new Element('textarea', { From e84f2aa04c0f8cafdf1760f4a1264d2767f8b0bb Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 4 Oct 2014 17:47:21 +0200 Subject: [PATCH 168/202] Don't load charts if suggestion tab is enabled --- .../core/media/movie/charts/static/charts.js | 24 ++++++++++++++++------ couchpotato/static/scripts/page/home.js | 2 +- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/couchpotato/core/media/movie/charts/static/charts.js b/couchpotato/core/media/movie/charts/static/charts.js index a04e248..e41b7ba 100644 --- a/couchpotato/core/media/movie/charts/static/charts.js +++ b/couchpotato/core/media/movie/charts/static/charts.js @@ -2,6 +2,8 @@ var Charts = new Class({ Implements: [Options, Events], + shown_once: false, + initialize: function(options){ var self = this; self.setOptions(options); @@ -40,17 +42,27 @@ var Charts = new Class({ ) ); - if( Cookie.read('suggestions_charts_menu_selected') === 'charts') - self.el.show(); + if( Cookie.read('suggestions_charts_menu_selected') === 'charts'){ + self.show(); + self.fireEvent.delay(0, self, 'created'); + } else self.el.hide(); - self.api_request = Api.request('charts.view', { - 'onComplete': self.fill.bind(self) - }); + }, + + show: function(){ + var self = this; + + self.el.show(); - self.fireEvent.delay(0, self, 'created'); + if(!self.shown_once){ + self.api_request = Api.request('charts.view', { + 'onComplete': self.fill.bind(self) + }); + self.shown_once = true; + } }, fill: function(json){ diff --git a/couchpotato/static/scripts/page/home.js b/couchpotato/static/scripts/page/home.js index 792b4a0..fc1702e 100644 --- a/couchpotato/static/scripts/page/home.js +++ b/couchpotato/static/scripts/page/home.js @@ -218,7 +218,7 @@ Page.Home = new Class({ self.el_toggle_menu_charts.removeClass('active'); break; case 'charts': - if($(self.charts)) $(self.charts).show(); + if($(self.charts)) self.charts.show(); self.el_toggle_menu_charts.addClass('active'); if($(self.suggestion_list)) $(self.suggestion_list).hide(); self.el_toggle_menu_suggestions.removeClass('active'); From 20f10760375d08ac0d61507b7ef8ab589a4517b7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 4 Oct 2014 21:48:46 +0200 Subject: [PATCH 169/202] Don't load suggestions when chartview active --- .../core/media/movie/charts/static/charts.js | 32 +++++---- .../core/media/movie/suggestion/static/suggest.js | 31 ++++++-- couchpotato/static/scripts/page/home.js | 84 ++++++++++------------ 3 files changed, 80 insertions(+), 67 deletions(-) diff --git a/couchpotato/core/media/movie/charts/static/charts.js b/couchpotato/core/media/movie/charts/static/charts.js index e41b7ba..3d70f7f 100644 --- a/couchpotato/core/media/movie/charts/static/charts.js +++ b/couchpotato/core/media/movie/charts/static/charts.js @@ -51,20 +51,6 @@ var Charts = new Class({ }, - show: function(){ - var self = this; - - self.el.show(); - - if(!self.shown_once){ - self.api_request = Api.request('charts.view', { - 'onComplete': self.fill.bind(self) - }); - - self.shown_once = true; - } - }, - fill: function(json){ var self = this; @@ -169,6 +155,24 @@ var Charts = new Class({ }, + show: function(){ + var self = this; + + self.el.show(); + + if(!self.shown_once){ + self.api_request = Api.request('charts.view', { + 'onComplete': self.fill.bind(self) + }); + + self.shown_once = true; + } + }, + + hide: function(){ + this.el.hide(); + }, + afterAdded: function(m){ $(m).getElement('div.chart_number') diff --git a/couchpotato/core/media/movie/suggestion/static/suggest.js b/couchpotato/core/media/movie/suggestion/static/suggest.js index 494f045..ca4b07c 100644 --- a/couchpotato/core/media/movie/suggestion/static/suggest.js +++ b/couchpotato/core/media/movie/suggestion/static/suggest.js @@ -2,6 +2,8 @@ var SuggestList = new Class({ Implements: [Options, Events], + shown_once: false, + initialize: function(options){ var self = this; self.setOptions(options); @@ -44,12 +46,13 @@ var SuggestList = new Class({ } }); - var cookie_menu_select = Cookie.read('suggestions_charts_menu_selected'); - if( cookie_menu_select === 'suggestions' || cookie_menu_select === null ) self.el.show(); else self.el.hide(); + var cookie_menu_select = Cookie.read('suggestions_charts_menu_selected') || 'suggestions'; + if( cookie_menu_select === 'suggestions') + self.show(); + else + self.hide(); - self.api_request = Api.request('suggestion.view', { - 'onComplete': self.fill.bind(self) - }); + self.fireEvent('created'); }, @@ -145,6 +148,24 @@ var SuggestList = new Class({ }, + show: function(){ + var self = this; + + self.el.show(); + + if(!self.shown_once){ + self.api_request = Api.request('suggestion.view', { + 'onComplete': self.fill.bind(self) + }); + + self.shown_once = true; + } + }, + + hide: function(){ + this.el.hide(); + }, + toElement: function(){ return this.el; } diff --git a/couchpotato/static/scripts/page/home.js b/couchpotato/static/scripts/page/home.js index fc1702e..4d18cac 100644 --- a/couchpotato/static/scripts/page/home.js +++ b/couchpotato/static/scripts/page/home.js @@ -146,13 +146,13 @@ Page.Home = new Class({ var self = this; // Suggest - self.suggestion_list = new SuggestList({ - 'onLoaded': function(){ + self.suggestions_list = new SuggestList({ + 'onCreated': function(){ self.chain.callChain(); } }); - $(self.suggestion_list).inject(self.el); + $(self.suggestions_list).inject(self.el); }, @@ -160,46 +160,38 @@ Page.Home = new Class({ var self = this; // Charts - self.charts = new Charts({ + self.charts_list = new Charts({ 'onCreated': function(){ self.chain.callChain(); } }); - $(self.charts).inject(self.el); + $(self.charts_list).inject(self.el); }, createSuggestionsChartsMenu: function(){ - var self = this; + var self = this, + suggestion_tab, charts_tab; + + self.el_toggle_menu = new Element('div.toggle_menu', { + 'events': { + 'click:relay(a)': function(e, el) { + e.preventDefault(); + self.toggleSuggestionsCharts(el.get('data-container'), el); + } + } + }).adopt( + suggestion_tab = new Element('a.toggle_suggestions', { + 'data-container': 'suggestions' + }).grab(new Element('h2', {'text': 'Suggestions'})), + charts_tab = new Element('a.toggle_charts', { + 'data-container': 'charts' + }).grab( new Element('h2', {'text': 'Charts'})) + ); - self.el_toggle_menu_suggestions = new Element('a.toggle_suggestions.active', { - 'href': '#', - 'events': { 'click': function(e) { - e.preventDefault(); - self.toggleSuggestionsCharts('suggestions'); - } - } - }).grab( new Element('h2', {'text': 'Suggestions'})); - - self.el_toggle_menu_charts = new Element('a.toggle_charts', { - 'href': '#', - 'events': { 'click': function(e) { - e.preventDefault(); - self.toggleSuggestionsCharts('charts'); - } - } - }).grab( new Element('h2', {'text': 'Charts'})); - - self.el_toggle_menu = new Element('div.toggle_menu').grab( - self.el_toggle_menu_suggestions - ).grab( - self.el_toggle_menu_charts - ); - - var menu_selected = Cookie.read('suggestions_charts_menu_selected'); - if( menu_selected === null ) menu_selected = 'suggestions'; - self.toggleSuggestionsCharts( menu_selected ); + var menu_selected = Cookie.read('suggestions_charts_menu_selected') || 'suggestions'; + self.toggleSuggestionsCharts(menu_selected, menu_selected == 'suggestions' ? suggestion_tab : charts_tab); self.el_toggle_menu.inject(self.el); @@ -207,23 +199,19 @@ Page.Home = new Class({ }, - toggleSuggestionsCharts: function(menu_id){ + toggleSuggestionsCharts: function(menu_id, el){ var self = this; - switch(menu_id) { - case 'suggestions': - if($(self.suggestion_list)) $(self.suggestion_list).show(); - self.el_toggle_menu_suggestions.addClass('active'); - if($(self.charts)) $(self.charts).hide(); - self.el_toggle_menu_charts.removeClass('active'); - break; - case 'charts': - if($(self.charts)) self.charts.show(); - self.el_toggle_menu_charts.addClass('active'); - if($(self.suggestion_list)) $(self.suggestion_list).hide(); - self.el_toggle_menu_suggestions.removeClass('active'); - break; - } + // Toggle ta + self.el_toggle_menu.getElements('.active').removeClass('active'); + if(el) el.addClass('active'); + + // Hide both + if(self.suggestions_list) self.suggestions_list.hide(); + if(self.charts_list) self.charts_list.hide(); + + var toggle_to = self[menu_id + '_list']; + if(toggle_to) toggle_to.show(); Cookie.write('suggestions_charts_menu_selected', menu_id, {'duration': 365}); }, From f8674f9baa2448dd17701c26d6a908b754f51b7e Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 4 Oct 2014 22:17:58 +0200 Subject: [PATCH 170/202] Grammar --- couchpotato/core/media/movie/searcher.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index 50b2d44..2dad737 100755 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -55,6 +55,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): if self.conf('run_on_launch'): addEvent('app.load', self.searchAll) + addEvent('app.load', self.searchAll) def searchAllView(self, **kwargs): @@ -141,17 +142,17 @@ class MovieSearcher(SearcherBase, MovieTypeBase): previous_releases = movie.get('releases', []) too_early_to_search = [] outside_eta_results = 0 - alway_search = self.conf('always_search') + always_search = self.conf('always_search') ignore_eta = manual total_result_count = 0 fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'_id': movie['_id']}, message = 'Searching for "%s"' % default_title) # Ignore eta once every 7 days - if not alway_search: + if not always_search: prop_name = 'last_ignored_eta.%s' % movie['_id'] last_ignored_eta = float(Env.prop(prop_name, default = 0)) - if last_ignored_eta > time.time() - 604800: + if last_ignored_eta < time.time() - 604800: ignore_eta = True Env.prop(prop_name, value = time.time()) @@ -170,7 +171,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): } could_not_be_released = not self.couldBeReleased(q_identifier in pre_releases, release_dates, movie['info']['year']) - if not alway_search and could_not_be_released: + if not always_search and could_not_be_released: too_early_to_search.append(q_identifier) # Skip release, if ETA isn't ignored @@ -196,7 +197,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): break quality = fireEvent('quality.single', identifier = q_identifier, single = True) - log.info('Search for %s in %s%s', (default_title, quality['label'], ' ignoring ETA' if alway_search or ignore_eta else '')) + log.info('Search for %s in %s%s', (default_title, quality['label'], ' ignoring ETA' if always_search or ignore_eta else '')) # Extend quality with profile customs quality['custom'] = quality_custom @@ -223,7 +224,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): log.debug('Found %s releases for "%s", but ETA isn\'t correct yet.', (results_count, default_title)) # Try find a valid result and download it - if (force_download or not could_not_be_released or alway_search) and fireEvent('release.try_download_result', results, movie, quality_custom, single = True): + if (force_download or not could_not_be_released or always_search) and fireEvent('release.try_download_result', results, movie, quality_custom, single = True): ret = True # Remove releases that aren't found anymore From d9bb1bfbfba899c55452ca1944ab2ef82dda82d2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 4 Oct 2014 22:57:18 +0200 Subject: [PATCH 171/202] - Debug code --- couchpotato/core/media/movie/searcher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index 2dad737..e44e8e6 100755 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -55,7 +55,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase): if self.conf('run_on_launch'): addEvent('app.load', self.searchAll) - addEvent('app.load', self.searchAll) def searchAllView(self, **kwargs): From 58878d8a0f3e3341cfc31cc722c14ec29666eecd Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 5 Oct 2014 00:18:22 +0200 Subject: [PATCH 172/202] Replace non existing chars --- couchpotato/core/helpers/encoding.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/helpers/encoding.py b/couchpotato/core/helpers/encoding.py index 4159168..c65fe87 100644 --- a/couchpotato/core/helpers/encoding.py +++ b/couchpotato/core/helpers/encoding.py @@ -5,6 +5,7 @@ import re import traceback import unicodedata +from chardet import detect from couchpotato.core.logger import CPLog import six @@ -35,6 +36,9 @@ def toUnicode(original, *args): return six.text_type(original, *args) except: try: + detected = detect(original) + if detected.get('encoding') == 'utf-8': + return original.decode('utf-8') return ek(original, *args) except: raise @@ -52,7 +56,10 @@ def ss(original, *args): return u_original.encode(Env.get('encoding')) except Exception as e: log.debug('Failed ss encoding char, force UTF8: %s', e) - return u_original.encode('UTF-8') + try: + return u_original.encode(Env.get('encoding'), 'replace') + except: + return u_original.encode('utf-8', 'replace') def sp(path, *args): From 230b7f47cc7cd8dece5b036b6a024cc2a7b8b886 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 5 Oct 2014 00:41:20 +0200 Subject: [PATCH 173/202] Always save encode logger stuff --- couchpotato/core/logger.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/couchpotato/core/logger.py b/couchpotato/core/logger.py index fba8d62..32320e4 100644 --- a/couchpotato/core/logger.py +++ b/couchpotato/core/logger.py @@ -59,15 +59,12 @@ class CPLog(object): msg = ss(msg) try: - msg = msg % replace_tuple - except: - try: - if isinstance(replace_tuple, tuple): - msg = msg % tuple([ss(x) for x in list(replace_tuple)]) - else: - msg = msg % ss(replace_tuple) - except Exception as e: - self.logger.error('Failed encoding stuff to log "%s": %s' % (msg, e)) + if isinstance(replace_tuple, tuple): + msg = msg % tuple([ss(x) for x in list(replace_tuple)]) + else: + msg = msg % ss(replace_tuple) + except Exception as e: + self.logger.error('Failed encoding stuff to log "%s": %s' % (msg, e)) self.setup() if not self.is_develop: From 9de8ed2dee8c7eee995e7fcff288052ec60749e0 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 5 Oct 2014 11:05:56 +0200 Subject: [PATCH 174/202] Don't migrate old databases --- couchpotato/core/database.py | 577 ++++++++++++++++++++++--------------------- 1 file changed, 296 insertions(+), 281 deletions(-) diff --git a/couchpotato/core/database.py b/couchpotato/core/database.py index 37841bf..944dca6 100644 --- a/couchpotato/core/database.py +++ b/couchpotato/core/database.py @@ -2,6 +2,7 @@ import json import os import time import traceback +from sqlite3 import OperationalError from CodernityDB.database import RecordNotFound from CodernityDB.index import IndexException, IndexNotFoundException, IndexConflict @@ -9,7 +10,7 @@ from couchpotato import CPLog from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, sp -from couchpotato.core.helpers.variable import getImdb, tryInt +from couchpotato.core.helpers.variable import getImdb, tryInt, randomString log = CPLog(__name__) @@ -311,312 +312,326 @@ class Database(object): } migrate_data = {} + rename_old = False - c = conn.cursor() - - for ml in migrate_list: - migrate_data[ml] = {} - rows = migrate_list[ml] + try: - try: - c.execute('SELECT %s FROM `%s`' % ('`' + '`,`'.join(rows) + '`', ml)) - except: - # ignore faulty destination_id database - if ml == 'category': - migrate_data[ml] = {} - else: - raise + c = conn.cursor() - for p in c.fetchall(): - columns = {} - for row in migrate_list[ml]: - columns[row] = p[rows.index(row)] + for ml in migrate_list: + migrate_data[ml] = {} + rows = migrate_list[ml] - if not migrate_data[ml].get(p[0]): - migrate_data[ml][p[0]] = columns + try: + c.execute('SELECT %s FROM `%s`' % ('`' + '`,`'.join(rows) + '`', ml)) + except: + # ignore faulty destination_id database + if ml == 'category': + migrate_data[ml] = {} + else: + rename_old = True + raise + + for p in c.fetchall(): + columns = {} + for row in migrate_list[ml]: + columns[row] = p[rows.index(row)] + + if not migrate_data[ml].get(p[0]): + migrate_data[ml][p[0]] = columns + else: + if not isinstance(migrate_data[ml][p[0]], list): + migrate_data[ml][p[0]] = [migrate_data[ml][p[0]]] + migrate_data[ml][p[0]].append(columns) + + conn.close() + + log.info('Getting data took %s', time.time() - migrate_start) + + db = self.getDB() + if not db.opened: + return + + # Use properties + properties = migrate_data['properties'] + log.info('Importing %s properties', len(properties)) + for x in properties: + property = properties[x] + Env.prop(property.get('identifier'), property.get('value')) + + # Categories + categories = migrate_data.get('category', []) + log.info('Importing %s categories', len(categories)) + category_link = {} + for x in categories: + c = categories[x] + + new_c = db.insert({ + '_t': 'category', + 'order': c.get('order', 999), + 'label': toUnicode(c.get('label', '')), + 'ignored': toUnicode(c.get('ignored', '')), + 'preferred': toUnicode(c.get('preferred', '')), + 'required': toUnicode(c.get('required', '')), + 'destination': toUnicode(c.get('destination', '')), + }) + + category_link[x] = new_c.get('_id') + + # Profiles + log.info('Importing profiles') + new_profiles = db.all('profile', with_doc = True) + new_profiles_by_label = {} + for x in new_profiles: + + # Remove default non core profiles + if not x['doc'].get('core'): + db.delete(x['doc']) else: - if not isinstance(migrate_data[ml][p[0]], list): - migrate_data[ml][p[0]] = [migrate_data[ml][p[0]]] - migrate_data[ml][p[0]].append(columns) + new_profiles_by_label[x['doc']['label']] = x['_id'] - conn.close() + profiles = migrate_data['profile'] + profile_link = {} + for x in profiles: + p = profiles[x] - log.info('Getting data took %s', time.time() - migrate_start) + exists = new_profiles_by_label.get(p.get('label')) - db = self.getDB() - if not db.opened: - return - - # Use properties - properties = migrate_data['properties'] - log.info('Importing %s properties', len(properties)) - for x in properties: - property = properties[x] - Env.prop(property.get('identifier'), property.get('value')) - - # Categories - categories = migrate_data.get('category', []) - log.info('Importing %s categories', len(categories)) - category_link = {} - for x in categories: - c = categories[x] - - new_c = db.insert({ - '_t': 'category', - 'order': c.get('order', 999), - 'label': toUnicode(c.get('label', '')), - 'ignored': toUnicode(c.get('ignored', '')), - 'preferred': toUnicode(c.get('preferred', '')), - 'required': toUnicode(c.get('required', '')), - 'destination': toUnicode(c.get('destination', '')), - }) - - category_link[x] = new_c.get('_id') - - # Profiles - log.info('Importing profiles') - new_profiles = db.all('profile', with_doc = True) - new_profiles_by_label = {} - for x in new_profiles: - - # Remove default non core profiles - if not x['doc'].get('core'): - db.delete(x['doc']) - else: - new_profiles_by_label[x['doc']['label']] = x['_id'] - - profiles = migrate_data['profile'] - profile_link = {} - for x in profiles: - p = profiles[x] + # Update existing with order only + if exists and p.get('core'): + profile = db.get('id', exists) + profile['order'] = tryInt(p.get('order')) + profile['hide'] = p.get('hide') in [1, True, 'true', 'True'] + db.update(profile) - exists = new_profiles_by_label.get(p.get('label')) + profile_link[x] = profile.get('_id') + else: - # Update existing with order only - if exists and p.get('core'): - profile = db.get('id', exists) - profile['order'] = tryInt(p.get('order')) - profile['hide'] = p.get('hide') in [1, True, 'true', 'True'] - db.update(profile) + new_profile = { + '_t': 'profile', + 'label': p.get('label'), + 'order': int(p.get('order', 999)), + 'core': p.get('core', False), + 'qualities': [], + 'wait_for': [], + 'finish': [] + } - profile_link[x] = profile.get('_id') - else: + types = migrate_data['profiletype'] + for profile_type in types: + p_type = types[profile_type] + if types[profile_type]['profile_id'] == p['id']: + if p_type['quality_id']: + new_profile['finish'].append(p_type['finish']) + new_profile['wait_for'].append(p_type['wait_for']) + new_profile['qualities'].append(migrate_data['quality'][p_type['quality_id']]['identifier']) + + if len(new_profile['qualities']) > 0: + new_profile.update(db.insert(new_profile)) + profile_link[x] = new_profile.get('_id') + else: + log.error('Corrupt profile list for "%s", using default.', p.get('label')) + + # Qualities + log.info('Importing quality sizes') + new_qualities = db.all('quality', with_doc = True) + new_qualities_by_identifier = {} + for x in new_qualities: + new_qualities_by_identifier[x['doc']['identifier']] = x['_id'] + + qualities = migrate_data['quality'] + quality_link = {} + for x in qualities: + q = qualities[x] + q_id = new_qualities_by_identifier[q.get('identifier')] + + quality = db.get('id', q_id) + quality['order'] = q.get('order') + quality['size_min'] = tryInt(q.get('size_min')) + quality['size_max'] = tryInt(q.get('size_max')) + db.update(quality) + + quality_link[x] = quality + + # Titles + titles = migrate_data['librarytitle'] + titles_by_library = {} + for x in titles: + title = titles[x] + if title.get('default'): + titles_by_library[title.get('libraries_id')] = title.get('title') + + # Releases + releaseinfos = migrate_data['releaseinfo'] + for x in releaseinfos: + info = releaseinfos[x] + + # Skip if release doesn't exist for this info + if not migrate_data['release'].get(info.get('release_id')): + continue - new_profile = { - '_t': 'profile', - 'label': p.get('label'), - 'order': int(p.get('order', 999)), - 'core': p.get('core', False), - 'qualities': [], - 'wait_for': [], - 'finish': [] - } - - types = migrate_data['profiletype'] - for profile_type in types: - p_type = types[profile_type] - if types[profile_type]['profile_id'] == p['id']: - if p_type['quality_id']: - new_profile['finish'].append(p_type['finish']) - new_profile['wait_for'].append(p_type['wait_for']) - new_profile['qualities'].append(migrate_data['quality'][p_type['quality_id']]['identifier']) - - if len(new_profile['qualities']) > 0: - new_profile.update(db.insert(new_profile)) - profile_link[x] = new_profile.get('_id') - else: - log.error('Corrupt profile list for "%s", using default.', p.get('label')) - - # Qualities - log.info('Importing quality sizes') - new_qualities = db.all('quality', with_doc = True) - new_qualities_by_identifier = {} - for x in new_qualities: - new_qualities_by_identifier[x['doc']['identifier']] = x['_id'] - - qualities = migrate_data['quality'] - quality_link = {} - for x in qualities: - q = qualities[x] - q_id = new_qualities_by_identifier[q.get('identifier')] - - quality = db.get('id', q_id) - quality['order'] = q.get('order') - quality['size_min'] = tryInt(q.get('size_min')) - quality['size_max'] = tryInt(q.get('size_max')) - db.update(quality) - - quality_link[x] = quality - - # Titles - titles = migrate_data['librarytitle'] - titles_by_library = {} - for x in titles: - title = titles[x] - if title.get('default'): - titles_by_library[title.get('libraries_id')] = title.get('title') - - # Releases - releaseinfos = migrate_data['releaseinfo'] - for x in releaseinfos: - info = releaseinfos[x] - - # Skip if release doesn't exist for this info - if not migrate_data['release'].get(info.get('release_id')): - continue - - if not migrate_data['release'][info.get('release_id')].get('info'): - migrate_data['release'][info.get('release_id')]['info'] = {} - - migrate_data['release'][info.get('release_id')]['info'][info.get('identifier')] = info.get('value') - - releases = migrate_data['release'] - releases_by_media = {} - for x in releases: - release = releases[x] - if not releases_by_media.get(release.get('movie_id')): - releases_by_media[release.get('movie_id')] = [] - - releases_by_media[release.get('movie_id')].append(release) - - # Type ids - types = migrate_data['filetype'] - type_by_id = {} - for t in types: - type = types[t] - type_by_id[type.get('id')] = type - - # Media - log.info('Importing %s media items', len(migrate_data['movie'])) - statuses = migrate_data['status'] - libraries = migrate_data['library'] - library_files = migrate_data['library_files__file_library'] - releases_files = migrate_data['release_files__file_release'] - all_files = migrate_data['file'] - poster_type = migrate_data['filetype']['poster'] - medias = migrate_data['movie'] - for x in medias: - m = medias[x] - - status = statuses.get(m['status_id']).get('identifier') - l = libraries.get(m['library_id']) - - # Only migrate wanted movies, Skip if no identifier present - if not l or not getImdb(l.get('identifier')): continue - - profile_id = profile_link.get(m['profile_id']) - category_id = category_link.get(m['category_id']) - title = titles_by_library.get(m['library_id']) - releases = releases_by_media.get(x, []) - info = json.loads(l.get('info', '')) - - files = library_files.get(m['library_id'], []) - if not isinstance(files, list): - files = [files] - - added_media = fireEvent('movie.add', { - 'info': info, - 'identifier': l.get('identifier'), - 'profile_id': profile_id, - 'category_id': category_id, - 'title': title - }, force_readd = False, search_after = False, update_after = False, notify_after = False, status = status, single = True) - - if not added_media: - log.error('Failed adding media %s: %s', (l.get('identifier'), info)) - continue - - added_media['files'] = added_media.get('files', {}) - for f in files: - ffile = all_files[f.get('file_id')] - - # Only migrate posters - if ffile.get('type_id') == poster_type.get('id'): - if ffile.get('path') not in added_media['files'].get('image_poster', []) and os.path.isfile(ffile.get('path')): - added_media['files']['image_poster'] = [ffile.get('path')] - break - - if 'image_poster' in added_media['files']: - db.update(added_media) - - for rel in releases: - - empty_info = False - if not rel.get('info'): - empty_info = True - rel['info'] = {} - - quality = quality_link.get(rel.get('quality_id')) - if not quality: + if not migrate_data['release'][info.get('release_id')].get('info'): + migrate_data['release'][info.get('release_id')]['info'] = {} + + migrate_data['release'][info.get('release_id')]['info'][info.get('identifier')] = info.get('value') + + releases = migrate_data['release'] + releases_by_media = {} + for x in releases: + release = releases[x] + if not releases_by_media.get(release.get('movie_id')): + releases_by_media[release.get('movie_id')] = [] + + releases_by_media[release.get('movie_id')].append(release) + + # Type ids + types = migrate_data['filetype'] + type_by_id = {} + for t in types: + type = types[t] + type_by_id[type.get('id')] = type + + # Media + log.info('Importing %s media items', len(migrate_data['movie'])) + statuses = migrate_data['status'] + libraries = migrate_data['library'] + library_files = migrate_data['library_files__file_library'] + releases_files = migrate_data['release_files__file_release'] + all_files = migrate_data['file'] + poster_type = migrate_data['filetype']['poster'] + medias = migrate_data['movie'] + for x in medias: + m = medias[x] + + status = statuses.get(m['status_id']).get('identifier') + l = libraries.get(m['library_id']) + + # Only migrate wanted movies, Skip if no identifier present + if not l or not getImdb(l.get('identifier')): continue + + profile_id = profile_link.get(m['profile_id']) + category_id = category_link.get(m['category_id']) + title = titles_by_library.get(m['library_id']) + releases = releases_by_media.get(x, []) + info = json.loads(l.get('info', '')) + + files = library_files.get(m['library_id'], []) + if not isinstance(files, list): + files = [files] + + added_media = fireEvent('movie.add', { + 'info': info, + 'identifier': l.get('identifier'), + 'profile_id': profile_id, + 'category_id': category_id, + 'title': title + }, force_readd = False, search_after = False, update_after = False, notify_after = False, status = status, single = True) + + if not added_media: + log.error('Failed adding media %s: %s', (l.get('identifier'), info)) continue - release_status = statuses.get(rel.get('status_id')).get('identifier') + added_media['files'] = added_media.get('files', {}) + for f in files: + ffile = all_files[f.get('file_id')] - if rel['info'].get('download_id'): - status_support = rel['info'].get('download_status_support', False) in [True, 'true', 'True'] - rel['info']['download_info'] = { - 'id': rel['info'].get('download_id'), - 'downloader': rel['info'].get('download_downloader'), - 'status_support': status_support, - } + # Only migrate posters + if ffile.get('type_id') == poster_type.get('id'): + if ffile.get('path') not in added_media['files'].get('image_poster', []) and os.path.isfile(ffile.get('path')): + added_media['files']['image_poster'] = [ffile.get('path')] + break - # Add status to keys - rel['info']['status'] = release_status - if not empty_info: - fireEvent('release.create_from_search', [rel['info']], added_media, quality, single = True) - else: - release = { - '_t': 'release', - 'identifier': rel.get('identifier'), - 'media_id': added_media.get('_id'), - 'quality': quality.get('identifier'), - 'status': release_status, - 'last_edit': int(time.time()), - 'files': {} - } + if 'image_poster' in added_media['files']: + db.update(added_media) - # Add downloader info if provided - try: - release['download_info'] = rel['info']['download_info'] - del rel['download_info'] - except: - pass + for rel in releases: - # Add files - release_files = releases_files.get(rel.get('id'), []) - if not isinstance(release_files, list): - release_files = [release_files] + empty_info = False + if not rel.get('info'): + empty_info = True + rel['info'] = {} - if len(release_files) == 0: + quality = quality_link.get(rel.get('quality_id')) + if not quality: continue - for f in release_files: - rfile = all_files.get(f.get('file_id')) - if not rfile: + release_status = statuses.get(rel.get('status_id')).get('identifier') + + if rel['info'].get('download_id'): + status_support = rel['info'].get('download_status_support', False) in [True, 'true', 'True'] + rel['info']['download_info'] = { + 'id': rel['info'].get('download_id'), + 'downloader': rel['info'].get('download_downloader'), + 'status_support': status_support, + } + + # Add status to keys + rel['info']['status'] = release_status + if not empty_info: + fireEvent('release.create_from_search', [rel['info']], added_media, quality, single = True) + else: + release = { + '_t': 'release', + 'identifier': rel.get('identifier'), + 'media_id': added_media.get('_id'), + 'quality': quality.get('identifier'), + 'status': release_status, + 'last_edit': int(time.time()), + 'files': {} + } + + # Add downloader info if provided + try: + release['download_info'] = rel['info']['download_info'] + del rel['download_info'] + except: + pass + + # Add files + release_files = releases_files.get(rel.get('id'), []) + if not isinstance(release_files, list): + release_files = [release_files] + + if len(release_files) == 0: continue - file_type = type_by_id.get(rfile.get('type_id')).get('identifier') + for f in release_files: + rfile = all_files.get(f.get('file_id')) + if not rfile: + continue - if not release['files'].get(file_type): - release['files'][file_type] = [] + file_type = type_by_id.get(rfile.get('type_id')).get('identifier') - release['files'][file_type].append(rfile.get('path')) + if not release['files'].get(file_type): + release['files'][file_type] = [] - try: - rls = db.get('release_identifier', rel.get('identifier'), with_doc = True)['doc'] - rls.update(release) - db.update(rls) - except: - db.insert(release) + release['files'][file_type].append(rfile.get('path')) - log.info('Total migration took %s', time.time() - migrate_start) - log.info('=' * 30) + try: + rls = db.get('release_identifier', rel.get('identifier'), with_doc = True)['doc'] + rls.update(release) + db.update(rls) + except: + db.insert(release) - # rename old database - log.info('Renaming old database to %s ', old_db + '.old') - os.rename(old_db, old_db + '.old') + log.info('Total migration took %s', time.time() - migrate_start) + log.info('=' * 30) + + rename_old = True - if os.path.isfile(old_db + '-wal'): - os.rename(old_db + '-wal', old_db + '-wal.old') - if os.path.isfile(old_db + '-shm'): - os.rename(old_db + '-shm', old_db + '-shm.old') + except OperationalError: + log.error('Migrating from faulty database, probably a (too) old version: %s', traceback.format_exc()) + except: + log.error('Migration failed: %s', traceback.format_exc()) + + + # rename old database + if rename_old: + random = randomString() + log.info('Renaming old database to %s ', '%s.%s_old' % (old_db, random)) + os.rename(old_db, '%s.%s_old' % (old_db, random)) + + if os.path.isfile(old_db + '-wal'): + os.rename(old_db + '-wal', '%s-wal.%s_old' % (old_db, random)) + if os.path.isfile(old_db + '-shm'): + os.rename(old_db + '-shm', '%s-shm.%s_old' % (old_db, random)) From dbc254efbe08b44db5be734704e8d438bd9c336b Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 5 Oct 2014 11:15:55 +0200 Subject: [PATCH 175/202] Also encode log with names tuples --- couchpotato/core/logger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/logger.py b/couchpotato/core/logger.py index 32320e4..f97f6ae 100644 --- a/couchpotato/core/logger.py +++ b/couchpotato/core/logger.py @@ -61,6 +61,8 @@ class CPLog(object): try: if isinstance(replace_tuple, tuple): msg = msg % tuple([ss(x) for x in list(replace_tuple)]) + elif isinstance(replace_tuple, dict): + msg = msg % dict((k, ss(v)) for k, v in replace_tuple.iteritems()) else: msg = msg % ss(replace_tuple) except Exception as e: From 755873c5e79b87ac4d3805f53e3d2d04cf6cc051 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 5 Oct 2014 12:33:13 +0200 Subject: [PATCH 176/202] Don't ss int and float --- couchpotato/core/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/logger.py b/couchpotato/core/logger.py index f97f6ae..a1b5e7d 100644 --- a/couchpotato/core/logger.py +++ b/couchpotato/core/logger.py @@ -60,7 +60,7 @@ class CPLog(object): try: if isinstance(replace_tuple, tuple): - msg = msg % tuple([ss(x) for x in list(replace_tuple)]) + msg = msg % tuple([ss(x) if not isinstance(x, (int, float)) else x for x in list(replace_tuple)]) elif isinstance(replace_tuple, dict): msg = msg % dict((k, ss(v)) for k, v in replace_tuple.iteritems()) else: From f897eebb4117046db12ffb7acc907dd5248337c1 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 5 Oct 2014 12:37:00 +0200 Subject: [PATCH 177/202] Allow regex in required and ignored words close #3376 --- couchpotato/core/media/_base/searcher/main.py | 45 ++++++++++++++------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index 5845c0d..e9ba95d 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -178,6 +178,25 @@ class Searcher(SearcherBase): return False + def containsWords(self, rel_name, rel_words, conf, media): + + # Make sure it has required words + words = splitString(self.conf('%s_words' % conf, section = 'searcher').lower()) + try: words = removeDuplicate(words + splitString(media['category'][conf].lower())) + except: pass + + req_match = 0 + for req_set in words: + if len(req_set) >= 2 and (req_set[:1] + req_set[-1:]) == '//': + if re.search(req_set[1:-1], rel_name): + log.debug('Regex match: %s', req_set[1:-1]) + req_match += 1 + else: + req = splitString(req_set, '&') + req_match += len(list(set(rel_words) & set(req))) == len(req) + + return words, req_match > 0 + def correctWords(self, rel_name, media): media_title = fireEvent('searcher.get_search_title', media, single = True) media_words = re.split('\W+', simplifyString(media_title)) @@ -185,31 +204,13 @@ class Searcher(SearcherBase): rel_name = simplifyString(rel_name) rel_words = re.split('\W+', rel_name) - # Make sure it has required words - required_words = splitString(self.conf('required_words', section = 'searcher').lower()) - try: required_words = removeDuplicate(required_words + splitString(media['category']['required'].lower())) - except: pass - - req_match = 0 - for req_set in required_words: - req = splitString(req_set, '&') - req_match += len(list(set(rel_words) & set(req))) == len(req) - - if len(required_words) > 0 and req_match == 0: + required_words, contains_required = self.containsWords(rel_name, rel_words, 'required', media) + if len(required_words) > 0 and not contains_required: log.info2('Wrong: Required word missing: %s', rel_name) return False - # Ignore releases - ignored_words = splitString(self.conf('ignored_words', section = 'searcher').lower()) - try: ignored_words = removeDuplicate(ignored_words + splitString(media['category']['ignored'].lower())) - except: pass - - ignored_match = 0 - for ignored_set in ignored_words: - ignored = splitString(ignored_set, '&') - ignored_match += len(list(set(rel_words) & set(ignored))) == len(ignored) - - if len(ignored_words) > 0 and ignored_match: + ignored_words, contains_ignored = self.containsWords(rel_name, rel_words, 'ignored', media) + if len(ignored_words) > 0 and contains_ignored: log.info2("Wrong: '%s' contains 'ignored words'", rel_name) return False From ab118ea580e9ac07b84401d3e77c4ac6b204358d Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 5 Oct 2014 13:38:22 +0200 Subject: [PATCH 178/202] Windows special chars not returning any folders --- couchpotato/core/helpers/variable.py | 4 ++-- couchpotato/core/plugins/browser.py | 29 ++++++++++++++++++++--------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index 9e064b7..519c369 100755 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -41,11 +41,11 @@ def symlink(src, dst): def getUserDir(): try: import pwd - os.environ['HOME'] = pwd.getpwuid(os.geteuid()).pw_dir + os.environ['HOME'] = sp(pwd.getpwuid(os.geteuid()).pw_dir) except: pass - return os.path.expanduser('~') + return sp(os.path.expanduser('~')) def getDownloadDir(): diff --git a/couchpotato/core/plugins/browser.py b/couchpotato/core/plugins/browser.py index 013a482..632375d 100644 --- a/couchpotato/core/plugins/browser.py +++ b/couchpotato/core/plugins/browser.py @@ -1,12 +1,18 @@ import ctypes import os import string +import traceback +import time +from couchpotato import CPLog from couchpotato.api import addApiView -from couchpotato.core.helpers.encoding import sp +from couchpotato.core.event import addEvent +from couchpotato.core.helpers.encoding import sp, ss, toUnicode from couchpotato.core.helpers.variable import getUserDir from couchpotato.core.plugins.base import Plugin -import six + + +log = CPLog(__name__) if os.name == 'nt': @@ -53,9 +59,9 @@ class FileBrowser(Plugin): dirs = [] path = sp(path) for f in os.listdir(path): - p = os.path.join(path, f) + p = sp(os.path.join(path, f)) if os.path.isdir(p) and ((self.is_hidden(p) and bool(int(show_hidden))) or not self.is_hidden(p)): - dirs.append(p + os.path.sep) + dirs.append(toUnicode('%s%s' % (p, os.path.sep))) return sorted(dirs) @@ -66,8 +72,8 @@ class FileBrowser(Plugin): driveletters = [] for drive in string.ascii_uppercase: - if win32file.GetDriveType(drive + ":") in [win32file.DRIVE_FIXED, win32file.DRIVE_REMOTE, win32file.DRIVE_RAMDISK, win32file.DRIVE_REMOVABLE]: - driveletters.append(drive + ":\\") + if win32file.GetDriveType(drive + ':') in [win32file.DRIVE_FIXED, win32file.DRIVE_REMOTE, win32file.DRIVE_RAMDISK, win32file.DRIVE_REMOVABLE]: + driveletters.append(drive + ':\\') return driveletters @@ -100,14 +106,19 @@ class FileBrowser(Plugin): def is_hidden(self, filepath): - name = os.path.basename(os.path.abspath(filepath)) + name = ss(os.path.basename(os.path.abspath(filepath))) return name.startswith('.') or self.has_hidden_attribute(filepath) def has_hidden_attribute(self, filepath): + + result = False try: - attrs = ctypes.windll.kernel32.GetFileAttributesW(six.text_type(filepath)) #@UndefinedVariable + attrs = ctypes.windll.kernel32.GetFileAttributesW(sp(filepath)) #@UndefinedVariable assert attrs != -1 result = bool(attrs & 2) except (AttributeError, AssertionError): - result = False + pass + except: + log.error('Failed getting hidden attribute: %s', traceback.format_exc()) + return result From 984ee7580d6442f2b07b742537b04afc3874533c Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 5 Oct 2014 13:53:57 +0200 Subject: [PATCH 179/202] Force unicode on file loading for guessit --- libs/guessit/fileutils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libs/guessit/fileutils.py b/libs/guessit/fileutils.py index 9531f82..de30b8d 100755 --- a/libs/guessit/fileutils.py +++ b/libs/guessit/fileutils.py @@ -79,7 +79,9 @@ def file_in_same_dir(ref_file, desired_file): def load_file_in_same_dir(ref_file, filename): """Load a given file. Works even when the file is contained inside a zip.""" - path = split_path(ref_file)[:-1] + [filename] + + from couchpotato.core.helpers.encoding import toUnicode + path = split_path(toUnicode(ref_file))[:-1] + [filename] for i, p in enumerate(path): if p.endswith('.zip'): From e08d06ba31102be3ef10325e34d918a1db1ab25a Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 5 Oct 2014 14:40:27 +0200 Subject: [PATCH 180/202] Update chardet --- libs/chardet/__init__.py | 14 +- libs/chardet/big5freq.py | 20 +- libs/chardet/big5prober.py | 17 +- libs/chardet/chardetect.py | 46 ++ libs/chardet/chardistribution.py | 159 ++++--- libs/chardet/charsetgroupprober.py | 36 +- libs/chardet/charsetprober.py | 24 +- libs/chardet/codingstatemachine.py | 15 +- libs/chardet/compat.py | 34 ++ libs/chardet/constants.py | 8 - libs/chardet/cp949prober.py | 44 ++ libs/chardet/escprober.py | 39 +- libs/chardet/escsm.py | 338 +++++++-------- libs/chardet/eucjpprober.py | 45 +- libs/chardet/euckrfreq.py | 2 + libs/chardet/euckrprober.py | 13 +- libs/chardet/euctwfreq.py | 16 +- libs/chardet/euctwprober.py | 8 +- libs/chardet/gb2312freq.py | 9 +- libs/chardet/gb2312prober.py | 8 +- libs/chardet/hebrewprober.py | 184 ++++---- libs/chardet/jisfreq.py | 16 +- libs/chardet/jpcntx.py | 89 ++-- libs/chardet/langbulgarianmodel.py | 29 +- libs/chardet/langcyrillicmodel.py | 50 +-- libs/chardet/langgreekmodel.py | 26 +- libs/chardet/langhebrewmodel.py | 20 +- libs/chardet/langhungarianmodel.py | 26 +- libs/chardet/langthaimodel.py | 22 +- libs/chardet/latin1prober.py | 141 ++++--- libs/chardet/mbcharsetprober.py | 36 +- libs/chardet/mbcsgroupprober.py | 28 +- libs/chardet/mbcssm.py | 835 ++++++++++++++++++++----------------- libs/chardet/sbcharsetprober.py | 52 ++- libs/chardet/sbcsgroupprober.py | 39 +- libs/chardet/sjisprober.py | 48 ++- libs/chardet/universaldetector.py | 94 +++-- libs/chardet/utf8prober.py | 20 +- 38 files changed, 1491 insertions(+), 1159 deletions(-) create mode 100644 libs/chardet/chardetect.py create mode 100644 libs/chardet/compat.py create mode 100644 libs/chardet/cp949prober.py diff --git a/libs/chardet/__init__.py b/libs/chardet/__init__.py index b1872fe..e4f0799 100755 --- a/libs/chardet/__init__.py +++ b/libs/chardet/__init__.py @@ -3,22 +3,28 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -__version__ = "1.0.1" +__version__ = "2.2.1" +from sys import version_info + def detect(aBuf): - import universaldetector + if ((version_info < (3, 0) and isinstance(aBuf, unicode)) or + (version_info >= (3, 0) and not isinstance(aBuf, bytes))): + raise ValueError('Expected a bytes object, not a unicode object') + + from . import universaldetector u = universaldetector.UniversalDetector() u.reset() u.feed(aBuf) diff --git a/libs/chardet/big5freq.py b/libs/chardet/big5freq.py index c1b0f3c..65bffc0 100755 --- a/libs/chardet/big5freq.py +++ b/libs/chardet/big5freq.py @@ -1,11 +1,11 @@ ######################## BEGIN LICENSE BLOCK ######################## # The Original Code is Mozilla Communicator client code. -# +# # The Initial Developer of the Original Code is # Netscape Communications Corporation. # Portions created by the Initial Developer are Copyright (C) 1998 # the Initial Developer. All Rights Reserved. -# +# # Contributor(s): # Mark Pilgrim - port to Python # @@ -13,12 +13,12 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA @@ -26,18 +26,18 @@ ######################### END LICENSE BLOCK ######################### # Big5 frequency table -# by Taiwan's Mandarin Promotion Council +# by Taiwan's Mandarin Promotion Council # -# +# # 128 --> 0.42261 # 256 --> 0.57851 # 512 --> 0.74851 # 1024 --> 0.89384 # 2048 --> 0.97583 -# +# # Ideal Distribution Ratio = 0.74851/(1-0.74851) =2.98 # Random Distribution Ration = 512/(5401-512)=0.105 -# +# # Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR BIG5_TYPICAL_DISTRIBUTION_RATIO = 0.75 @@ -45,7 +45,7 @@ BIG5_TYPICAL_DISTRIBUTION_RATIO = 0.75 #Char to FreqOrder table BIG5_TABLE_SIZE = 5376 -Big5CharToFreqOrder = ( \ +Big5CharToFreqOrder = ( 1,1801,1506, 255,1431, 198, 9, 82, 6,5008, 177, 202,3681,1256,2821, 110, # 16 3814, 33,3274, 261, 76, 44,2114, 16,2946,2187,1176, 659,3971, 26,3451,2653, # 32 1198,3972,3350,4202, 410,2215, 302, 590, 361,1964, 8, 204, 58,4510,5009,1932, # 48 @@ -921,3 +921,5 @@ Big5CharToFreqOrder = ( \ 13936,13937,13938,13939,13940,13941,13942,13943,13944,13945,13946,13947,13948,13949,13950,13951, #13952 13952,13953,13954,13955,13956,13957,13958,13959,13960,13961,13962,13963,13964,13965,13966,13967, #13968 13968,13969,13970,13971,13972) #13973 + +# flake8: noqa diff --git a/libs/chardet/big5prober.py b/libs/chardet/big5prober.py index e6b52aa..becce81 100755 --- a/libs/chardet/big5prober.py +++ b/libs/chardet/big5prober.py @@ -1,11 +1,11 @@ ######################## BEGIN LICENSE BLOCK ######################## # The Original Code is Mozilla Communicator client code. -# +# # The Initial Developer of the Original Code is # Netscape Communications Corporation. # Portions created by the Initial Developer are Copyright (C) 1998 # the Initial Developer. All Rights Reserved. -# +# # Contributor(s): # Mark Pilgrim - port to Python # @@ -13,22 +13,23 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -from mbcharsetprober import MultiByteCharSetProber -from codingstatemachine import CodingStateMachine -from chardistribution import Big5DistributionAnalysis -from mbcssm import Big5SMModel +from .mbcharsetprober import MultiByteCharSetProber +from .codingstatemachine import CodingStateMachine +from .chardistribution import Big5DistributionAnalysis +from .mbcssm import Big5SMModel + class Big5Prober(MultiByteCharSetProber): def __init__(self): diff --git a/libs/chardet/chardetect.py b/libs/chardet/chardetect.py new file mode 100644 index 0000000..ecd0163 --- /dev/null +++ b/libs/chardet/chardetect.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +""" +Script which takes one or more file paths and reports on their detected +encodings + +Example:: + + % chardetect somefile someotherfile + somefile: windows-1252 with confidence 0.5 + someotherfile: ascii with confidence 1.0 + +If no paths are provided, it takes its input from stdin. + +""" +from io import open +from sys import argv, stdin + +from chardet.universaldetector import UniversalDetector + + +def description_of(file, name='stdin'): + """Return a string describing the probable encoding of a file.""" + u = UniversalDetector() + for line in file: + u.feed(line) + u.close() + result = u.result + if result['encoding']: + return '%s: %s with confidence %s' % (name, + result['encoding'], + result['confidence']) + else: + return '%s: no result' % name + + +def main(): + if len(argv) <= 1: + print(description_of(stdin)) + else: + for path in argv[1:]: + with open(path, 'rb') as f: + print(description_of(f, path)) + + +if __name__ == '__main__': + main() diff --git a/libs/chardet/chardistribution.py b/libs/chardet/chardistribution.py index b893341..4e64a00 100755 --- a/libs/chardet/chardistribution.py +++ b/libs/chardet/chardistribution.py @@ -1,11 +1,11 @@ ######################## BEGIN LICENSE BLOCK ######################## # The Original Code is Mozilla Communicator client code. -# +# # The Initial Developer of the Original Code is # Netscape Communications Corporation. # Portions created by the Initial Developer are Copyright (C) 1998 # the Initial Developer. All Rights Reserved. -# +# # Contributor(s): # Mark Pilgrim - port to Python # @@ -13,47 +13,63 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants -from euctwfreq import EUCTWCharToFreqOrder, EUCTW_TABLE_SIZE, EUCTW_TYPICAL_DISTRIBUTION_RATIO -from euckrfreq import EUCKRCharToFreqOrder, EUCKR_TABLE_SIZE, EUCKR_TYPICAL_DISTRIBUTION_RATIO -from gb2312freq import GB2312CharToFreqOrder, GB2312_TABLE_SIZE, GB2312_TYPICAL_DISTRIBUTION_RATIO -from big5freq import Big5CharToFreqOrder, BIG5_TABLE_SIZE, BIG5_TYPICAL_DISTRIBUTION_RATIO -from jisfreq import JISCharToFreqOrder, JIS_TABLE_SIZE, JIS_TYPICAL_DISTRIBUTION_RATIO +from .euctwfreq import (EUCTWCharToFreqOrder, EUCTW_TABLE_SIZE, + EUCTW_TYPICAL_DISTRIBUTION_RATIO) +from .euckrfreq import (EUCKRCharToFreqOrder, EUCKR_TABLE_SIZE, + EUCKR_TYPICAL_DISTRIBUTION_RATIO) +from .gb2312freq import (GB2312CharToFreqOrder, GB2312_TABLE_SIZE, + GB2312_TYPICAL_DISTRIBUTION_RATIO) +from .big5freq import (Big5CharToFreqOrder, BIG5_TABLE_SIZE, + BIG5_TYPICAL_DISTRIBUTION_RATIO) +from .jisfreq import (JISCharToFreqOrder, JIS_TABLE_SIZE, + JIS_TYPICAL_DISTRIBUTION_RATIO) +from .compat import wrap_ord ENOUGH_DATA_THRESHOLD = 1024 SURE_YES = 0.99 SURE_NO = 0.01 +MINIMUM_DATA_THRESHOLD = 3 + class CharDistributionAnalysis: def __init__(self): - self._mCharToFreqOrder = None # Mapping table to get frequency order from char order (get from GetOrder()) - self._mTableSize = None # Size of above table - self._mTypicalDistributionRatio = None # This is a constant value which varies from language to language, used in calculating confidence. See http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html for further detail. + # Mapping table to get frequency order from char order (get from + # GetOrder()) + self._mCharToFreqOrder = None + self._mTableSize = None # Size of above table + # This is a constant value which varies from language to language, + # used in calculating confidence. See + # http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html + # for further detail. + self._mTypicalDistributionRatio = None self.reset() - + def reset(self): """reset analyser, clear any state""" - self._mDone = constants.False # If this flag is set to constants.True, detection is done and conclusion has been made - self._mTotalChars = 0 # Total characters encountered - self._mFreqChars = 0 # The number of characters whose frequency order is less than 512 - - def feed(self, aStr, aCharLen): + # If this flag is set to True, detection is done and conclusion has + # been made + self._mDone = False + self._mTotalChars = 0 # Total characters encountered + # The number of characters whose frequency order is less than 512 + self._mFreqChars = 0 + + def feed(self, aBuf, aCharLen): """feed a character with known length""" if aCharLen == 2: # we only care about 2-bytes character in our distribution analysis - order = self.get_order(aStr) + order = self.get_order(aBuf) else: order = -1 if order >= 0: @@ -65,12 +81,14 @@ class CharDistributionAnalysis: def get_confidence(self): """return confidence based on existing data""" - # if we didn't receive any character in our consideration range, return negative answer - if self._mTotalChars <= 0: + # if we didn't receive any character in our consideration range, + # return negative answer + if self._mTotalChars <= 0 or self._mFreqChars <= MINIMUM_DATA_THRESHOLD: return SURE_NO if self._mTotalChars != self._mFreqChars: - r = self._mFreqChars / ((self._mTotalChars - self._mFreqChars) * self._mTypicalDistributionRatio) + r = (self._mFreqChars / ((self._mTotalChars - self._mFreqChars) + * self._mTypicalDistributionRatio)) if r < SURE_YES: return r @@ -78,16 +96,18 @@ class CharDistributionAnalysis: return SURE_YES def got_enough_data(self): - # It is not necessary to receive all data to draw conclusion. For charset detection, - # certain amount of data is enough + # It is not necessary to receive all data to draw conclusion. + # For charset detection, certain amount of data is enough return self._mTotalChars > ENOUGH_DATA_THRESHOLD - def get_order(self, aStr): - # We do not handle characters based on the original encoding string, but - # convert this encoding string to a number, here called order. - # This allows multiple encodings of a language to share one frequency table. + def get_order(self, aBuf): + # We do not handle characters based on the original encoding string, + # but convert this encoding string to a number, here called order. + # This allows multiple encodings of a language to share one frequency + # table. return -1 - + + class EUCTWDistributionAnalysis(CharDistributionAnalysis): def __init__(self): CharDistributionAnalysis.__init__(self) @@ -95,16 +115,18 @@ class EUCTWDistributionAnalysis(CharDistributionAnalysis): self._mTableSize = EUCTW_TABLE_SIZE self._mTypicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO - def get_order(self, aStr): - # for euc-TW encoding, we are interested + def get_order(self, aBuf): + # for euc-TW encoding, we are interested # first byte range: 0xc4 -- 0xfe # second byte range: 0xa1 -- 0xfe # no validation needed here. State machine has done that - if aStr[0] >= '\xC4': - return 94 * (ord(aStr[0]) - 0xC4) + ord(aStr[1]) - 0xA1 + first_char = wrap_ord(aBuf[0]) + if first_char >= 0xC4: + return 94 * (first_char - 0xC4) + wrap_ord(aBuf[1]) - 0xA1 else: return -1 + class EUCKRDistributionAnalysis(CharDistributionAnalysis): def __init__(self): CharDistributionAnalysis.__init__(self) @@ -112,15 +134,17 @@ class EUCKRDistributionAnalysis(CharDistributionAnalysis): self._mTableSize = EUCKR_TABLE_SIZE self._mTypicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO - def get_order(self, aStr): - # for euc-KR encoding, we are interested + def get_order(self, aBuf): + # for euc-KR encoding, we are interested # first byte range: 0xb0 -- 0xfe # second byte range: 0xa1 -- 0xfe # no validation needed here. State machine has done that - if aStr[0] >= '\xB0': - return 94 * (ord(aStr[0]) - 0xB0) + ord(aStr[1]) - 0xA1 + first_char = wrap_ord(aBuf[0]) + if first_char >= 0xB0: + return 94 * (first_char - 0xB0) + wrap_ord(aBuf[1]) - 0xA1 else: - return -1; + return -1 + class GB2312DistributionAnalysis(CharDistributionAnalysis): def __init__(self): @@ -129,15 +153,17 @@ class GB2312DistributionAnalysis(CharDistributionAnalysis): self._mTableSize = GB2312_TABLE_SIZE self._mTypicalDistributionRatio = GB2312_TYPICAL_DISTRIBUTION_RATIO - def get_order(self, aStr): - # for GB2312 encoding, we are interested + def get_order(self, aBuf): + # for GB2312 encoding, we are interested # first byte range: 0xb0 -- 0xfe # second byte range: 0xa1 -- 0xfe # no validation needed here. State machine has done that - if (aStr[0] >= '\xB0') and (aStr[1] >= '\xA1'): - return 94 * (ord(aStr[0]) - 0xB0) + ord(aStr[1]) - 0xA1 + first_char, second_char = wrap_ord(aBuf[0]), wrap_ord(aBuf[1]) + if (first_char >= 0xB0) and (second_char >= 0xA1): + return 94 * (first_char - 0xB0) + second_char - 0xA1 else: - return -1; + return -1 + class Big5DistributionAnalysis(CharDistributionAnalysis): def __init__(self): @@ -146,19 +172,21 @@ class Big5DistributionAnalysis(CharDistributionAnalysis): self._mTableSize = BIG5_TABLE_SIZE self._mTypicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO - def get_order(self, aStr): - # for big5 encoding, we are interested + def get_order(self, aBuf): + # for big5 encoding, we are interested # first byte range: 0xa4 -- 0xfe # second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe # no validation needed here. State machine has done that - if aStr[0] >= '\xA4': - if aStr[1] >= '\xA1': - return 157 * (ord(aStr[0]) - 0xA4) + ord(aStr[1]) - 0xA1 + 63 + first_char, second_char = wrap_ord(aBuf[0]), wrap_ord(aBuf[1]) + if first_char >= 0xA4: + if second_char >= 0xA1: + return 157 * (first_char - 0xA4) + second_char - 0xA1 + 63 else: - return 157 * (ord(aStr[0]) - 0xA4) + ord(aStr[1]) - 0x40 + return 157 * (first_char - 0xA4) + second_char - 0x40 else: return -1 + class SJISDistributionAnalysis(CharDistributionAnalysis): def __init__(self): CharDistributionAnalysis.__init__(self) @@ -166,22 +194,24 @@ class SJISDistributionAnalysis(CharDistributionAnalysis): self._mTableSize = JIS_TABLE_SIZE self._mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO - def get_order(self, aStr): - # for sjis encoding, we are interested + def get_order(self, aBuf): + # for sjis encoding, we are interested # first byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe # second byte range: 0x40 -- 0x7e, 0x81 -- oxfe # no validation needed here. State machine has done that - if (aStr[0] >= '\x81') and (aStr[0] <= '\x9F'): - order = 188 * (ord(aStr[0]) - 0x81) - elif (aStr[0] >= '\xE0') and (aStr[0] <= '\xEF'): - order = 188 * (ord(aStr[0]) - 0xE0 + 31) + first_char, second_char = wrap_ord(aBuf[0]), wrap_ord(aBuf[1]) + if (first_char >= 0x81) and (first_char <= 0x9F): + order = 188 * (first_char - 0x81) + elif (first_char >= 0xE0) and (first_char <= 0xEF): + order = 188 * (first_char - 0xE0 + 31) else: - return -1; - order = order + ord(aStr[1]) - 0x40 - if aStr[1] > '\x7F': - order =- 1 + return -1 + order = order + second_char - 0x40 + if second_char > 0x7F: + order = -1 return order + class EUCJPDistributionAnalysis(CharDistributionAnalysis): def __init__(self): CharDistributionAnalysis.__init__(self) @@ -189,12 +219,13 @@ class EUCJPDistributionAnalysis(CharDistributionAnalysis): self._mTableSize = JIS_TABLE_SIZE self._mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO - def get_order(self, aStr): - # for euc-JP encoding, we are interested + def get_order(self, aBuf): + # for euc-JP encoding, we are interested # first byte range: 0xa0 -- 0xfe # second byte range: 0xa1 -- 0xfe # no validation needed here. State machine has done that - if aStr[0] >= '\xA0': - return 94 * (ord(aStr[0]) - 0xA1) + ord(aStr[1]) - 0xa1 + char = wrap_ord(aBuf[0]) + if char >= 0xA0: + return 94 * (char - 0xA1) + wrap_ord(aBuf[1]) - 0xa1 else: return -1 diff --git a/libs/chardet/charsetgroupprober.py b/libs/chardet/charsetgroupprober.py index 5188069..85e7a1c 100755 --- a/libs/chardet/charsetgroupprober.py +++ b/libs/chardet/charsetgroupprober.py @@ -25,8 +25,10 @@ # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants, sys -from charsetprober import CharSetProber +from . import constants +import sys +from .charsetprober import CharSetProber + class CharSetGroupProber(CharSetProber): def __init__(self): @@ -34,35 +36,39 @@ class CharSetGroupProber(CharSetProber): self._mActiveNum = 0 self._mProbers = [] self._mBestGuessProber = None - + def reset(self): CharSetProber.reset(self) self._mActiveNum = 0 for prober in self._mProbers: if prober: prober.reset() - prober.active = constants.True + prober.active = True self._mActiveNum += 1 self._mBestGuessProber = None def get_charset_name(self): if not self._mBestGuessProber: self.get_confidence() - if not self._mBestGuessProber: return None + if not self._mBestGuessProber: + return None # self._mBestGuessProber = self._mProbers[0] return self._mBestGuessProber.get_charset_name() def feed(self, aBuf): for prober in self._mProbers: - if not prober: continue - if not prober.active: continue + if not prober: + continue + if not prober.active: + continue st = prober.feed(aBuf) - if not st: continue + if not st: + continue if st == constants.eFoundIt: self._mBestGuessProber = prober return self.get_state() elif st == constants.eNotMe: - prober.active = constants.False + prober.active = False self._mActiveNum -= 1 if self._mActiveNum <= 0: self._mState = constants.eNotMe @@ -78,18 +84,22 @@ class CharSetGroupProber(CharSetProber): bestConf = 0.0 self._mBestGuessProber = None for prober in self._mProbers: - if not prober: continue + if not prober: + continue if not prober.active: if constants._debug: - sys.stderr.write(prober.get_charset_name() + ' not active\n') + sys.stderr.write(prober.get_charset_name() + + ' not active\n') continue cf = prober.get_confidence() if constants._debug: - sys.stderr.write('%s confidence = %s\n' % (prober.get_charset_name(), cf)) + sys.stderr.write('%s confidence = %s\n' % + (prober.get_charset_name(), cf)) if bestConf < cf: bestConf = cf self._mBestGuessProber = prober - if not self._mBestGuessProber: return 0.0 + if not self._mBestGuessProber: + return 0.0 return bestConf # else: # self._mBestGuessProber = self._mProbers[0] diff --git a/libs/chardet/charsetprober.py b/libs/chardet/charsetprober.py index 3ac1683..9758171 100755 --- a/libs/chardet/charsetprober.py +++ b/libs/chardet/charsetprober.py @@ -1,11 +1,11 @@ ######################## BEGIN LICENSE BLOCK ######################## # The Original Code is Mozilla Universal charset detector code. -# +# # The Initial Developer of the Original Code is # Netscape Communications Corporation. # Portions created by the Initial Developer are Copyright (C) 2001 # the Initial Developer. All Rights Reserved. -# +# # Contributor(s): # Mark Pilgrim - port to Python # Shy Shalom - original C code @@ -14,27 +14,29 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants, re +from . import constants +import re + class CharSetProber: def __init__(self): pass - + def reset(self): self._mState = constants.eDetecting - + def get_charset_name(self): return None @@ -48,13 +50,13 @@ class CharSetProber: return 0.0 def filter_high_bit_only(self, aBuf): - aBuf = re.sub(r'([\x00-\x7F])+', ' ', aBuf) + aBuf = re.sub(b'([\x00-\x7F])+', b' ', aBuf) return aBuf - + def filter_without_english_letters(self, aBuf): - aBuf = re.sub(r'([A-Za-z])+', ' ', aBuf) + aBuf = re.sub(b'([A-Za-z])+', b' ', aBuf) return aBuf - + def filter_with_english_letters(self, aBuf): # TODO return aBuf diff --git a/libs/chardet/codingstatemachine.py b/libs/chardet/codingstatemachine.py index 452d3b0..8dd8c91 100755 --- a/libs/chardet/codingstatemachine.py +++ b/libs/chardet/codingstatemachine.py @@ -13,19 +13,21 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -from constants import eStart, eError, eItsMe +from .constants import eStart +from .compat import wrap_ord + class CodingStateMachine: def __init__(self, sm): @@ -40,12 +42,15 @@ class CodingStateMachine: def next_state(self, c): # for each byte we get its class # if it is first byte, we also get byte length - byteCls = self._mModel['classTable'][ord(c)] + # PY3K: aBuf is a byte stream, so c is an int, not a byte + byteCls = self._mModel['classTable'][wrap_ord(c)] if self._mCurrentState == eStart: self._mCurrentBytePos = 0 self._mCurrentCharLen = self._mModel['charLenTable'][byteCls] # from byte's class and stateTable, we get its next state - self._mCurrentState = self._mModel['stateTable'][self._mCurrentState * self._mModel['classFactor'] + byteCls] + curr_state = (self._mCurrentState * self._mModel['classFactor'] + + byteCls) + self._mCurrentState = self._mModel['stateTable'][curr_state] self._mCurrentBytePos += 1 return self._mCurrentState diff --git a/libs/chardet/compat.py b/libs/chardet/compat.py new file mode 100644 index 0000000..d9e30ad --- /dev/null +++ b/libs/chardet/compat.py @@ -0,0 +1,34 @@ +######################## BEGIN LICENSE BLOCK ######################## +# Contributor(s): +# Ian Cordasco - port to Python +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA +######################### END LICENSE BLOCK ######################### + +import sys + + +if sys.version_info < (3, 0): + base_str = (str, unicode) +else: + base_str = (bytes, str) + + +def wrap_ord(a): + if sys.version_info < (3, 0) and isinstance(a, base_str): + return ord(a) + else: + return a diff --git a/libs/chardet/constants.py b/libs/chardet/constants.py index e94e226..e4d148b 100755 --- a/libs/chardet/constants.py +++ b/libs/chardet/constants.py @@ -37,11 +37,3 @@ eError = 1 eItsMe = 2 SHORTCUT_THRESHOLD = 0.95 - -import __builtin__ -if not hasattr(__builtin__, 'False'): - False = 0 - True = 1 -else: - False = __builtin__.False - True = __builtin__.True diff --git a/libs/chardet/cp949prober.py b/libs/chardet/cp949prober.py new file mode 100644 index 0000000..ff4272f --- /dev/null +++ b/libs/chardet/cp949prober.py @@ -0,0 +1,44 @@ +######################## BEGIN LICENSE BLOCK ######################## +# The Original Code is mozilla.org code. +# +# The Initial Developer of the Original Code is +# Netscape Communications Corporation. +# Portions created by the Initial Developer are Copyright (C) 1998 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Mark Pilgrim - port to Python +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA +######################### END LICENSE BLOCK ######################### + +from .mbcharsetprober import MultiByteCharSetProber +from .codingstatemachine import CodingStateMachine +from .chardistribution import EUCKRDistributionAnalysis +from .mbcssm import CP949SMModel + + +class CP949Prober(MultiByteCharSetProber): + def __init__(self): + MultiByteCharSetProber.__init__(self) + self._mCodingSM = CodingStateMachine(CP949SMModel) + # NOTE: CP949 is a superset of EUC-KR, so the distribution should be + # not different. + self._mDistributionAnalyzer = EUCKRDistributionAnalysis() + self.reset() + + def get_charset_name(self): + return "CP949" diff --git a/libs/chardet/escprober.py b/libs/chardet/escprober.py index 572ed7b..80a844f 100755 --- a/libs/chardet/escprober.py +++ b/libs/chardet/escprober.py @@ -13,39 +13,43 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants, sys -from escsm import HZSMModel, ISO2022CNSMModel, ISO2022JPSMModel, ISO2022KRSMModel -from charsetprober import CharSetProber -from codingstatemachine import CodingStateMachine +from . import constants +from .escsm import (HZSMModel, ISO2022CNSMModel, ISO2022JPSMModel, + ISO2022KRSMModel) +from .charsetprober import CharSetProber +from .codingstatemachine import CodingStateMachine +from .compat import wrap_ord + class EscCharSetProber(CharSetProber): def __init__(self): CharSetProber.__init__(self) - self._mCodingSM = [ \ + self._mCodingSM = [ CodingStateMachine(HZSMModel), CodingStateMachine(ISO2022CNSMModel), CodingStateMachine(ISO2022JPSMModel), CodingStateMachine(ISO2022KRSMModel) - ] + ] self.reset() def reset(self): CharSetProber.reset(self) for codingSM in self._mCodingSM: - if not codingSM: continue - codingSM.active = constants.True + if not codingSM: + continue + codingSM.active = True codingSM.reset() self._mActiveSM = len(self._mCodingSM) self._mDetectedCharset = None @@ -61,19 +65,22 @@ class EscCharSetProber(CharSetProber): def feed(self, aBuf): for c in aBuf: + # PY3K: aBuf is a byte array, so c is an int, not a byte for codingSM in self._mCodingSM: - if not codingSM: continue - if not codingSM.active: continue - codingState = codingSM.next_state(c) + if not codingSM: + continue + if not codingSM.active: + continue + codingState = codingSM.next_state(wrap_ord(c)) if codingState == constants.eError: - codingSM.active = constants.False + codingSM.active = False self._mActiveSM -= 1 if self._mActiveSM <= 0: self._mState = constants.eNotMe return self.get_state() elif codingState == constants.eItsMe: self._mState = constants.eFoundIt - self._mDetectedCharset = codingSM.get_coding_state_machine() + self._mDetectedCharset = codingSM.get_coding_state_machine() # nopep8 return self.get_state() - + return self.get_state() diff --git a/libs/chardet/escsm.py b/libs/chardet/escsm.py index 9fa2295..bd302b4 100755 --- a/libs/chardet/escsm.py +++ b/libs/chardet/escsm.py @@ -13,62 +13,62 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -from constants import eStart, eError, eItsMe - -HZ_cls = ( \ -1,0,0,0,0,0,0,0, # 00 - 07 -0,0,0,0,0,0,0,0, # 08 - 0f -0,0,0,0,0,0,0,0, # 10 - 17 -0,0,0,1,0,0,0,0, # 18 - 1f -0,0,0,0,0,0,0,0, # 20 - 27 -0,0,0,0,0,0,0,0, # 28 - 2f -0,0,0,0,0,0,0,0, # 30 - 37 -0,0,0,0,0,0,0,0, # 38 - 3f -0,0,0,0,0,0,0,0, # 40 - 47 -0,0,0,0,0,0,0,0, # 48 - 4f -0,0,0,0,0,0,0,0, # 50 - 57 -0,0,0,0,0,0,0,0, # 58 - 5f -0,0,0,0,0,0,0,0, # 60 - 67 -0,0,0,0,0,0,0,0, # 68 - 6f -0,0,0,0,0,0,0,0, # 70 - 77 -0,0,0,4,0,5,2,0, # 78 - 7f -1,1,1,1,1,1,1,1, # 80 - 87 -1,1,1,1,1,1,1,1, # 88 - 8f -1,1,1,1,1,1,1,1, # 90 - 97 -1,1,1,1,1,1,1,1, # 98 - 9f -1,1,1,1,1,1,1,1, # a0 - a7 -1,1,1,1,1,1,1,1, # a8 - af -1,1,1,1,1,1,1,1, # b0 - b7 -1,1,1,1,1,1,1,1, # b8 - bf -1,1,1,1,1,1,1,1, # c0 - c7 -1,1,1,1,1,1,1,1, # c8 - cf -1,1,1,1,1,1,1,1, # d0 - d7 -1,1,1,1,1,1,1,1, # d8 - df -1,1,1,1,1,1,1,1, # e0 - e7 -1,1,1,1,1,1,1,1, # e8 - ef -1,1,1,1,1,1,1,1, # f0 - f7 -1,1,1,1,1,1,1,1, # f8 - ff +from .constants import eStart, eError, eItsMe + +HZ_cls = ( +1,0,0,0,0,0,0,0, # 00 - 07 +0,0,0,0,0,0,0,0, # 08 - 0f +0,0,0,0,0,0,0,0, # 10 - 17 +0,0,0,1,0,0,0,0, # 18 - 1f +0,0,0,0,0,0,0,0, # 20 - 27 +0,0,0,0,0,0,0,0, # 28 - 2f +0,0,0,0,0,0,0,0, # 30 - 37 +0,0,0,0,0,0,0,0, # 38 - 3f +0,0,0,0,0,0,0,0, # 40 - 47 +0,0,0,0,0,0,0,0, # 48 - 4f +0,0,0,0,0,0,0,0, # 50 - 57 +0,0,0,0,0,0,0,0, # 58 - 5f +0,0,0,0,0,0,0,0, # 60 - 67 +0,0,0,0,0,0,0,0, # 68 - 6f +0,0,0,0,0,0,0,0, # 70 - 77 +0,0,0,4,0,5,2,0, # 78 - 7f +1,1,1,1,1,1,1,1, # 80 - 87 +1,1,1,1,1,1,1,1, # 88 - 8f +1,1,1,1,1,1,1,1, # 90 - 97 +1,1,1,1,1,1,1,1, # 98 - 9f +1,1,1,1,1,1,1,1, # a0 - a7 +1,1,1,1,1,1,1,1, # a8 - af +1,1,1,1,1,1,1,1, # b0 - b7 +1,1,1,1,1,1,1,1, # b8 - bf +1,1,1,1,1,1,1,1, # c0 - c7 +1,1,1,1,1,1,1,1, # c8 - cf +1,1,1,1,1,1,1,1, # d0 - d7 +1,1,1,1,1,1,1,1, # d8 - df +1,1,1,1,1,1,1,1, # e0 - e7 +1,1,1,1,1,1,1,1, # e8 - ef +1,1,1,1,1,1,1,1, # f0 - f7 +1,1,1,1,1,1,1,1, # f8 - ff ) -HZ_st = ( \ -eStart,eError, 3,eStart,eStart,eStart,eError,eError,# 00-07 -eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 08-0f -eItsMe,eItsMe,eError,eError,eStart,eStart, 4,eError,# 10-17 - 5,eError, 6,eError, 5, 5, 4,eError,# 18-1f - 4,eError, 4, 4, 4,eError, 4,eError,# 20-27 - 4,eItsMe,eStart,eStart,eStart,eStart,eStart,eStart,# 28-2f +HZ_st = ( +eStart,eError, 3,eStart,eStart,eStart,eError,eError,# 00-07 +eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 08-0f +eItsMe,eItsMe,eError,eError,eStart,eStart, 4,eError,# 10-17 + 5,eError, 6,eError, 5, 5, 4,eError,# 18-1f + 4,eError, 4, 4, 4,eError, 4,eError,# 20-27 + 4,eItsMe,eStart,eStart,eStart,eStart,eStart,eStart,# 28-2f ) HZCharLenTable = (0, 0, 0, 0, 0, 0) @@ -79,50 +79,50 @@ HZSMModel = {'classTable': HZ_cls, 'charLenTable': HZCharLenTable, 'name': "HZ-GB-2312"} -ISO2022CN_cls = ( \ -2,0,0,0,0,0,0,0, # 00 - 07 -0,0,0,0,0,0,0,0, # 08 - 0f -0,0,0,0,0,0,0,0, # 10 - 17 -0,0,0,1,0,0,0,0, # 18 - 1f -0,0,0,0,0,0,0,0, # 20 - 27 -0,3,0,0,0,0,0,0, # 28 - 2f -0,0,0,0,0,0,0,0, # 30 - 37 -0,0,0,0,0,0,0,0, # 38 - 3f -0,0,0,4,0,0,0,0, # 40 - 47 -0,0,0,0,0,0,0,0, # 48 - 4f -0,0,0,0,0,0,0,0, # 50 - 57 -0,0,0,0,0,0,0,0, # 58 - 5f -0,0,0,0,0,0,0,0, # 60 - 67 -0,0,0,0,0,0,0,0, # 68 - 6f -0,0,0,0,0,0,0,0, # 70 - 77 -0,0,0,0,0,0,0,0, # 78 - 7f -2,2,2,2,2,2,2,2, # 80 - 87 -2,2,2,2,2,2,2,2, # 88 - 8f -2,2,2,2,2,2,2,2, # 90 - 97 -2,2,2,2,2,2,2,2, # 98 - 9f -2,2,2,2,2,2,2,2, # a0 - a7 -2,2,2,2,2,2,2,2, # a8 - af -2,2,2,2,2,2,2,2, # b0 - b7 -2,2,2,2,2,2,2,2, # b8 - bf -2,2,2,2,2,2,2,2, # c0 - c7 -2,2,2,2,2,2,2,2, # c8 - cf -2,2,2,2,2,2,2,2, # d0 - d7 -2,2,2,2,2,2,2,2, # d8 - df -2,2,2,2,2,2,2,2, # e0 - e7 -2,2,2,2,2,2,2,2, # e8 - ef -2,2,2,2,2,2,2,2, # f0 - f7 -2,2,2,2,2,2,2,2, # f8 - ff +ISO2022CN_cls = ( +2,0,0,0,0,0,0,0, # 00 - 07 +0,0,0,0,0,0,0,0, # 08 - 0f +0,0,0,0,0,0,0,0, # 10 - 17 +0,0,0,1,0,0,0,0, # 18 - 1f +0,0,0,0,0,0,0,0, # 20 - 27 +0,3,0,0,0,0,0,0, # 28 - 2f +0,0,0,0,0,0,0,0, # 30 - 37 +0,0,0,0,0,0,0,0, # 38 - 3f +0,0,0,4,0,0,0,0, # 40 - 47 +0,0,0,0,0,0,0,0, # 48 - 4f +0,0,0,0,0,0,0,0, # 50 - 57 +0,0,0,0,0,0,0,0, # 58 - 5f +0,0,0,0,0,0,0,0, # 60 - 67 +0,0,0,0,0,0,0,0, # 68 - 6f +0,0,0,0,0,0,0,0, # 70 - 77 +0,0,0,0,0,0,0,0, # 78 - 7f +2,2,2,2,2,2,2,2, # 80 - 87 +2,2,2,2,2,2,2,2, # 88 - 8f +2,2,2,2,2,2,2,2, # 90 - 97 +2,2,2,2,2,2,2,2, # 98 - 9f +2,2,2,2,2,2,2,2, # a0 - a7 +2,2,2,2,2,2,2,2, # a8 - af +2,2,2,2,2,2,2,2, # b0 - b7 +2,2,2,2,2,2,2,2, # b8 - bf +2,2,2,2,2,2,2,2, # c0 - c7 +2,2,2,2,2,2,2,2, # c8 - cf +2,2,2,2,2,2,2,2, # d0 - d7 +2,2,2,2,2,2,2,2, # d8 - df +2,2,2,2,2,2,2,2, # e0 - e7 +2,2,2,2,2,2,2,2, # e8 - ef +2,2,2,2,2,2,2,2, # f0 - f7 +2,2,2,2,2,2,2,2, # f8 - ff ) -ISO2022CN_st = ( \ -eStart, 3,eError,eStart,eStart,eStart,eStart,eStart,# 00-07 -eStart,eError,eError,eError,eError,eError,eError,eError,# 08-0f -eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,# 10-17 -eItsMe,eItsMe,eItsMe,eError,eError,eError, 4,eError,# 18-1f -eError,eError,eError,eItsMe,eError,eError,eError,eError,# 20-27 - 5, 6,eError,eError,eError,eError,eError,eError,# 28-2f -eError,eError,eError,eItsMe,eError,eError,eError,eError,# 30-37 -eError,eError,eError,eError,eError,eItsMe,eError,eStart,# 38-3f +ISO2022CN_st = ( +eStart, 3,eError,eStart,eStart,eStart,eStart,eStart,# 00-07 +eStart,eError,eError,eError,eError,eError,eError,eError,# 08-0f +eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,# 10-17 +eItsMe,eItsMe,eItsMe,eError,eError,eError, 4,eError,# 18-1f +eError,eError,eError,eItsMe,eError,eError,eError,eError,# 20-27 + 5, 6,eError,eError,eError,eError,eError,eError,# 28-2f +eError,eError,eError,eItsMe,eError,eError,eError,eError,# 30-37 +eError,eError,eError,eError,eError,eItsMe,eError,eStart,# 38-3f ) ISO2022CNCharLenTable = (0, 0, 0, 0, 0, 0, 0, 0, 0) @@ -133,51 +133,51 @@ ISO2022CNSMModel = {'classTable': ISO2022CN_cls, 'charLenTable': ISO2022CNCharLenTable, 'name': "ISO-2022-CN"} -ISO2022JP_cls = ( \ -2,0,0,0,0,0,0,0, # 00 - 07 -0,0,0,0,0,0,2,2, # 08 - 0f -0,0,0,0,0,0,0,0, # 10 - 17 -0,0,0,1,0,0,0,0, # 18 - 1f -0,0,0,0,7,0,0,0, # 20 - 27 -3,0,0,0,0,0,0,0, # 28 - 2f -0,0,0,0,0,0,0,0, # 30 - 37 -0,0,0,0,0,0,0,0, # 38 - 3f -6,0,4,0,8,0,0,0, # 40 - 47 -0,9,5,0,0,0,0,0, # 48 - 4f -0,0,0,0,0,0,0,0, # 50 - 57 -0,0,0,0,0,0,0,0, # 58 - 5f -0,0,0,0,0,0,0,0, # 60 - 67 -0,0,0,0,0,0,0,0, # 68 - 6f -0,0,0,0,0,0,0,0, # 70 - 77 -0,0,0,0,0,0,0,0, # 78 - 7f -2,2,2,2,2,2,2,2, # 80 - 87 -2,2,2,2,2,2,2,2, # 88 - 8f -2,2,2,2,2,2,2,2, # 90 - 97 -2,2,2,2,2,2,2,2, # 98 - 9f -2,2,2,2,2,2,2,2, # a0 - a7 -2,2,2,2,2,2,2,2, # a8 - af -2,2,2,2,2,2,2,2, # b0 - b7 -2,2,2,2,2,2,2,2, # b8 - bf -2,2,2,2,2,2,2,2, # c0 - c7 -2,2,2,2,2,2,2,2, # c8 - cf -2,2,2,2,2,2,2,2, # d0 - d7 -2,2,2,2,2,2,2,2, # d8 - df -2,2,2,2,2,2,2,2, # e0 - e7 -2,2,2,2,2,2,2,2, # e8 - ef -2,2,2,2,2,2,2,2, # f0 - f7 -2,2,2,2,2,2,2,2, # f8 - ff +ISO2022JP_cls = ( +2,0,0,0,0,0,0,0, # 00 - 07 +0,0,0,0,0,0,2,2, # 08 - 0f +0,0,0,0,0,0,0,0, # 10 - 17 +0,0,0,1,0,0,0,0, # 18 - 1f +0,0,0,0,7,0,0,0, # 20 - 27 +3,0,0,0,0,0,0,0, # 28 - 2f +0,0,0,0,0,0,0,0, # 30 - 37 +0,0,0,0,0,0,0,0, # 38 - 3f +6,0,4,0,8,0,0,0, # 40 - 47 +0,9,5,0,0,0,0,0, # 48 - 4f +0,0,0,0,0,0,0,0, # 50 - 57 +0,0,0,0,0,0,0,0, # 58 - 5f +0,0,0,0,0,0,0,0, # 60 - 67 +0,0,0,0,0,0,0,0, # 68 - 6f +0,0,0,0,0,0,0,0, # 70 - 77 +0,0,0,0,0,0,0,0, # 78 - 7f +2,2,2,2,2,2,2,2, # 80 - 87 +2,2,2,2,2,2,2,2, # 88 - 8f +2,2,2,2,2,2,2,2, # 90 - 97 +2,2,2,2,2,2,2,2, # 98 - 9f +2,2,2,2,2,2,2,2, # a0 - a7 +2,2,2,2,2,2,2,2, # a8 - af +2,2,2,2,2,2,2,2, # b0 - b7 +2,2,2,2,2,2,2,2, # b8 - bf +2,2,2,2,2,2,2,2, # c0 - c7 +2,2,2,2,2,2,2,2, # c8 - cf +2,2,2,2,2,2,2,2, # d0 - d7 +2,2,2,2,2,2,2,2, # d8 - df +2,2,2,2,2,2,2,2, # e0 - e7 +2,2,2,2,2,2,2,2, # e8 - ef +2,2,2,2,2,2,2,2, # f0 - f7 +2,2,2,2,2,2,2,2, # f8 - ff ) -ISO2022JP_st = ( \ -eStart, 3,eError,eStart,eStart,eStart,eStart,eStart,# 00-07 -eStart,eStart,eError,eError,eError,eError,eError,eError,# 08-0f -eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 10-17 -eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,# 18-1f -eError, 5,eError,eError,eError, 4,eError,eError,# 20-27 -eError,eError,eError, 6,eItsMe,eError,eItsMe,eError,# 28-2f -eError,eError,eError,eError,eError,eError,eItsMe,eItsMe,# 30-37 -eError,eError,eError,eItsMe,eError,eError,eError,eError,# 38-3f -eError,eError,eError,eError,eItsMe,eError,eStart,eStart,# 40-47 +ISO2022JP_st = ( +eStart, 3,eError,eStart,eStart,eStart,eStart,eStart,# 00-07 +eStart,eStart,eError,eError,eError,eError,eError,eError,# 08-0f +eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 10-17 +eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,# 18-1f +eError, 5,eError,eError,eError, 4,eError,eError,# 20-27 +eError,eError,eError, 6,eItsMe,eError,eItsMe,eError,# 28-2f +eError,eError,eError,eError,eError,eError,eItsMe,eItsMe,# 30-37 +eError,eError,eError,eItsMe,eError,eError,eError,eError,# 38-3f +eError,eError,eError,eError,eItsMe,eError,eStart,eStart,# 40-47 ) ISO2022JPCharLenTable = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0) @@ -188,47 +188,47 @@ ISO2022JPSMModel = {'classTable': ISO2022JP_cls, 'charLenTable': ISO2022JPCharLenTable, 'name': "ISO-2022-JP"} -ISO2022KR_cls = ( \ -2,0,0,0,0,0,0,0, # 00 - 07 -0,0,0,0,0,0,0,0, # 08 - 0f -0,0,0,0,0,0,0,0, # 10 - 17 -0,0,0,1,0,0,0,0, # 18 - 1f -0,0,0,0,3,0,0,0, # 20 - 27 -0,4,0,0,0,0,0,0, # 28 - 2f -0,0,0,0,0,0,0,0, # 30 - 37 -0,0,0,0,0,0,0,0, # 38 - 3f -0,0,0,5,0,0,0,0, # 40 - 47 -0,0,0,0,0,0,0,0, # 48 - 4f -0,0,0,0,0,0,0,0, # 50 - 57 -0,0,0,0,0,0,0,0, # 58 - 5f -0,0,0,0,0,0,0,0, # 60 - 67 -0,0,0,0,0,0,0,0, # 68 - 6f -0,0,0,0,0,0,0,0, # 70 - 77 -0,0,0,0,0,0,0,0, # 78 - 7f -2,2,2,2,2,2,2,2, # 80 - 87 -2,2,2,2,2,2,2,2, # 88 - 8f -2,2,2,2,2,2,2,2, # 90 - 97 -2,2,2,2,2,2,2,2, # 98 - 9f -2,2,2,2,2,2,2,2, # a0 - a7 -2,2,2,2,2,2,2,2, # a8 - af -2,2,2,2,2,2,2,2, # b0 - b7 -2,2,2,2,2,2,2,2, # b8 - bf -2,2,2,2,2,2,2,2, # c0 - c7 -2,2,2,2,2,2,2,2, # c8 - cf -2,2,2,2,2,2,2,2, # d0 - d7 -2,2,2,2,2,2,2,2, # d8 - df -2,2,2,2,2,2,2,2, # e0 - e7 -2,2,2,2,2,2,2,2, # e8 - ef -2,2,2,2,2,2,2,2, # f0 - f7 -2,2,2,2,2,2,2,2, # f8 - ff +ISO2022KR_cls = ( +2,0,0,0,0,0,0,0, # 00 - 07 +0,0,0,0,0,0,0,0, # 08 - 0f +0,0,0,0,0,0,0,0, # 10 - 17 +0,0,0,1,0,0,0,0, # 18 - 1f +0,0,0,0,3,0,0,0, # 20 - 27 +0,4,0,0,0,0,0,0, # 28 - 2f +0,0,0,0,0,0,0,0, # 30 - 37 +0,0,0,0,0,0,0,0, # 38 - 3f +0,0,0,5,0,0,0,0, # 40 - 47 +0,0,0,0,0,0,0,0, # 48 - 4f +0,0,0,0,0,0,0,0, # 50 - 57 +0,0,0,0,0,0,0,0, # 58 - 5f +0,0,0,0,0,0,0,0, # 60 - 67 +0,0,0,0,0,0,0,0, # 68 - 6f +0,0,0,0,0,0,0,0, # 70 - 77 +0,0,0,0,0,0,0,0, # 78 - 7f +2,2,2,2,2,2,2,2, # 80 - 87 +2,2,2,2,2,2,2,2, # 88 - 8f +2,2,2,2,2,2,2,2, # 90 - 97 +2,2,2,2,2,2,2,2, # 98 - 9f +2,2,2,2,2,2,2,2, # a0 - a7 +2,2,2,2,2,2,2,2, # a8 - af +2,2,2,2,2,2,2,2, # b0 - b7 +2,2,2,2,2,2,2,2, # b8 - bf +2,2,2,2,2,2,2,2, # c0 - c7 +2,2,2,2,2,2,2,2, # c8 - cf +2,2,2,2,2,2,2,2, # d0 - d7 +2,2,2,2,2,2,2,2, # d8 - df +2,2,2,2,2,2,2,2, # e0 - e7 +2,2,2,2,2,2,2,2, # e8 - ef +2,2,2,2,2,2,2,2, # f0 - f7 +2,2,2,2,2,2,2,2, # f8 - ff ) -ISO2022KR_st = ( \ -eStart, 3,eError,eStart,eStart,eStart,eError,eError,# 00-07 -eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 08-0f -eItsMe,eItsMe,eError,eError,eError, 4,eError,eError,# 10-17 -eError,eError,eError,eError, 5,eError,eError,eError,# 18-1f -eError,eError,eError,eItsMe,eStart,eStart,eStart,eStart,# 20-27 +ISO2022KR_st = ( +eStart, 3,eError,eStart,eStart,eStart,eError,eError,# 00-07 +eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 08-0f +eItsMe,eItsMe,eError,eError,eError, 4,eError,eError,# 10-17 +eError,eError,eError,eError, 5,eError,eError,eError,# 18-1f +eError,eError,eError,eItsMe,eStart,eStart,eStart,eStart,# 20-27 ) ISO2022KRCharLenTable = (0, 0, 0, 0, 0, 0) @@ -238,3 +238,5 @@ ISO2022KRSMModel = {'classTable': ISO2022KR_cls, 'stateTable': ISO2022KR_st, 'charLenTable': ISO2022KRCharLenTable, 'name': "ISO-2022-KR"} + +# flake8: noqa diff --git a/libs/chardet/eucjpprober.py b/libs/chardet/eucjpprober.py index 46a8b38..8e64fdc 100755 --- a/libs/chardet/eucjpprober.py +++ b/libs/chardet/eucjpprober.py @@ -13,25 +13,26 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants, sys -from constants import eStart, eError, eItsMe -from mbcharsetprober import MultiByteCharSetProber -from codingstatemachine import CodingStateMachine -from chardistribution import EUCJPDistributionAnalysis -from jpcntx import EUCJPContextAnalysis -from mbcssm import EUCJPSMModel +import sys +from . import constants +from .mbcharsetprober import MultiByteCharSetProber +from .codingstatemachine import CodingStateMachine +from .chardistribution import EUCJPDistributionAnalysis +from .jpcntx import EUCJPContextAnalysis +from .mbcssm import EUCJPSMModel + class EUCJPProber(MultiByteCharSetProber): def __init__(self): @@ -44,37 +45,41 @@ class EUCJPProber(MultiByteCharSetProber): def reset(self): MultiByteCharSetProber.reset(self) self._mContextAnalyzer.reset() - + def get_charset_name(self): return "EUC-JP" def feed(self, aBuf): aLen = len(aBuf) for i in range(0, aLen): + # PY3K: aBuf is a byte array, so aBuf[i] is an int, not a byte codingState = self._mCodingSM.next_state(aBuf[i]) - if codingState == eError: + if codingState == constants.eError: if constants._debug: - sys.stderr.write(self.get_charset_name() + ' prober hit error at byte ' + str(i) + '\n') + sys.stderr.write(self.get_charset_name() + + ' prober hit error at byte ' + str(i) + + '\n') self._mState = constants.eNotMe break - elif codingState == eItsMe: + elif codingState == constants.eItsMe: self._mState = constants.eFoundIt break - elif codingState == eStart: + elif codingState == constants.eStart: charLen = self._mCodingSM.get_current_charlen() if i == 0: self._mLastChar[1] = aBuf[0] self._mContextAnalyzer.feed(self._mLastChar, charLen) self._mDistributionAnalyzer.feed(self._mLastChar, charLen) else: - self._mContextAnalyzer.feed(aBuf[i-1:i+1], charLen) - self._mDistributionAnalyzer.feed(aBuf[i-1:i+1], charLen) - + self._mContextAnalyzer.feed(aBuf[i - 1:i + 1], charLen) + self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], + charLen) + self._mLastChar[0] = aBuf[aLen - 1] - + if self.get_state() == constants.eDetecting: - if self._mContextAnalyzer.got_enough_data() and \ - (self.get_confidence() > constants.SHORTCUT_THRESHOLD): + if (self._mContextAnalyzer.got_enough_data() and + (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): self._mState = constants.eFoundIt return self.get_state() diff --git a/libs/chardet/euckrfreq.py b/libs/chardet/euckrfreq.py index 1463fa1..a179e4c 100755 --- a/libs/chardet/euckrfreq.py +++ b/libs/chardet/euckrfreq.py @@ -592,3 +592,5 @@ EUCKRCharToFreqOrder = ( \ 8704,8705,8706,8707,8708,8709,8710,8711,8712,8713,8714,8715,8716,8717,8718,8719, 8720,8721,8722,8723,8724,8725,8726,8727,8728,8729,8730,8731,8732,8733,8734,8735, 8736,8737,8738,8739,8740,8741) + +# flake8: noqa diff --git a/libs/chardet/euckrprober.py b/libs/chardet/euckrprober.py index bd697eb..5982a46 100755 --- a/libs/chardet/euckrprober.py +++ b/libs/chardet/euckrprober.py @@ -13,22 +13,23 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -from mbcharsetprober import MultiByteCharSetProber -from codingstatemachine import CodingStateMachine -from chardistribution import EUCKRDistributionAnalysis -from mbcssm import EUCKRSMModel +from .mbcharsetprober import MultiByteCharSetProber +from .codingstatemachine import CodingStateMachine +from .chardistribution import EUCKRDistributionAnalysis +from .mbcssm import EUCKRSMModel + class EUCKRProber(MultiByteCharSetProber): def __init__(self): diff --git a/libs/chardet/euctwfreq.py b/libs/chardet/euctwfreq.py index c057209..576e750 100755 --- a/libs/chardet/euctwfreq.py +++ b/libs/chardet/euctwfreq.py @@ -13,12 +13,12 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA @@ -26,8 +26,8 @@ ######################### END LICENSE BLOCK ######################### # EUCTW frequency table -# Converted from big5 work -# by Taiwan's Mandarin Promotion Council +# Converted from big5 work +# by Taiwan's Mandarin Promotion Council # # 128 --> 0.42261 @@ -38,15 +38,15 @@ # # Idea Distribution Ratio = 0.74851/(1-0.74851) =2.98 # Random Distribution Ration = 512/(5401-512)=0.105 -# +# # Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR EUCTW_TYPICAL_DISTRIBUTION_RATIO = 0.75 -# Char to FreqOrder table , +# Char to FreqOrder table , EUCTW_TABLE_SIZE = 8102 -EUCTWCharToFreqOrder = ( \ +EUCTWCharToFreqOrder = ( 1,1800,1506, 255,1431, 198, 9, 82, 6,7310, 177, 202,3615,1256,2808, 110, # 2742 3735, 33,3241, 261, 76, 44,2113, 16,2931,2184,1176, 659,3868, 26,3404,2643, # 2758 1198,3869,3313,4060, 410,2211, 302, 590, 361,1963, 8, 204, 58,4296,7311,1931, # 2774 @@ -424,3 +424,5 @@ EUCTWCharToFreqOrder = ( \ 8694,8695,8696,8697,8698,8699,8700,8701,8702,8703,8704,8705,8706,8707,8708,8709, # 8710 8710,8711,8712,8713,8714,8715,8716,8717,8718,8719,8720,8721,8722,8723,8724,8725, # 8726 8726,8727,8728,8729,8730,8731,8732,8733,8734,8735,8736,8737,8738,8739,8740,8741) # 8742 + +# flake8: noqa diff --git a/libs/chardet/euctwprober.py b/libs/chardet/euctwprober.py index b073f13..fe652fe 100755 --- a/libs/chardet/euctwprober.py +++ b/libs/chardet/euctwprober.py @@ -25,10 +25,10 @@ # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -from mbcharsetprober import MultiByteCharSetProber -from codingstatemachine import CodingStateMachine -from chardistribution import EUCTWDistributionAnalysis -from mbcssm import EUCTWSMModel +from .mbcharsetprober import MultiByteCharSetProber +from .codingstatemachine import CodingStateMachine +from .chardistribution import EUCTWDistributionAnalysis +from .mbcssm import EUCTWSMModel class EUCTWProber(MultiByteCharSetProber): def __init__(self): diff --git a/libs/chardet/gb2312freq.py b/libs/chardet/gb2312freq.py index 7a4d5a1..1238f51 100755 --- a/libs/chardet/gb2312freq.py +++ b/libs/chardet/gb2312freq.py @@ -13,12 +13,12 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA @@ -36,14 +36,14 @@ # # Ideal Distribution Ratio = 0.79135/(1-0.79135) = 3.79 # Random Distribution Ration = 512 / (3755 - 512) = 0.157 -# +# # Typical Distribution Ratio about 25% of Ideal one, still much higher that RDR GB2312_TYPICAL_DISTRIBUTION_RATIO = 0.9 GB2312_TABLE_SIZE = 3760 -GB2312CharToFreqOrder = ( \ +GB2312CharToFreqOrder = ( 1671, 749,1443,2364,3924,3807,2330,3921,1704,3463,2691,1511,1515, 572,3191,2205, 2361, 224,2558, 479,1711, 963,3162, 440,4060,1905,2966,2947,3580,2647,3961,3842, 2204, 869,4207, 970,2678,5626,2944,2956,1479,4048, 514,3595, 588,1346,2820,3409, @@ -469,3 +469,4 @@ GB2312CharToFreqOrder = ( \ 5867,5507,6273,4206,6274,4789,6098,6764,3619,3646,3833,3804,2394,3788,4936,3978, 4866,4899,6099,6100,5559,6478,6765,3599,5868,6101,5869,5870,6275,6766,4527,6767) +# flake8: noqa diff --git a/libs/chardet/gb2312prober.py b/libs/chardet/gb2312prober.py index 91eb392..0325a2d 100755 --- a/libs/chardet/gb2312prober.py +++ b/libs/chardet/gb2312prober.py @@ -25,10 +25,10 @@ # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -from mbcharsetprober import MultiByteCharSetProber -from codingstatemachine import CodingStateMachine -from chardistribution import GB2312DistributionAnalysis -from mbcssm import GB2312SMModel +from .mbcharsetprober import MultiByteCharSetProber +from .codingstatemachine import CodingStateMachine +from .chardistribution import GB2312DistributionAnalysis +from .mbcssm import GB2312SMModel class GB2312Prober(MultiByteCharSetProber): def __init__(self): diff --git a/libs/chardet/hebrewprober.py b/libs/chardet/hebrewprober.py index a2b1eaa..ba225c5 100755 --- a/libs/chardet/hebrewprober.py +++ b/libs/chardet/hebrewprober.py @@ -13,20 +13,21 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -from charsetprober import CharSetProber -import constants +from .charsetprober import CharSetProber +from .constants import eNotMe, eDetecting +from .compat import wrap_ord # This prober doesn't actually recognize a language or a charset. # It is a helper prober for the use of the Hebrew model probers @@ -35,40 +36,40 @@ import constants # # Four main charsets exist in Hebrew: # "ISO-8859-8" - Visual Hebrew -# "windows-1255" - Logical Hebrew +# "windows-1255" - Logical Hebrew # "ISO-8859-8-I" - Logical Hebrew # "x-mac-hebrew" - ?? Logical Hebrew ?? # # Both "ISO" charsets use a completely identical set of code points, whereas -# "windows-1255" and "x-mac-hebrew" are two different proper supersets of +# "windows-1255" and "x-mac-hebrew" are two different proper supersets of # these code points. windows-1255 defines additional characters in the range -# 0x80-0x9F as some misc punctuation marks as well as some Hebrew-specific +# 0x80-0x9F as some misc punctuation marks as well as some Hebrew-specific # diacritics and additional 'Yiddish' ligature letters in the range 0xc0-0xd6. -# x-mac-hebrew defines similar additional code points but with a different +# x-mac-hebrew defines similar additional code points but with a different # mapping. # -# As far as an average Hebrew text with no diacritics is concerned, all four -# charsets are identical with respect to code points. Meaning that for the -# main Hebrew alphabet, all four map the same values to all 27 Hebrew letters +# As far as an average Hebrew text with no diacritics is concerned, all four +# charsets are identical with respect to code points. Meaning that for the +# main Hebrew alphabet, all four map the same values to all 27 Hebrew letters # (including final letters). # # The dominant difference between these charsets is their directionality. # "Visual" directionality means that the text is ordered as if the renderer is -# not aware of a BIDI rendering algorithm. The renderer sees the text and -# draws it from left to right. The text itself when ordered naturally is read +# not aware of a BIDI rendering algorithm. The renderer sees the text and +# draws it from left to right. The text itself when ordered naturally is read # backwards. A buffer of Visual Hebrew generally looks like so: # "[last word of first line spelled backwards] [whole line ordered backwards -# and spelled backwards] [first word of first line spelled backwards] +# and spelled backwards] [first word of first line spelled backwards] # [end of line] [last word of second line] ... etc' " # adding punctuation marks, numbers and English text to visual text is # naturally also "visual" and from left to right. -# +# # "Logical" directionality means the text is ordered "naturally" according to -# the order it is read. It is the responsibility of the renderer to display -# the text from right to left. A BIDI algorithm is used to place general +# the order it is read. It is the responsibility of the renderer to display +# the text from right to left. A BIDI algorithm is used to place general # punctuation marks, numbers and English text in the text. # -# Texts in x-mac-hebrew are almost impossible to find on the Internet. From +# Texts in x-mac-hebrew are almost impossible to find on the Internet. From # what little evidence I could find, it seems that its general directionality # is Logical. # @@ -76,17 +77,17 @@ import constants # charsets: # Visual Hebrew - "ISO-8859-8" - backwards text - Words and sentences are # backwards while line order is natural. For charset recognition purposes -# the line order is unimportant (In fact, for this implementation, even +# the line order is unimportant (In fact, for this implementation, even # word order is unimportant). # Logical Hebrew - "windows-1255" - normal, naturally ordered text. # -# "ISO-8859-8-I" is a subset of windows-1255 and doesn't need to be +# "ISO-8859-8-I" is a subset of windows-1255 and doesn't need to be # specifically identified. # "x-mac-hebrew" is also identified as windows-1255. A text in x-mac-hebrew # that contain special punctuation marks or diacritics is displayed with # some unconverted characters showing as question marks. This problem might # be corrected using another model prober for x-mac-hebrew. Due to the fact -# that x-mac-hebrew texts are so rare, writing another model prober isn't +# that x-mac-hebrew texts are so rare, writing another model prober isn't # worth the effort and performance hit. # #### The Prober #### @@ -126,28 +127,31 @@ import constants # charset identified, either "windows-1255" or "ISO-8859-8". # windows-1255 / ISO-8859-8 code points of interest -FINAL_KAF = '\xea' -NORMAL_KAF = '\xeb' -FINAL_MEM = '\xed' -NORMAL_MEM = '\xee' -FINAL_NUN = '\xef' -NORMAL_NUN = '\xf0' -FINAL_PE = '\xf3' -NORMAL_PE = '\xf4' -FINAL_TSADI = '\xf5' -NORMAL_TSADI = '\xf6' +FINAL_KAF = 0xea +NORMAL_KAF = 0xeb +FINAL_MEM = 0xed +NORMAL_MEM = 0xee +FINAL_NUN = 0xef +NORMAL_NUN = 0xf0 +FINAL_PE = 0xf3 +NORMAL_PE = 0xf4 +FINAL_TSADI = 0xf5 +NORMAL_TSADI = 0xf6 # Minimum Visual vs Logical final letter score difference. -# If the difference is below this, don't rely solely on the final letter score distance. +# If the difference is below this, don't rely solely on the final letter score +# distance. MIN_FINAL_CHAR_DISTANCE = 5 # Minimum Visual vs Logical model score difference. -# If the difference is below this, don't rely at all on the model score distance. +# If the difference is below this, don't rely at all on the model score +# distance. MIN_MODEL_DISTANCE = 0.01 VISUAL_HEBREW_NAME = "ISO-8859-8" LOGICAL_HEBREW_NAME = "windows-1255" + class HebrewProber(CharSetProber): def __init__(self): CharSetProber.__init__(self) @@ -159,84 +163,91 @@ class HebrewProber(CharSetProber): self._mFinalCharLogicalScore = 0 self._mFinalCharVisualScore = 0 # The two last characters seen in the previous buffer, - # mPrev and mBeforePrev are initialized to space in order to simulate a word - # delimiter at the beginning of the data + # mPrev and mBeforePrev are initialized to space in order to simulate + # a word delimiter at the beginning of the data self._mPrev = ' ' self._mBeforePrev = ' ' # These probers are owned by the group prober. - + def set_model_probers(self, logicalProber, visualProber): self._mLogicalProber = logicalProber self._mVisualProber = visualProber def is_final(self, c): - return c in [FINAL_KAF, FINAL_MEM, FINAL_NUN, FINAL_PE, FINAL_TSADI] + return wrap_ord(c) in [FINAL_KAF, FINAL_MEM, FINAL_NUN, FINAL_PE, + FINAL_TSADI] def is_non_final(self, c): - # The normal Tsadi is not a good Non-Final letter due to words like - # 'lechotet' (to chat) containing an apostrophe after the tsadi. This - # apostrophe is converted to a space in FilterWithoutEnglishLetters causing - # the Non-Final tsadi to appear at an end of a word even though this is not - # the case in the original text. - # The letters Pe and Kaf rarely display a related behavior of not being a - # good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak' for - # example legally end with a Non-Final Pe or Kaf. However, the benefit of - # these letters as Non-Final letters outweighs the damage since these words - # are quite rare. - return c in [NORMAL_KAF, NORMAL_MEM, NORMAL_NUN, NORMAL_PE] - + # The normal Tsadi is not a good Non-Final letter due to words like + # 'lechotet' (to chat) containing an apostrophe after the tsadi. This + # apostrophe is converted to a space in FilterWithoutEnglishLetters + # causing the Non-Final tsadi to appear at an end of a word even + # though this is not the case in the original text. + # The letters Pe and Kaf rarely display a related behavior of not being + # a good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak' + # for example legally end with a Non-Final Pe or Kaf. However, the + # benefit of these letters as Non-Final letters outweighs the damage + # since these words are quite rare. + return wrap_ord(c) in [NORMAL_KAF, NORMAL_MEM, NORMAL_NUN, NORMAL_PE] + def feed(self, aBuf): # Final letter analysis for logical-visual decision. - # Look for evidence that the received buffer is either logical Hebrew or - # visual Hebrew. + # Look for evidence that the received buffer is either logical Hebrew + # or visual Hebrew. # The following cases are checked: - # 1) A word longer than 1 letter, ending with a final letter. This is an - # indication that the text is laid out "naturally" since the final letter - # really appears at the end. +1 for logical score. - # 2) A word longer than 1 letter, ending with a Non-Final letter. In normal - # Hebrew, words ending with Kaf, Mem, Nun, Pe or Tsadi, should not end with - # the Non-Final form of that letter. Exceptions to this rule are mentioned - # above in isNonFinal(). This is an indication that the text is laid out - # backwards. +1 for visual score - # 3) A word longer than 1 letter, starting with a final letter. Final letters - # should not appear at the beginning of a word. This is an indication that - # the text is laid out backwards. +1 for visual score. - # - # The visual score and logical score are accumulated throughout the text and - # are finally checked against each other in GetCharSetName(). - # No checking for final letters in the middle of words is done since that case - # is not an indication for either Logical or Visual text. - # - # We automatically filter out all 7-bit characters (replace them with spaces) - # so the word boundary detection works properly. [MAP] + # 1) A word longer than 1 letter, ending with a final letter. This is + # an indication that the text is laid out "naturally" since the + # final letter really appears at the end. +1 for logical score. + # 2) A word longer than 1 letter, ending with a Non-Final letter. In + # normal Hebrew, words ending with Kaf, Mem, Nun, Pe or Tsadi, + # should not end with the Non-Final form of that letter. Exceptions + # to this rule are mentioned above in isNonFinal(). This is an + # indication that the text is laid out backwards. +1 for visual + # score + # 3) A word longer than 1 letter, starting with a final letter. Final + # letters should not appear at the beginning of a word. This is an + # indication that the text is laid out backwards. +1 for visual + # score. + # + # The visual score and logical score are accumulated throughout the + # text and are finally checked against each other in GetCharSetName(). + # No checking for final letters in the middle of words is done since + # that case is not an indication for either Logical or Visual text. + # + # We automatically filter out all 7-bit characters (replace them with + # spaces) so the word boundary detection works properly. [MAP] - if self.get_state() == constants.eNotMe: + if self.get_state() == eNotMe: # Both model probers say it's not them. No reason to continue. - return constants.eNotMe + return eNotMe aBuf = self.filter_high_bit_only(aBuf) - + for cur in aBuf: if cur == ' ': # We stand on a space - a word just ended if self._mBeforePrev != ' ': - # next-to-last char was not a space so self._mPrev is not a 1 letter word + # next-to-last char was not a space so self._mPrev is not a + # 1 letter word if self.is_final(self._mPrev): # case (1) [-2:not space][-1:final letter][cur:space] self._mFinalCharLogicalScore += 1 elif self.is_non_final(self._mPrev): - # case (2) [-2:not space][-1:Non-Final letter][cur:space] + # case (2) [-2:not space][-1:Non-Final letter][ + # cur:space] self._mFinalCharVisualScore += 1 else: # Not standing on a space - if (self._mBeforePrev == ' ') and (self.is_final(self._mPrev)) and (cur != ' '): + if ((self._mBeforePrev == ' ') and + (self.is_final(self._mPrev)) and (cur != ' ')): # case (3) [-2:space][-1:final letter][cur:not space] self._mFinalCharVisualScore += 1 self._mBeforePrev = self._mPrev self._mPrev = cur - # Forever detecting, till the end or until both model probers return eNotMe (handled above) - return constants.eDetecting + # Forever detecting, till the end or until both model probers return + # eNotMe (handled above) + return eDetecting def get_charset_name(self): # Make the decision: is it Logical or Visual? @@ -248,22 +259,25 @@ class HebrewProber(CharSetProber): return VISUAL_HEBREW_NAME # It's not dominant enough, try to rely on the model scores instead. - modelsub = self._mLogicalProber.get_confidence() - self._mVisualProber.get_confidence() + modelsub = (self._mLogicalProber.get_confidence() + - self._mVisualProber.get_confidence()) if modelsub > MIN_MODEL_DISTANCE: return LOGICAL_HEBREW_NAME if modelsub < -MIN_MODEL_DISTANCE: return VISUAL_HEBREW_NAME - # Still no good, back to final letter distance, maybe it'll save the day. + # Still no good, back to final letter distance, maybe it'll save the + # day. if finalsub < 0.0: return VISUAL_HEBREW_NAME - # (finalsub > 0 - Logical) or (don't know what to do) default to Logical. + # (finalsub > 0 - Logical) or (don't know what to do) default to + # Logical. return LOGICAL_HEBREW_NAME def get_state(self): # Remain active as long as any of the model probers are active. - if (self._mLogicalProber.get_state() == constants.eNotMe) and \ - (self._mVisualProber.get_state() == constants.eNotMe): - return constants.eNotMe - return constants.eDetecting + if (self._mLogicalProber.get_state() == eNotMe) and \ + (self._mVisualProber.get_state() == eNotMe): + return eNotMe + return eDetecting diff --git a/libs/chardet/jisfreq.py b/libs/chardet/jisfreq.py index 5fe4a5c..064345b 100755 --- a/libs/chardet/jisfreq.py +++ b/libs/chardet/jisfreq.py @@ -13,12 +13,12 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA @@ -28,7 +28,7 @@ # Sampling from about 20M text materials include literature and computer technology # # Japanese frequency table, applied to both S-JIS and EUC-JP -# They are sorted in order. +# They are sorted in order. # 128 --> 0.77094 # 256 --> 0.85710 @@ -38,15 +38,15 @@ # # Ideal Distribution Ratio = 0.92635 / (1-0.92635) = 12.58 # Random Distribution Ration = 512 / (2965+62+83+86-512) = 0.191 -# -# Typical Distribution Ratio, 25% of IDR +# +# Typical Distribution Ratio, 25% of IDR JIS_TYPICAL_DISTRIBUTION_RATIO = 3.0 -# Char to FreqOrder table , +# Char to FreqOrder table , JIS_TABLE_SIZE = 4368 -JISCharToFreqOrder = ( \ +JISCharToFreqOrder = ( 40, 1, 6, 182, 152, 180, 295,2127, 285, 381,3295,4304,3068,4606,3165,3510, # 16 3511,1822,2785,4607,1193,2226,5070,4608, 171,2996,1247, 18, 179,5071, 856,1661, # 32 1262,5072, 619, 127,3431,3512,3230,1899,1700, 232, 228,1294,1298, 284, 283,2041, # 48 @@ -565,3 +565,5 @@ JISCharToFreqOrder = ( \ 8224,8225,8226,8227,8228,8229,8230,8231,8232,8233,8234,8235,8236,8237,8238,8239, # 8240 8240,8241,8242,8243,8244,8245,8246,8247,8248,8249,8250,8251,8252,8253,8254,8255, # 8256 8256,8257,8258,8259,8260,8261,8262,8263,8264,8265,8266,8267,8268,8269,8270,8271) # 8272 + +# flake8: noqa diff --git a/libs/chardet/jpcntx.py b/libs/chardet/jpcntx.py index 93db4a9..f7f69ba 100755 --- a/libs/chardet/jpcntx.py +++ b/libs/chardet/jpcntx.py @@ -13,19 +13,19 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants +from .compat import wrap_ord NUM_OF_CATEGORY = 6 DONT_KNOW = -1 @@ -34,7 +34,7 @@ MAX_REL_THRESHOLD = 1000 MINIMUM_DATA_THRESHOLD = 4 # This is hiragana 2-char sequence table, the number in each cell represents its frequency category -jp2CharContext = ( \ +jp2CharContext = ( (0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1), (2,4,0,4,0,3,0,4,0,3,4,4,4,2,4,3,3,4,3,2,3,3,4,2,3,3,3,2,4,1,4,3,3,1,5,4,3,4,3,4,3,5,3,0,3,5,4,2,0,3,1,0,3,3,0,3,3,0,1,1,0,4,3,0,3,3,0,4,0,2,0,3,5,5,5,5,4,0,4,1,0,3,4), (0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2), @@ -123,26 +123,33 @@ jp2CharContext = ( \ class JapaneseContextAnalysis: def __init__(self): self.reset() - + def reset(self): - self._mTotalRel = 0 # total sequence received - self._mRelSample = [0] * NUM_OF_CATEGORY # category counters, each interger counts sequence in its category - self._mNeedToSkipCharNum = 0 # if last byte in current buffer is not the last byte of a character, we need to know how many bytes to skip in next buffer - self._mLastCharOrder = -1 # The order of previous char - self._mDone = constants.False # If this flag is set to constants.True, detection is done and conclusion has been made + self._mTotalRel = 0 # total sequence received + # category counters, each interger counts sequence in its category + self._mRelSample = [0] * NUM_OF_CATEGORY + # if last byte in current buffer is not the last byte of a character, + # we need to know how many bytes to skip in next buffer + self._mNeedToSkipCharNum = 0 + self._mLastCharOrder = -1 # The order of previous char + # If this flag is set to True, detection is done and conclusion has + # been made + self._mDone = False def feed(self, aBuf, aLen): - if self._mDone: return - + if self._mDone: + return + # The buffer we got is byte oriented, and a character may span in more than one - # buffers. In case the last one or two byte in last buffer is not complete, we - # record how many byte needed to complete that character and skip these bytes here. - # We can choose to record those bytes as well and analyse the character once it - # is complete, but since a character will not make much difference, by simply skipping + # buffers. In case the last one or two byte in last buffer is not + # complete, we record how many byte needed to complete that character + # and skip these bytes here. We can choose to record those bytes as + # well and analyse the character once it is complete, but since a + # character will not make much difference, by simply skipping # this character will simply our logic and improve performance. i = self._mNeedToSkipCharNum while i < aLen: - order, charLen = self.get_order(aBuf[i:i+2]) + order, charLen = self.get_order(aBuf[i:i + 2]) i += charLen if i > aLen: self._mNeedToSkipCharNum = i - aLen @@ -151,14 +158,14 @@ class JapaneseContextAnalysis: if (order != -1) and (self._mLastCharOrder != -1): self._mTotalRel += 1 if self._mTotalRel > MAX_REL_THRESHOLD: - self._mDone = constants.True + self._mDone = True break self._mRelSample[jp2CharContext[self._mLastCharOrder][order]] += 1 self._mLastCharOrder = order def got_enough_data(self): return self._mTotalRel > ENOUGH_REL_THRESHOLD - + def get_confidence(self): # This is just one way to calculate confidence. It works well for me. if self._mTotalRel > MINIMUM_DATA_THRESHOLD: @@ -166,45 +173,47 @@ class JapaneseContextAnalysis: else: return DONT_KNOW - def get_order(self, aStr): + def get_order(self, aBuf): return -1, 1 - + class SJISContextAnalysis(JapaneseContextAnalysis): - def get_order(self, aStr): - if not aStr: return -1, 1 + def get_order(self, aBuf): + if not aBuf: + return -1, 1 # find out current char's byte length - if ((aStr[0] >= '\x81') and (aStr[0] <= '\x9F')) or \ - ((aStr[0] >= '\xE0') and (aStr[0] <= '\xFC')): + first_char = wrap_ord(aBuf[0]) + if ((0x81 <= first_char <= 0x9F) or (0xE0 <= first_char <= 0xFC)): charLen = 2 else: charLen = 1 # return its order if it is hiragana - if len(aStr) > 1: - if (aStr[0] == '\202') and \ - (aStr[1] >= '\x9F') and \ - (aStr[1] <= '\xF1'): - return ord(aStr[1]) - 0x9F, charLen + if len(aBuf) > 1: + second_char = wrap_ord(aBuf[1]) + if (first_char == 202) and (0x9F <= second_char <= 0xF1): + return second_char - 0x9F, charLen return -1, charLen class EUCJPContextAnalysis(JapaneseContextAnalysis): - def get_order(self, aStr): - if not aStr: return -1, 1 + def get_order(self, aBuf): + if not aBuf: + return -1, 1 # find out current char's byte length - if (aStr[0] == '\x8E') or \ - ((aStr[0] >= '\xA1') and (aStr[0] <= '\xFE')): + first_char = wrap_ord(aBuf[0]) + if (first_char == 0x8E) or (0xA1 <= first_char <= 0xFE): charLen = 2 - elif aStr[0] == '\x8F': + elif first_char == 0x8F: charLen = 3 else: charLen = 1 # return its order if it is hiragana - if len(aStr) > 1: - if (aStr[0] == '\xA4') and \ - (aStr[1] >= '\xA1') and \ - (aStr[1] <= '\xF3'): - return ord(aStr[1]) - 0xA1, charLen + if len(aBuf) > 1: + second_char = wrap_ord(aBuf[1]) + if (first_char == 0xA4) and (0xA1 <= second_char <= 0xF3): + return second_char - 0xA1, charLen return -1, charLen + +# flake8: noqa diff --git a/libs/chardet/langbulgarianmodel.py b/libs/chardet/langbulgarianmodel.py index bf5641e..e5788fc 100755 --- a/libs/chardet/langbulgarianmodel.py +++ b/libs/chardet/langbulgarianmodel.py @@ -13,30 +13,28 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants - # 255: Control characters that usually does not exist in any text # 254: Carriage/Return # 253: symbol (punctuation) that does not belong to word # 252: 0 - 9 # Character Mapping Table: -# this table is modified base on win1251BulgarianCharToOrderMap, so +# this table is modified base on win1251BulgarianCharToOrderMap, so # only number <64 is sure valid -Latin5_BulgarianCharToOrderMap = ( \ +Latin5_BulgarianCharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -55,7 +53,7 @@ Latin5_BulgarianCharToOrderMap = ( \ 62,242,243,244, 58,245, 98,246,247,248,249,250,251, 91,252,253, # f0 ) -win1251BulgarianCharToOrderMap = ( \ +win1251BulgarianCharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -74,13 +72,13 @@ win1251BulgarianCharToOrderMap = ( \ 7, 8, 5, 19, 29, 25, 22, 21, 27, 24, 17, 75, 52,253, 42, 16, # f0 ) -# Model Table: +# Model Table: # total sequences: 100% # first 512 sequences: 96.9392% # first 1024 sequences:3.0618% # rest sequences: 0.2992% -# negative sequences: 0.0020% -BulgarianLangModel = ( \ +# negative sequences: 0.0020% +BulgarianLangModel = ( 0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,2,3,3,3,3,3, 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,2,2,3,2,2,1,2,2, 3,1,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,3,3,3,3,3,3,3,0,3,0,1, @@ -211,18 +209,21 @@ BulgarianLangModel = ( \ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, ) -Latin5BulgarianModel = { \ +Latin5BulgarianModel = { 'charToOrderMap': Latin5_BulgarianCharToOrderMap, 'precedenceMatrix': BulgarianLangModel, 'mTypicalPositiveRatio': 0.969392, - 'keepEnglishLetter': constants.False, + 'keepEnglishLetter': False, 'charsetName': "ISO-8859-5" } -Win1251BulgarianModel = { \ +Win1251BulgarianModel = { 'charToOrderMap': win1251BulgarianCharToOrderMap, 'precedenceMatrix': BulgarianLangModel, 'mTypicalPositiveRatio': 0.969392, - 'keepEnglishLetter': constants.False, + 'keepEnglishLetter': False, 'charsetName': "windows-1251" } + + +# flake8: noqa diff --git a/libs/chardet/langcyrillicmodel.py b/libs/chardet/langcyrillicmodel.py index e604cc7..a86f54b 100755 --- a/libs/chardet/langcyrillicmodel.py +++ b/libs/chardet/langcyrillicmodel.py @@ -13,23 +13,21 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants - # KOI8-R language model # Character Mapping Table: -KOI8R_CharToOrderMap = ( \ +KOI8R_CharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -48,7 +46,7 @@ KOI8R_CharToOrderMap = ( \ 35, 43, 45, 32, 40, 52, 56, 33, 61, 62, 51, 57, 47, 63, 50, 70, # f0 ) -win1251_CharToOrderMap = ( \ +win1251_CharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -67,7 +65,7 @@ win1251_CharToOrderMap = ( \ 9, 7, 6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27, 16, ) -latin5_CharToOrderMap = ( \ +latin5_CharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -86,7 +84,7 @@ latin5_CharToOrderMap = ( \ 239, 68,240,241,242,243,244,245,246,247,248,249,250,251,252,255, ) -macCyrillic_CharToOrderMap = ( \ +macCyrillic_CharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -105,7 +103,7 @@ macCyrillic_CharToOrderMap = ( \ 9, 7, 6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27,255, ) -IBM855_CharToOrderMap = ( \ +IBM855_CharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -124,7 +122,7 @@ IBM855_CharToOrderMap = ( \ 250, 18, 62, 20, 51, 25, 57, 30, 47, 29, 63, 22, 50,251,252,255, ) -IBM866_CharToOrderMap = ( \ +IBM866_CharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -143,13 +141,13 @@ IBM866_CharToOrderMap = ( \ 239, 68,240,241,242,243,244,245,246,247,248,249,250,251,252,255, ) -# Model Table: +# Model Table: # total sequences: 100% # first 512 sequences: 97.6601% # first 1024 sequences: 2.3389% # rest sequences: 0.1237% -# negative sequences: 0.0009% -RussianLangModel = ( \ +# negative sequences: 0.0009% +RussianLangModel = ( 0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,1,3,3,3,3,1,3,3,3,2,3,2,3,3, 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,2,2,2,2,2,0,0,2, 3,3,3,2,3,3,3,3,3,3,3,3,3,3,2,3,3,0,0,3,3,3,3,3,3,3,3,3,2,3,2,0, @@ -280,50 +278,52 @@ RussianLangModel = ( \ 0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0, ) -Koi8rModel = { \ +Koi8rModel = { 'charToOrderMap': KOI8R_CharToOrderMap, 'precedenceMatrix': RussianLangModel, 'mTypicalPositiveRatio': 0.976601, - 'keepEnglishLetter': constants.False, + 'keepEnglishLetter': False, 'charsetName': "KOI8-R" } -Win1251CyrillicModel = { \ +Win1251CyrillicModel = { 'charToOrderMap': win1251_CharToOrderMap, 'precedenceMatrix': RussianLangModel, 'mTypicalPositiveRatio': 0.976601, - 'keepEnglishLetter': constants.False, + 'keepEnglishLetter': False, 'charsetName': "windows-1251" } -Latin5CyrillicModel = { \ +Latin5CyrillicModel = { 'charToOrderMap': latin5_CharToOrderMap, 'precedenceMatrix': RussianLangModel, 'mTypicalPositiveRatio': 0.976601, - 'keepEnglishLetter': constants.False, + 'keepEnglishLetter': False, 'charsetName': "ISO-8859-5" } -MacCyrillicModel = { \ +MacCyrillicModel = { 'charToOrderMap': macCyrillic_CharToOrderMap, 'precedenceMatrix': RussianLangModel, 'mTypicalPositiveRatio': 0.976601, - 'keepEnglishLetter': constants.False, + 'keepEnglishLetter': False, 'charsetName': "MacCyrillic" }; -Ibm866Model = { \ +Ibm866Model = { 'charToOrderMap': IBM866_CharToOrderMap, 'precedenceMatrix': RussianLangModel, 'mTypicalPositiveRatio': 0.976601, - 'keepEnglishLetter': constants.False, + 'keepEnglishLetter': False, 'charsetName': "IBM866" } -Ibm855Model = { \ +Ibm855Model = { 'charToOrderMap': IBM855_CharToOrderMap, 'precedenceMatrix': RussianLangModel, 'mTypicalPositiveRatio': 0.976601, - 'keepEnglishLetter': constants.False, + 'keepEnglishLetter': False, 'charsetName': "IBM855" } + +# flake8: noqa diff --git a/libs/chardet/langgreekmodel.py b/libs/chardet/langgreekmodel.py index ec6d49e..ddb5837 100755 --- a/libs/chardet/langgreekmodel.py +++ b/libs/chardet/langgreekmodel.py @@ -13,27 +13,25 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants - # 255: Control characters that usually does not exist in any text # 254: Carriage/Return # 253: symbol (punctuation) that does not belong to word # 252: 0 - 9 # Character Mapping Table: -Latin7_CharToOrderMap = ( \ +Latin7_CharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -52,7 +50,7 @@ Latin7_CharToOrderMap = ( \ 9, 8, 14, 7, 2, 12, 28, 23, 42, 24, 64, 75, 19, 26, 27,253, # f0 ) -win1253_CharToOrderMap = ( \ +win1253_CharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -71,13 +69,13 @@ win1253_CharToOrderMap = ( \ 9, 8, 14, 7, 2, 12, 28, 23, 42, 24, 64, 75, 19, 26, 27,253, # f0 ) -# Model Table: +# Model Table: # total sequences: 100% # first 512 sequences: 98.2851% # first 1024 sequences:1.7001% # rest sequences: 0.0359% -# negative sequences: 0.0148% -GreekLangModel = ( \ +# negative sequences: 0.0148% +GreekLangModel = ( 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,3,2,2,3,3,3,3,3,3,3,3,1,3,3,3,0,2,2,3,3,0,3,0,3,2,0,3,3,3,0, @@ -208,18 +206,20 @@ GreekLangModel = ( \ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, ) -Latin7GreekModel = { \ +Latin7GreekModel = { 'charToOrderMap': Latin7_CharToOrderMap, 'precedenceMatrix': GreekLangModel, 'mTypicalPositiveRatio': 0.982851, - 'keepEnglishLetter': constants.False, + 'keepEnglishLetter': False, 'charsetName': "ISO-8859-7" } -Win1253GreekModel = { \ +Win1253GreekModel = { 'charToOrderMap': win1253_CharToOrderMap, 'precedenceMatrix': GreekLangModel, 'mTypicalPositiveRatio': 0.982851, - 'keepEnglishLetter': constants.False, + 'keepEnglishLetter': False, 'charsetName': "windows-1253" } + +# flake8: noqa diff --git a/libs/chardet/langhebrewmodel.py b/libs/chardet/langhebrewmodel.py index a8bcc65..75f2bc7 100755 --- a/libs/chardet/langhebrewmodel.py +++ b/libs/chardet/langhebrewmodel.py @@ -15,20 +15,18 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants - # 255: Control characters that usually does not exist in any text # 254: Carriage/Return # 253: symbol (punctuation) that does not belong to word @@ -36,7 +34,7 @@ import constants # Windows-1255 language model # Character Mapping Table: -win1255_CharToOrderMap = ( \ +win1255_CharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -55,13 +53,13 @@ win1255_CharToOrderMap = ( \ 12, 19, 13, 26, 18, 27, 21, 17, 7, 10, 5,251,252,128, 96,253, ) -# Model Table: +# Model Table: # total sequences: 100% # first 512 sequences: 98.4004% # first 1024 sequences: 1.5981% # rest sequences: 0.087% -# negative sequences: 0.0015% -HebrewLangModel = ( \ +# negative sequences: 0.0015% +HebrewLangModel = ( 0,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,2,3,2,1,2,0,1,0,0, 3,0,3,1,0,0,1,3,2,0,1,1,2,0,2,2,2,1,1,1,1,2,1,1,1,2,0,0,2,2,0,1, 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2, @@ -192,10 +190,12 @@ HebrewLangModel = ( \ 0,0,0,0,0,0,0,0,0,0,1,2,1,0,0,0,0,0,1,1,1,1,1,0,1,0,0,0,1,1,0,0, ) -Win1255HebrewModel = { \ +Win1255HebrewModel = { 'charToOrderMap': win1255_CharToOrderMap, 'precedenceMatrix': HebrewLangModel, 'mTypicalPositiveRatio': 0.984004, - 'keepEnglishLetter': constants.False, + 'keepEnglishLetter': False, 'charsetName': "windows-1255" } + +# flake8: noqa diff --git a/libs/chardet/langhungarianmodel.py b/libs/chardet/langhungarianmodel.py index d635f03..49d2f0f 100755 --- a/libs/chardet/langhungarianmodel.py +++ b/libs/chardet/langhungarianmodel.py @@ -13,27 +13,25 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants - # 255: Control characters that usually does not exist in any text # 254: Carriage/Return # 253: symbol (punctuation) that does not belong to word # 252: 0 - 9 # Character Mapping Table: -Latin2_HungarianCharToOrderMap = ( \ +Latin2_HungarianCharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -52,7 +50,7 @@ Latin2_HungarianCharToOrderMap = ( \ 245,246,247, 25, 73, 42, 24,248,249,250, 31, 56, 29,251,252,253, ) -win1250HungarianCharToOrderMap = ( \ +win1250HungarianCharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -71,13 +69,13 @@ win1250HungarianCharToOrderMap = ( \ 245,246,247, 25, 74, 42, 24,248,249,250, 31, 56, 29,251,252,253, ) -# Model Table: +# Model Table: # total sequences: 100% # first 512 sequences: 94.7368% # first 1024 sequences:5.2623% # rest sequences: 0.8894% -# negative sequences: 0.0009% -HungarianLangModel = ( \ +# negative sequences: 0.0009% +HungarianLangModel = ( 0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,2,2,3,3,1,1,2,2,2,2,2,1,2, 3,2,2,3,3,3,3,3,2,3,3,3,3,3,3,1,2,3,3,3,3,2,3,3,1,1,3,3,0,1,1,1, @@ -208,18 +206,20 @@ HungarianLangModel = ( \ 0,1,1,1,1,1,1,0,1,1,0,1,0,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0, ) -Latin2HungarianModel = { \ +Latin2HungarianModel = { 'charToOrderMap': Latin2_HungarianCharToOrderMap, 'precedenceMatrix': HungarianLangModel, 'mTypicalPositiveRatio': 0.947368, - 'keepEnglishLetter': constants.True, + 'keepEnglishLetter': True, 'charsetName': "ISO-8859-2" } -Win1250HungarianModel = { \ +Win1250HungarianModel = { 'charToOrderMap': win1250HungarianCharToOrderMap, 'precedenceMatrix': HungarianLangModel, 'mTypicalPositiveRatio': 0.947368, - 'keepEnglishLetter': constants.True, + 'keepEnglishLetter': True, 'charsetName': "windows-1250" } + +# flake8: noqa diff --git a/libs/chardet/langthaimodel.py b/libs/chardet/langthaimodel.py index 96ec054..0508b1b 100755 --- a/libs/chardet/langthaimodel.py +++ b/libs/chardet/langthaimodel.py @@ -13,29 +13,27 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants - # 255: Control characters that usually does not exist in any text # 254: Carriage/Return # 253: symbol (punctuation) that does not belong to word # 252: 0 - 9 -# The following result for thai was collected from a limited sample (1M). +# The following result for thai was collected from a limited sample (1M). # Character Mapping Table: -TIS620CharToOrderMap = ( \ +TIS620CharToOrderMap = ( 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20 @@ -54,13 +52,13 @@ TIS620CharToOrderMap = ( \ 68, 56, 59, 65, 69, 60, 70, 80, 71, 87,248,249,250,251,252,253, ) -# Model Table: +# Model Table: # total sequences: 100% # first 512 sequences: 92.6386% # first 1024 sequences:7.3177% # rest sequences: 1.0230% -# negative sequences: 0.0436% -ThaiLangModel = ( \ +# negative sequences: 0.0436% +ThaiLangModel = ( 0,1,3,3,3,3,0,0,3,3,0,3,3,0,3,3,3,3,3,3,3,3,0,0,3,3,3,0,3,3,3,3, 0,3,3,0,0,0,1,3,0,3,3,2,3,3,0,1,2,3,3,3,3,0,2,0,2,0,0,3,2,1,2,2, 3,0,3,3,2,3,0,0,3,3,0,3,3,0,3,3,3,3,3,3,3,3,3,0,3,2,3,0,2,2,2,3, @@ -191,10 +189,12 @@ ThaiLangModel = ( \ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, ) -TIS620ThaiModel = { \ +TIS620ThaiModel = { 'charToOrderMap': TIS620CharToOrderMap, 'precedenceMatrix': ThaiLangModel, 'mTypicalPositiveRatio': 0.926386, - 'keepEnglishLetter': constants.False, + 'keepEnglishLetter': False, 'charsetName': "TIS-620" } + +# flake8: noqa diff --git a/libs/chardet/latin1prober.py b/libs/chardet/latin1prober.py index b46129b..ad695f5 100755 --- a/libs/chardet/latin1prober.py +++ b/libs/chardet/latin1prober.py @@ -14,85 +14,86 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -from charsetprober import CharSetProber -import constants -import operator +from .charsetprober import CharSetProber +from .constants import eNotMe +from .compat import wrap_ord FREQ_CAT_NUM = 4 -UDF = 0 # undefined -OTH = 1 # other -ASC = 2 # ascii capital letter -ASS = 3 # ascii small letter -ACV = 4 # accent capital vowel -ACO = 5 # accent capital other -ASV = 6 # accent small vowel -ASO = 7 # accent small other -CLASS_NUM = 8 # total classes +UDF = 0 # undefined +OTH = 1 # other +ASC = 2 # ascii capital letter +ASS = 3 # ascii small letter +ACV = 4 # accent capital vowel +ACO = 5 # accent capital other +ASV = 6 # accent small vowel +ASO = 7 # accent small other +CLASS_NUM = 8 # total classes -Latin1_CharToClass = ( \ - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 00 - 07 - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 08 - 0F - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 10 - 17 - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 18 - 1F - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 20 - 27 - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 28 - 2F - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 30 - 37 - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 38 - 3F - OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 40 - 47 - ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 48 - 4F - ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 50 - 57 - ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH, # 58 - 5F - OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 60 - 67 - ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 68 - 6F - ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 70 - 77 - ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH, # 78 - 7F - OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH, # 80 - 87 - OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF, # 88 - 8F - UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 90 - 97 - OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO, # 98 - 9F - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A0 - A7 - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A8 - AF - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B0 - B7 - OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B8 - BF - ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO, # C0 - C7 - ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV, # C8 - CF - ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH, # D0 - D7 - ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO, # D8 - DF - ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO, # E0 - E7 - ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV, # E8 - EF - ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH, # F0 - F7 - ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO, # F8 - FF +Latin1_CharToClass = ( + OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 00 - 07 + OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 08 - 0F + OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 10 - 17 + OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 18 - 1F + OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 20 - 27 + OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 28 - 2F + OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 30 - 37 + OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 38 - 3F + OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 40 - 47 + ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 48 - 4F + ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 50 - 57 + ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH, # 58 - 5F + OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 60 - 67 + ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 68 - 6F + ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 70 - 77 + ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH, # 78 - 7F + OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH, # 80 - 87 + OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF, # 88 - 8F + UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 90 - 97 + OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO, # 98 - 9F + OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A0 - A7 + OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A8 - AF + OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B0 - B7 + OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B8 - BF + ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO, # C0 - C7 + ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV, # C8 - CF + ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH, # D0 - D7 + ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO, # D8 - DF + ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO, # E0 - E7 + ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV, # E8 - EF + ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH, # F0 - F7 + ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO, # F8 - FF ) -# 0 : illegal -# 1 : very unlikely -# 2 : normal +# 0 : illegal +# 1 : very unlikely +# 2 : normal # 3 : very likely -Latin1ClassModel = ( \ -# UDF OTH ASC ASS ACV ACO ASV ASO - 0, 0, 0, 0, 0, 0, 0, 0, # UDF - 0, 3, 3, 3, 3, 3, 3, 3, # OTH - 0, 3, 3, 3, 3, 3, 3, 3, # ASC - 0, 3, 3, 3, 1, 1, 3, 3, # ASS - 0, 3, 3, 3, 1, 2, 1, 2, # ACV - 0, 3, 3, 3, 3, 3, 3, 3, # ACO - 0, 3, 1, 3, 1, 1, 1, 3, # ASV - 0, 3, 1, 3, 1, 1, 3, 3, # ASO +Latin1ClassModel = ( + # UDF OTH ASC ASS ACV ACO ASV ASO + 0, 0, 0, 0, 0, 0, 0, 0, # UDF + 0, 3, 3, 3, 3, 3, 3, 3, # OTH + 0, 3, 3, 3, 3, 3, 3, 3, # ASC + 0, 3, 3, 3, 1, 1, 3, 3, # ASS + 0, 3, 3, 3, 1, 2, 1, 2, # ACV + 0, 3, 3, 3, 3, 3, 3, 3, # ACO + 0, 3, 1, 3, 1, 1, 1, 3, # ASV + 0, 3, 1, 3, 1, 1, 3, 3, # ASO ) + class Latin1Prober(CharSetProber): def __init__(self): CharSetProber.__init__(self) @@ -109,10 +110,11 @@ class Latin1Prober(CharSetProber): def feed(self, aBuf): aBuf = self.filter_with_english_letters(aBuf) for c in aBuf: - charClass = Latin1_CharToClass[ord(c)] - freq = Latin1ClassModel[(self._mLastCharClass * CLASS_NUM) + charClass] + charClass = Latin1_CharToClass[wrap_ord(c)] + freq = Latin1ClassModel[(self._mLastCharClass * CLASS_NUM) + + charClass] if freq == 0: - self._mState = constants.eNotMe + self._mState = eNotMe break self._mFreqCounter[freq] += 1 self._mLastCharClass = charClass @@ -120,17 +122,18 @@ class Latin1Prober(CharSetProber): return self.get_state() def get_confidence(self): - if self.get_state() == constants.eNotMe: + if self.get_state() == eNotMe: return 0.01 - - total = reduce(operator.add, self._mFreqCounter) + + total = sum(self._mFreqCounter) if total < 0.01: confidence = 0.0 else: - confidence = (self._mFreqCounter[3] / total) - (self._mFreqCounter[1] * 20.0 / total) + confidence = ((self._mFreqCounter[3] / total) + - (self._mFreqCounter[1] * 20.0 / total)) if confidence < 0.0: confidence = 0.0 - # lower the confidence of latin1 so that other more accurate detector - # can take priority. + # lower the confidence of latin1 so that other more accurate + # detector can take priority. confidence = confidence * 0.5 return confidence diff --git a/libs/chardet/mbcharsetprober.py b/libs/chardet/mbcharsetprober.py index a813144..bb42f2f 100755 --- a/libs/chardet/mbcharsetprober.py +++ b/libs/chardet/mbcharsetprober.py @@ -15,28 +15,29 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants, sys -from constants import eStart, eError, eItsMe -from charsetprober import CharSetProber +import sys +from . import constants +from .charsetprober import CharSetProber + class MultiByteCharSetProber(CharSetProber): def __init__(self): CharSetProber.__init__(self) self._mDistributionAnalyzer = None self._mCodingSM = None - self._mLastChar = ['\x00', '\x00'] + self._mLastChar = [0, 0] def reset(self): CharSetProber.reset(self) @@ -44,7 +45,7 @@ class MultiByteCharSetProber(CharSetProber): self._mCodingSM.reset() if self._mDistributionAnalyzer: self._mDistributionAnalyzer.reset() - self._mLastChar = ['\x00', '\x00'] + self._mLastChar = [0, 0] def get_charset_name(self): pass @@ -53,27 +54,30 @@ class MultiByteCharSetProber(CharSetProber): aLen = len(aBuf) for i in range(0, aLen): codingState = self._mCodingSM.next_state(aBuf[i]) - if codingState == eError: + if codingState == constants.eError: if constants._debug: - sys.stderr.write(self.get_charset_name() + ' prober hit error at byte ' + str(i) + '\n') + sys.stderr.write(self.get_charset_name() + + ' prober hit error at byte ' + str(i) + + '\n') self._mState = constants.eNotMe break - elif codingState == eItsMe: + elif codingState == constants.eItsMe: self._mState = constants.eFoundIt break - elif codingState == eStart: + elif codingState == constants.eStart: charLen = self._mCodingSM.get_current_charlen() if i == 0: self._mLastChar[1] = aBuf[0] self._mDistributionAnalyzer.feed(self._mLastChar, charLen) else: - self._mDistributionAnalyzer.feed(aBuf[i-1:i+1], charLen) - + self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], + charLen) + self._mLastChar[0] = aBuf[aLen - 1] - + if self.get_state() == constants.eDetecting: - if self._mDistributionAnalyzer.got_enough_data() and \ - (self.get_confidence() > constants.SHORTCUT_THRESHOLD): + if (self._mDistributionAnalyzer.got_enough_data() and + (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): self._mState = constants.eFoundIt return self.get_state() diff --git a/libs/chardet/mbcsgroupprober.py b/libs/chardet/mbcsgroupprober.py index 941cc3e..03c9dcf 100755 --- a/libs/chardet/mbcsgroupprober.py +++ b/libs/chardet/mbcsgroupprober.py @@ -15,36 +15,40 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -from charsetgroupprober import CharSetGroupProber -from utf8prober import UTF8Prober -from sjisprober import SJISProber -from eucjpprober import EUCJPProber -from gb2312prober import GB2312Prober -from euckrprober import EUCKRProber -from big5prober import Big5Prober -from euctwprober import EUCTWProber +from .charsetgroupprober import CharSetGroupProber +from .utf8prober import UTF8Prober +from .sjisprober import SJISProber +from .eucjpprober import EUCJPProber +from .gb2312prober import GB2312Prober +from .euckrprober import EUCKRProber +from .cp949prober import CP949Prober +from .big5prober import Big5Prober +from .euctwprober import EUCTWProber + class MBCSGroupProber(CharSetGroupProber): def __init__(self): CharSetGroupProber.__init__(self) - self._mProbers = [ \ + self._mProbers = [ UTF8Prober(), SJISProber(), EUCJPProber(), GB2312Prober(), EUCKRProber(), + CP949Prober(), Big5Prober(), - EUCTWProber()] + EUCTWProber() + ] self.reset() diff --git a/libs/chardet/mbcssm.py b/libs/chardet/mbcssm.py index e46c1ff..3f93cfb 100755 --- a/libs/chardet/mbcssm.py +++ b/libs/chardet/mbcssm.py @@ -13,60 +13,62 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -from constants import eStart, eError, eItsMe +from .constants import eStart, eError, eItsMe -# BIG5 +# BIG5 -BIG5_cls = ( \ +BIG5_cls = ( 1,1,1,1,1,1,1,1, # 00 - 07 #allow 0x00 as legal value - 1,1,1,1,1,1,0,0, # 08 - 0f - 1,1,1,1,1,1,1,1, # 10 - 17 - 1,1,1,0,1,1,1,1, # 18 - 1f - 1,1,1,1,1,1,1,1, # 20 - 27 - 1,1,1,1,1,1,1,1, # 28 - 2f - 1,1,1,1,1,1,1,1, # 30 - 37 - 1,1,1,1,1,1,1,1, # 38 - 3f - 2,2,2,2,2,2,2,2, # 40 - 47 - 2,2,2,2,2,2,2,2, # 48 - 4f - 2,2,2,2,2,2,2,2, # 50 - 57 - 2,2,2,2,2,2,2,2, # 58 - 5f - 2,2,2,2,2,2,2,2, # 60 - 67 - 2,2,2,2,2,2,2,2, # 68 - 6f - 2,2,2,2,2,2,2,2, # 70 - 77 - 2,2,2,2,2,2,2,1, # 78 - 7f - 4,4,4,4,4,4,4,4, # 80 - 87 - 4,4,4,4,4,4,4,4, # 88 - 8f - 4,4,4,4,4,4,4,4, # 90 - 97 - 4,4,4,4,4,4,4,4, # 98 - 9f - 4,3,3,3,3,3,3,3, # a0 - a7 - 3,3,3,3,3,3,3,3, # a8 - af - 3,3,3,3,3,3,3,3, # b0 - b7 - 3,3,3,3,3,3,3,3, # b8 - bf - 3,3,3,3,3,3,3,3, # c0 - c7 - 3,3,3,3,3,3,3,3, # c8 - cf - 3,3,3,3,3,3,3,3, # d0 - d7 - 3,3,3,3,3,3,3,3, # d8 - df - 3,3,3,3,3,3,3,3, # e0 - e7 - 3,3,3,3,3,3,3,3, # e8 - ef - 3,3,3,3,3,3,3,3, # f0 - f7 - 3,3,3,3,3,3,3,0) # f8 - ff - -BIG5_st = ( \ - eError,eStart,eStart, 3,eError,eError,eError,eError,#00-07 - eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,#08-0f - eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart)#10-17 + 1,1,1,1,1,1,0,0, # 08 - 0f + 1,1,1,1,1,1,1,1, # 10 - 17 + 1,1,1,0,1,1,1,1, # 18 - 1f + 1,1,1,1,1,1,1,1, # 20 - 27 + 1,1,1,1,1,1,1,1, # 28 - 2f + 1,1,1,1,1,1,1,1, # 30 - 37 + 1,1,1,1,1,1,1,1, # 38 - 3f + 2,2,2,2,2,2,2,2, # 40 - 47 + 2,2,2,2,2,2,2,2, # 48 - 4f + 2,2,2,2,2,2,2,2, # 50 - 57 + 2,2,2,2,2,2,2,2, # 58 - 5f + 2,2,2,2,2,2,2,2, # 60 - 67 + 2,2,2,2,2,2,2,2, # 68 - 6f + 2,2,2,2,2,2,2,2, # 70 - 77 + 2,2,2,2,2,2,2,1, # 78 - 7f + 4,4,4,4,4,4,4,4, # 80 - 87 + 4,4,4,4,4,4,4,4, # 88 - 8f + 4,4,4,4,4,4,4,4, # 90 - 97 + 4,4,4,4,4,4,4,4, # 98 - 9f + 4,3,3,3,3,3,3,3, # a0 - a7 + 3,3,3,3,3,3,3,3, # a8 - af + 3,3,3,3,3,3,3,3, # b0 - b7 + 3,3,3,3,3,3,3,3, # b8 - bf + 3,3,3,3,3,3,3,3, # c0 - c7 + 3,3,3,3,3,3,3,3, # c8 - cf + 3,3,3,3,3,3,3,3, # d0 - d7 + 3,3,3,3,3,3,3,3, # d8 - df + 3,3,3,3,3,3,3,3, # e0 - e7 + 3,3,3,3,3,3,3,3, # e8 - ef + 3,3,3,3,3,3,3,3, # f0 - f7 + 3,3,3,3,3,3,3,0 # f8 - ff +) + +BIG5_st = ( + eError,eStart,eStart, 3,eError,eError,eError,eError,#00-07 + eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,#08-0f + eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart#10-17 +) Big5CharLenTable = (0, 1, 1, 2, 0) @@ -76,48 +78,90 @@ Big5SMModel = {'classTable': BIG5_cls, 'charLenTable': Big5CharLenTable, 'name': 'Big5'} +# CP949 + +CP949_cls = ( + 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,0,0, # 00 - 0f + 1,1,1,1,1,1,1,1, 1,1,1,0,1,1,1,1, # 10 - 1f + 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, # 20 - 2f + 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, # 30 - 3f + 1,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, # 40 - 4f + 4,4,5,5,5,5,5,5, 5,5,5,1,1,1,1,1, # 50 - 5f + 1,5,5,5,5,5,5,5, 5,5,5,5,5,5,5,5, # 60 - 6f + 5,5,5,5,5,5,5,5, 5,5,5,1,1,1,1,1, # 70 - 7f + 0,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, # 80 - 8f + 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, # 90 - 9f + 6,7,7,7,7,7,7,7, 7,7,7,7,7,8,8,8, # a0 - af + 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7, # b0 - bf + 7,7,7,7,7,7,9,2, 2,3,2,2,2,2,2,2, # c0 - cf + 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, # d0 - df + 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, # e0 - ef + 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,0, # f0 - ff +) + +CP949_st = ( +#cls= 0 1 2 3 4 5 6 7 8 9 # previous state = + eError,eStart, 3,eError,eStart,eStart, 4, 5,eError, 6, # eStart + eError,eError,eError,eError,eError,eError,eError,eError,eError,eError, # eError + eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe, # eItsMe + eError,eError,eStart,eStart,eError,eError,eError,eStart,eStart,eStart, # 3 + eError,eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart,eStart, # 4 + eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart,eStart,eStart, # 5 + eError,eStart,eStart,eStart,eStart,eError,eError,eStart,eStart,eStart, # 6 +) + +CP949CharLenTable = (0, 1, 2, 0, 1, 1, 2, 2, 0, 2) + +CP949SMModel = {'classTable': CP949_cls, + 'classFactor': 10, + 'stateTable': CP949_st, + 'charLenTable': CP949CharLenTable, + 'name': 'CP949'} + # EUC-JP -EUCJP_cls = ( \ - 4,4,4,4,4,4,4,4, # 00 - 07 - 4,4,4,4,4,4,5,5, # 08 - 0f - 4,4,4,4,4,4,4,4, # 10 - 17 - 4,4,4,5,4,4,4,4, # 18 - 1f - 4,4,4,4,4,4,4,4, # 20 - 27 - 4,4,4,4,4,4,4,4, # 28 - 2f - 4,4,4,4,4,4,4,4, # 30 - 37 - 4,4,4,4,4,4,4,4, # 38 - 3f - 4,4,4,4,4,4,4,4, # 40 - 47 - 4,4,4,4,4,4,4,4, # 48 - 4f - 4,4,4,4,4,4,4,4, # 50 - 57 - 4,4,4,4,4,4,4,4, # 58 - 5f - 4,4,4,4,4,4,4,4, # 60 - 67 - 4,4,4,4,4,4,4,4, # 68 - 6f - 4,4,4,4,4,4,4,4, # 70 - 77 - 4,4,4,4,4,4,4,4, # 78 - 7f - 5,5,5,5,5,5,5,5, # 80 - 87 - 5,5,5,5,5,5,1,3, # 88 - 8f - 5,5,5,5,5,5,5,5, # 90 - 97 - 5,5,5,5,5,5,5,5, # 98 - 9f - 5,2,2,2,2,2,2,2, # a0 - a7 - 2,2,2,2,2,2,2,2, # a8 - af - 2,2,2,2,2,2,2,2, # b0 - b7 - 2,2,2,2,2,2,2,2, # b8 - bf - 2,2,2,2,2,2,2,2, # c0 - c7 - 2,2,2,2,2,2,2,2, # c8 - cf - 2,2,2,2,2,2,2,2, # d0 - d7 - 2,2,2,2,2,2,2,2, # d8 - df - 0,0,0,0,0,0,0,0, # e0 - e7 - 0,0,0,0,0,0,0,0, # e8 - ef - 0,0,0,0,0,0,0,0, # f0 - f7 - 0,0,0,0,0,0,0,5) # f8 - ff - -EUCJP_st = ( \ - 3, 4, 3, 5,eStart,eError,eError,eError,#00-07 - eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,#08-0f - eItsMe,eItsMe,eStart,eError,eStart,eError,eError,eError,#10-17 - eError,eError,eStart,eError,eError,eError, 3,eError,#18-1f - 3,eError,eError,eError,eStart,eStart,eStart,eStart)#20-27 +EUCJP_cls = ( + 4,4,4,4,4,4,4,4, # 00 - 07 + 4,4,4,4,4,4,5,5, # 08 - 0f + 4,4,4,4,4,4,4,4, # 10 - 17 + 4,4,4,5,4,4,4,4, # 18 - 1f + 4,4,4,4,4,4,4,4, # 20 - 27 + 4,4,4,4,4,4,4,4, # 28 - 2f + 4,4,4,4,4,4,4,4, # 30 - 37 + 4,4,4,4,4,4,4,4, # 38 - 3f + 4,4,4,4,4,4,4,4, # 40 - 47 + 4,4,4,4,4,4,4,4, # 48 - 4f + 4,4,4,4,4,4,4,4, # 50 - 57 + 4,4,4,4,4,4,4,4, # 58 - 5f + 4,4,4,4,4,4,4,4, # 60 - 67 + 4,4,4,4,4,4,4,4, # 68 - 6f + 4,4,4,4,4,4,4,4, # 70 - 77 + 4,4,4,4,4,4,4,4, # 78 - 7f + 5,5,5,5,5,5,5,5, # 80 - 87 + 5,5,5,5,5,5,1,3, # 88 - 8f + 5,5,5,5,5,5,5,5, # 90 - 97 + 5,5,5,5,5,5,5,5, # 98 - 9f + 5,2,2,2,2,2,2,2, # a0 - a7 + 2,2,2,2,2,2,2,2, # a8 - af + 2,2,2,2,2,2,2,2, # b0 - b7 + 2,2,2,2,2,2,2,2, # b8 - bf + 2,2,2,2,2,2,2,2, # c0 - c7 + 2,2,2,2,2,2,2,2, # c8 - cf + 2,2,2,2,2,2,2,2, # d0 - d7 + 2,2,2,2,2,2,2,2, # d8 - df + 0,0,0,0,0,0,0,0, # e0 - e7 + 0,0,0,0,0,0,0,0, # e8 - ef + 0,0,0,0,0,0,0,0, # f0 - f7 + 0,0,0,0,0,0,0,5 # f8 - ff +) + +EUCJP_st = ( + 3, 4, 3, 5,eStart,eError,eError,eError,#00-07 + eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,#08-0f + eItsMe,eItsMe,eStart,eError,eStart,eError,eError,eError,#10-17 + eError,eError,eStart,eError,eError,eError, 3,eError,#18-1f + 3,eError,eError,eError,eStart,eStart,eStart,eStart#20-27 +) EUCJPCharLenTable = (2, 2, 2, 3, 1, 0) @@ -129,43 +173,45 @@ EUCJPSMModel = {'classTable': EUCJP_cls, # EUC-KR -EUCKR_cls = ( \ - 1,1,1,1,1,1,1,1, # 00 - 07 - 1,1,1,1,1,1,0,0, # 08 - 0f - 1,1,1,1,1,1,1,1, # 10 - 17 - 1,1,1,0,1,1,1,1, # 18 - 1f - 1,1,1,1,1,1,1,1, # 20 - 27 - 1,1,1,1,1,1,1,1, # 28 - 2f - 1,1,1,1,1,1,1,1, # 30 - 37 - 1,1,1,1,1,1,1,1, # 38 - 3f - 1,1,1,1,1,1,1,1, # 40 - 47 - 1,1,1,1,1,1,1,1, # 48 - 4f - 1,1,1,1,1,1,1,1, # 50 - 57 - 1,1,1,1,1,1,1,1, # 58 - 5f - 1,1,1,1,1,1,1,1, # 60 - 67 - 1,1,1,1,1,1,1,1, # 68 - 6f - 1,1,1,1,1,1,1,1, # 70 - 77 - 1,1,1,1,1,1,1,1, # 78 - 7f - 0,0,0,0,0,0,0,0, # 80 - 87 - 0,0,0,0,0,0,0,0, # 88 - 8f - 0,0,0,0,0,0,0,0, # 90 - 97 - 0,0,0,0,0,0,0,0, # 98 - 9f - 0,2,2,2,2,2,2,2, # a0 - a7 - 2,2,2,2,2,3,3,3, # a8 - af - 2,2,2,2,2,2,2,2, # b0 - b7 - 2,2,2,2,2,2,2,2, # b8 - bf - 2,2,2,2,2,2,2,2, # c0 - c7 - 2,3,2,2,2,2,2,2, # c8 - cf - 2,2,2,2,2,2,2,2, # d0 - d7 - 2,2,2,2,2,2,2,2, # d8 - df - 2,2,2,2,2,2,2,2, # e0 - e7 - 2,2,2,2,2,2,2,2, # e8 - ef - 2,2,2,2,2,2,2,2, # f0 - f7 - 2,2,2,2,2,2,2,0) # f8 - ff +EUCKR_cls = ( + 1,1,1,1,1,1,1,1, # 00 - 07 + 1,1,1,1,1,1,0,0, # 08 - 0f + 1,1,1,1,1,1,1,1, # 10 - 17 + 1,1,1,0,1,1,1,1, # 18 - 1f + 1,1,1,1,1,1,1,1, # 20 - 27 + 1,1,1,1,1,1,1,1, # 28 - 2f + 1,1,1,1,1,1,1,1, # 30 - 37 + 1,1,1,1,1,1,1,1, # 38 - 3f + 1,1,1,1,1,1,1,1, # 40 - 47 + 1,1,1,1,1,1,1,1, # 48 - 4f + 1,1,1,1,1,1,1,1, # 50 - 57 + 1,1,1,1,1,1,1,1, # 58 - 5f + 1,1,1,1,1,1,1,1, # 60 - 67 + 1,1,1,1,1,1,1,1, # 68 - 6f + 1,1,1,1,1,1,1,1, # 70 - 77 + 1,1,1,1,1,1,1,1, # 78 - 7f + 0,0,0,0,0,0,0,0, # 80 - 87 + 0,0,0,0,0,0,0,0, # 88 - 8f + 0,0,0,0,0,0,0,0, # 90 - 97 + 0,0,0,0,0,0,0,0, # 98 - 9f + 0,2,2,2,2,2,2,2, # a0 - a7 + 2,2,2,2,2,3,3,3, # a8 - af + 2,2,2,2,2,2,2,2, # b0 - b7 + 2,2,2,2,2,2,2,2, # b8 - bf + 2,2,2,2,2,2,2,2, # c0 - c7 + 2,3,2,2,2,2,2,2, # c8 - cf + 2,2,2,2,2,2,2,2, # d0 - d7 + 2,2,2,2,2,2,2,2, # d8 - df + 2,2,2,2,2,2,2,2, # e0 - e7 + 2,2,2,2,2,2,2,2, # e8 - ef + 2,2,2,2,2,2,2,2, # f0 - f7 + 2,2,2,2,2,2,2,0 # f8 - ff +) EUCKR_st = ( - eError,eStart, 3,eError,eError,eError,eError,eError,#00-07 - eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,eStart)#08-0f + eError,eStart, 3,eError,eError,eError,eError,eError,#00-07 + eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,eStart #08-0f +) EUCKRCharLenTable = (0, 1, 2, 0) @@ -177,47 +223,49 @@ EUCKRSMModel = {'classTable': EUCKR_cls, # EUC-TW -EUCTW_cls = ( \ - 2,2,2,2,2,2,2,2, # 00 - 07 - 2,2,2,2,2,2,0,0, # 08 - 0f - 2,2,2,2,2,2,2,2, # 10 - 17 - 2,2,2,0,2,2,2,2, # 18 - 1f - 2,2,2,2,2,2,2,2, # 20 - 27 - 2,2,2,2,2,2,2,2, # 28 - 2f - 2,2,2,2,2,2,2,2, # 30 - 37 - 2,2,2,2,2,2,2,2, # 38 - 3f - 2,2,2,2,2,2,2,2, # 40 - 47 - 2,2,2,2,2,2,2,2, # 48 - 4f - 2,2,2,2,2,2,2,2, # 50 - 57 - 2,2,2,2,2,2,2,2, # 58 - 5f - 2,2,2,2,2,2,2,2, # 60 - 67 - 2,2,2,2,2,2,2,2, # 68 - 6f - 2,2,2,2,2,2,2,2, # 70 - 77 - 2,2,2,2,2,2,2,2, # 78 - 7f - 0,0,0,0,0,0,0,0, # 80 - 87 - 0,0,0,0,0,0,6,0, # 88 - 8f - 0,0,0,0,0,0,0,0, # 90 - 97 - 0,0,0,0,0,0,0,0, # 98 - 9f - 0,3,4,4,4,4,4,4, # a0 - a7 - 5,5,1,1,1,1,1,1, # a8 - af - 1,1,1,1,1,1,1,1, # b0 - b7 - 1,1,1,1,1,1,1,1, # b8 - bf - 1,1,3,1,3,3,3,3, # c0 - c7 - 3,3,3,3,3,3,3,3, # c8 - cf - 3,3,3,3,3,3,3,3, # d0 - d7 - 3,3,3,3,3,3,3,3, # d8 - df - 3,3,3,3,3,3,3,3, # e0 - e7 - 3,3,3,3,3,3,3,3, # e8 - ef - 3,3,3,3,3,3,3,3, # f0 - f7 - 3,3,3,3,3,3,3,0) # f8 - ff - -EUCTW_st = ( \ - eError,eError,eStart, 3, 3, 3, 4,eError,#00-07 - eError,eError,eError,eError,eError,eError,eItsMe,eItsMe,#08-0f - eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eStart,eError,#10-17 - eStart,eStart,eStart,eError,eError,eError,eError,eError,#18-1f - 5,eError,eError,eError,eStart,eError,eStart,eStart,#20-27 - eStart,eError,eStart,eStart,eStart,eStart,eStart,eStart)#28-2f +EUCTW_cls = ( + 2,2,2,2,2,2,2,2, # 00 - 07 + 2,2,2,2,2,2,0,0, # 08 - 0f + 2,2,2,2,2,2,2,2, # 10 - 17 + 2,2,2,0,2,2,2,2, # 18 - 1f + 2,2,2,2,2,2,2,2, # 20 - 27 + 2,2,2,2,2,2,2,2, # 28 - 2f + 2,2,2,2,2,2,2,2, # 30 - 37 + 2,2,2,2,2,2,2,2, # 38 - 3f + 2,2,2,2,2,2,2,2, # 40 - 47 + 2,2,2,2,2,2,2,2, # 48 - 4f + 2,2,2,2,2,2,2,2, # 50 - 57 + 2,2,2,2,2,2,2,2, # 58 - 5f + 2,2,2,2,2,2,2,2, # 60 - 67 + 2,2,2,2,2,2,2,2, # 68 - 6f + 2,2,2,2,2,2,2,2, # 70 - 77 + 2,2,2,2,2,2,2,2, # 78 - 7f + 0,0,0,0,0,0,0,0, # 80 - 87 + 0,0,0,0,0,0,6,0, # 88 - 8f + 0,0,0,0,0,0,0,0, # 90 - 97 + 0,0,0,0,0,0,0,0, # 98 - 9f + 0,3,4,4,4,4,4,4, # a0 - a7 + 5,5,1,1,1,1,1,1, # a8 - af + 1,1,1,1,1,1,1,1, # b0 - b7 + 1,1,1,1,1,1,1,1, # b8 - bf + 1,1,3,1,3,3,3,3, # c0 - c7 + 3,3,3,3,3,3,3,3, # c8 - cf + 3,3,3,3,3,3,3,3, # d0 - d7 + 3,3,3,3,3,3,3,3, # d8 - df + 3,3,3,3,3,3,3,3, # e0 - e7 + 3,3,3,3,3,3,3,3, # e8 - ef + 3,3,3,3,3,3,3,3, # f0 - f7 + 3,3,3,3,3,3,3,0 # f8 - ff +) + +EUCTW_st = ( + eError,eError,eStart, 3, 3, 3, 4,eError,#00-07 + eError,eError,eError,eError,eError,eError,eItsMe,eItsMe,#08-0f + eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eStart,eError,#10-17 + eStart,eStart,eStart,eError,eError,eError,eError,eError,#18-1f + 5,eError,eError,eError,eStart,eError,eStart,eStart,#20-27 + eStart,eError,eStart,eStart,eStart,eStart,eStart,eStart #28-2f +) EUCTWCharLenTable = (0, 0, 1, 2, 2, 2, 3) @@ -229,53 +277,55 @@ EUCTWSMModel = {'classTable': EUCTW_cls, # GB2312 -GB2312_cls = ( \ - 1,1,1,1,1,1,1,1, # 00 - 07 - 1,1,1,1,1,1,0,0, # 08 - 0f - 1,1,1,1,1,1,1,1, # 10 - 17 - 1,1,1,0,1,1,1,1, # 18 - 1f - 1,1,1,1,1,1,1,1, # 20 - 27 - 1,1,1,1,1,1,1,1, # 28 - 2f - 3,3,3,3,3,3,3,3, # 30 - 37 - 3,3,1,1,1,1,1,1, # 38 - 3f - 2,2,2,2,2,2,2,2, # 40 - 47 - 2,2,2,2,2,2,2,2, # 48 - 4f - 2,2,2,2,2,2,2,2, # 50 - 57 - 2,2,2,2,2,2,2,2, # 58 - 5f - 2,2,2,2,2,2,2,2, # 60 - 67 - 2,2,2,2,2,2,2,2, # 68 - 6f - 2,2,2,2,2,2,2,2, # 70 - 77 - 2,2,2,2,2,2,2,4, # 78 - 7f - 5,6,6,6,6,6,6,6, # 80 - 87 - 6,6,6,6,6,6,6,6, # 88 - 8f - 6,6,6,6,6,6,6,6, # 90 - 97 - 6,6,6,6,6,6,6,6, # 98 - 9f - 6,6,6,6,6,6,6,6, # a0 - a7 - 6,6,6,6,6,6,6,6, # a8 - af - 6,6,6,6,6,6,6,6, # b0 - b7 - 6,6,6,6,6,6,6,6, # b8 - bf - 6,6,6,6,6,6,6,6, # c0 - c7 - 6,6,6,6,6,6,6,6, # c8 - cf - 6,6,6,6,6,6,6,6, # d0 - d7 - 6,6,6,6,6,6,6,6, # d8 - df - 6,6,6,6,6,6,6,6, # e0 - e7 - 6,6,6,6,6,6,6,6, # e8 - ef - 6,6,6,6,6,6,6,6, # f0 - f7 - 6,6,6,6,6,6,6,0) # f8 - ff - -GB2312_st = ( \ - eError,eStart,eStart,eStart,eStart,eStart, 3,eError,#00-07 - eError,eError,eError,eError,eError,eError,eItsMe,eItsMe,#08-0f - eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,#10-17 - 4,eError,eStart,eStart,eError,eError,eError,eError,#18-1f - eError,eError, 5,eError,eError,eError,eItsMe,eError,#20-27 - eError,eError,eStart,eStart,eStart,eStart,eStart,eStart)#28-2f - -# To be accurate, the length of class 6 can be either 2 or 4. -# But it is not necessary to discriminate between the two since -# it is used for frequency analysis only, and we are validing -# each code range there as well. So it is safe to set it to be -# 2 here. +GB2312_cls = ( + 1,1,1,1,1,1,1,1, # 00 - 07 + 1,1,1,1,1,1,0,0, # 08 - 0f + 1,1,1,1,1,1,1,1, # 10 - 17 + 1,1,1,0,1,1,1,1, # 18 - 1f + 1,1,1,1,1,1,1,1, # 20 - 27 + 1,1,1,1,1,1,1,1, # 28 - 2f + 3,3,3,3,3,3,3,3, # 30 - 37 + 3,3,1,1,1,1,1,1, # 38 - 3f + 2,2,2,2,2,2,2,2, # 40 - 47 + 2,2,2,2,2,2,2,2, # 48 - 4f + 2,2,2,2,2,2,2,2, # 50 - 57 + 2,2,2,2,2,2,2,2, # 58 - 5f + 2,2,2,2,2,2,2,2, # 60 - 67 + 2,2,2,2,2,2,2,2, # 68 - 6f + 2,2,2,2,2,2,2,2, # 70 - 77 + 2,2,2,2,2,2,2,4, # 78 - 7f + 5,6,6,6,6,6,6,6, # 80 - 87 + 6,6,6,6,6,6,6,6, # 88 - 8f + 6,6,6,6,6,6,6,6, # 90 - 97 + 6,6,6,6,6,6,6,6, # 98 - 9f + 6,6,6,6,6,6,6,6, # a0 - a7 + 6,6,6,6,6,6,6,6, # a8 - af + 6,6,6,6,6,6,6,6, # b0 - b7 + 6,6,6,6,6,6,6,6, # b8 - bf + 6,6,6,6,6,6,6,6, # c0 - c7 + 6,6,6,6,6,6,6,6, # c8 - cf + 6,6,6,6,6,6,6,6, # d0 - d7 + 6,6,6,6,6,6,6,6, # d8 - df + 6,6,6,6,6,6,6,6, # e0 - e7 + 6,6,6,6,6,6,6,6, # e8 - ef + 6,6,6,6,6,6,6,6, # f0 - f7 + 6,6,6,6,6,6,6,0 # f8 - ff +) + +GB2312_st = ( + eError,eStart,eStart,eStart,eStart,eStart, 3,eError,#00-07 + eError,eError,eError,eError,eError,eError,eItsMe,eItsMe,#08-0f + eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,#10-17 + 4,eError,eStart,eStart,eError,eError,eError,eError,#18-1f + eError,eError, 5,eError,eError,eError,eItsMe,eError,#20-27 + eError,eError,eStart,eStart,eStart,eStart,eStart,eStart #28-2f +) + +# To be accurate, the length of class 6 can be either 2 or 4. +# But it is not necessary to discriminate between the two since +# it is used for frequency analysis only, and we are validing +# each code range there as well. So it is safe to set it to be +# 2 here. GB2312CharLenTable = (0, 1, 1, 1, 1, 1, 2) GB2312SMModel = {'classTable': GB2312_cls, @@ -286,46 +336,49 @@ GB2312SMModel = {'classTable': GB2312_cls, # Shift_JIS -SJIS_cls = ( \ - 1,1,1,1,1,1,1,1, # 00 - 07 - 1,1,1,1,1,1,0,0, # 08 - 0f - 1,1,1,1,1,1,1,1, # 10 - 17 - 1,1,1,0,1,1,1,1, # 18 - 1f - 1,1,1,1,1,1,1,1, # 20 - 27 - 1,1,1,1,1,1,1,1, # 28 - 2f - 1,1,1,1,1,1,1,1, # 30 - 37 - 1,1,1,1,1,1,1,1, # 38 - 3f - 2,2,2,2,2,2,2,2, # 40 - 47 - 2,2,2,2,2,2,2,2, # 48 - 4f - 2,2,2,2,2,2,2,2, # 50 - 57 - 2,2,2,2,2,2,2,2, # 58 - 5f - 2,2,2,2,2,2,2,2, # 60 - 67 - 2,2,2,2,2,2,2,2, # 68 - 6f - 2,2,2,2,2,2,2,2, # 70 - 77 - 2,2,2,2,2,2,2,1, # 78 - 7f - 3,3,3,3,3,3,3,3, # 80 - 87 - 3,3,3,3,3,3,3,3, # 88 - 8f - 3,3,3,3,3,3,3,3, # 90 - 97 - 3,3,3,3,3,3,3,3, # 98 - 9f - #0xa0 is illegal in sjis encoding, but some pages does +SJIS_cls = ( + 1,1,1,1,1,1,1,1, # 00 - 07 + 1,1,1,1,1,1,0,0, # 08 - 0f + 1,1,1,1,1,1,1,1, # 10 - 17 + 1,1,1,0,1,1,1,1, # 18 - 1f + 1,1,1,1,1,1,1,1, # 20 - 27 + 1,1,1,1,1,1,1,1, # 28 - 2f + 1,1,1,1,1,1,1,1, # 30 - 37 + 1,1,1,1,1,1,1,1, # 38 - 3f + 2,2,2,2,2,2,2,2, # 40 - 47 + 2,2,2,2,2,2,2,2, # 48 - 4f + 2,2,2,2,2,2,2,2, # 50 - 57 + 2,2,2,2,2,2,2,2, # 58 - 5f + 2,2,2,2,2,2,2,2, # 60 - 67 + 2,2,2,2,2,2,2,2, # 68 - 6f + 2,2,2,2,2,2,2,2, # 70 - 77 + 2,2,2,2,2,2,2,1, # 78 - 7f + 3,3,3,3,3,3,3,3, # 80 - 87 + 3,3,3,3,3,3,3,3, # 88 - 8f + 3,3,3,3,3,3,3,3, # 90 - 97 + 3,3,3,3,3,3,3,3, # 98 - 9f + #0xa0 is illegal in sjis encoding, but some pages does #contain such byte. We need to be more error forgiven. - 2,2,2,2,2,2,2,2, # a0 - a7 - 2,2,2,2,2,2,2,2, # a8 - af - 2,2,2,2,2,2,2,2, # b0 - b7 - 2,2,2,2,2,2,2,2, # b8 - bf - 2,2,2,2,2,2,2,2, # c0 - c7 - 2,2,2,2,2,2,2,2, # c8 - cf - 2,2,2,2,2,2,2,2, # d0 - d7 - 2,2,2,2,2,2,2,2, # d8 - df - 3,3,3,3,3,3,3,3, # e0 - e7 - 3,3,3,3,3,4,4,4, # e8 - ef - 4,4,4,4,4,4,4,4, # f0 - f7 - 4,4,4,4,4,0,0,0) # f8 - ff - -SJIS_st = ( \ - eError,eStart,eStart, 3,eError,eError,eError,eError,#00-07 - eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,#08-0f - eItsMe,eItsMe,eError,eError,eStart,eStart,eStart,eStart)#10-17 + 2,2,2,2,2,2,2,2, # a0 - a7 + 2,2,2,2,2,2,2,2, # a8 - af + 2,2,2,2,2,2,2,2, # b0 - b7 + 2,2,2,2,2,2,2,2, # b8 - bf + 2,2,2,2,2,2,2,2, # c0 - c7 + 2,2,2,2,2,2,2,2, # c8 - cf + 2,2,2,2,2,2,2,2, # d0 - d7 + 2,2,2,2,2,2,2,2, # d8 - df + 3,3,3,3,3,3,3,3, # e0 - e7 + 3,3,3,3,3,4,4,4, # e8 - ef + 4,4,4,4,4,4,4,4, # f0 - f7 + 4,4,4,4,4,0,0,0 # f8 - ff +) + + +SJIS_st = ( + eError,eStart,eStart, 3,eError,eError,eError,eError,#00-07 + eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,#08-0f + eItsMe,eItsMe,eError,eError,eStart,eStart,eStart,eStart #10-17 +) SJISCharLenTable = (0, 1, 1, 2, 0, 0) @@ -337,48 +390,50 @@ SJISSMModel = {'classTable': SJIS_cls, # UCS2-BE -UCS2BE_cls = ( \ - 0,0,0,0,0,0,0,0, # 00 - 07 - 0,0,1,0,0,2,0,0, # 08 - 0f - 0,0,0,0,0,0,0,0, # 10 - 17 - 0,0,0,3,0,0,0,0, # 18 - 1f - 0,0,0,0,0,0,0,0, # 20 - 27 - 0,3,3,3,3,3,0,0, # 28 - 2f - 0,0,0,0,0,0,0,0, # 30 - 37 - 0,0,0,0,0,0,0,0, # 38 - 3f - 0,0,0,0,0,0,0,0, # 40 - 47 - 0,0,0,0,0,0,0,0, # 48 - 4f - 0,0,0,0,0,0,0,0, # 50 - 57 - 0,0,0,0,0,0,0,0, # 58 - 5f - 0,0,0,0,0,0,0,0, # 60 - 67 - 0,0,0,0,0,0,0,0, # 68 - 6f - 0,0,0,0,0,0,0,0, # 70 - 77 - 0,0,0,0,0,0,0,0, # 78 - 7f - 0,0,0,0,0,0,0,0, # 80 - 87 - 0,0,0,0,0,0,0,0, # 88 - 8f - 0,0,0,0,0,0,0,0, # 90 - 97 - 0,0,0,0,0,0,0,0, # 98 - 9f - 0,0,0,0,0,0,0,0, # a0 - a7 - 0,0,0,0,0,0,0,0, # a8 - af - 0,0,0,0,0,0,0,0, # b0 - b7 - 0,0,0,0,0,0,0,0, # b8 - bf - 0,0,0,0,0,0,0,0, # c0 - c7 - 0,0,0,0,0,0,0,0, # c8 - cf - 0,0,0,0,0,0,0,0, # d0 - d7 - 0,0,0,0,0,0,0,0, # d8 - df - 0,0,0,0,0,0,0,0, # e0 - e7 - 0,0,0,0,0,0,0,0, # e8 - ef - 0,0,0,0,0,0,0,0, # f0 - f7 - 0,0,0,0,0,0,4,5) # f8 - ff - -UCS2BE_st = ( \ - 5, 7, 7,eError, 4, 3,eError,eError,#00-07 - eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,#08-0f - eItsMe,eItsMe, 6, 6, 6, 6,eError,eError,#10-17 - 6, 6, 6, 6, 6,eItsMe, 6, 6,#18-1f - 6, 6, 6, 6, 5, 7, 7,eError,#20-27 - 5, 8, 6, 6,eError, 6, 6, 6,#28-2f - 6, 6, 6, 6,eError,eError,eStart,eStart)#30-37 +UCS2BE_cls = ( + 0,0,0,0,0,0,0,0, # 00 - 07 + 0,0,1,0,0,2,0,0, # 08 - 0f + 0,0,0,0,0,0,0,0, # 10 - 17 + 0,0,0,3,0,0,0,0, # 18 - 1f + 0,0,0,0,0,0,0,0, # 20 - 27 + 0,3,3,3,3,3,0,0, # 28 - 2f + 0,0,0,0,0,0,0,0, # 30 - 37 + 0,0,0,0,0,0,0,0, # 38 - 3f + 0,0,0,0,0,0,0,0, # 40 - 47 + 0,0,0,0,0,0,0,0, # 48 - 4f + 0,0,0,0,0,0,0,0, # 50 - 57 + 0,0,0,0,0,0,0,0, # 58 - 5f + 0,0,0,0,0,0,0,0, # 60 - 67 + 0,0,0,0,0,0,0,0, # 68 - 6f + 0,0,0,0,0,0,0,0, # 70 - 77 + 0,0,0,0,0,0,0,0, # 78 - 7f + 0,0,0,0,0,0,0,0, # 80 - 87 + 0,0,0,0,0,0,0,0, # 88 - 8f + 0,0,0,0,0,0,0,0, # 90 - 97 + 0,0,0,0,0,0,0,0, # 98 - 9f + 0,0,0,0,0,0,0,0, # a0 - a7 + 0,0,0,0,0,0,0,0, # a8 - af + 0,0,0,0,0,0,0,0, # b0 - b7 + 0,0,0,0,0,0,0,0, # b8 - bf + 0,0,0,0,0,0,0,0, # c0 - c7 + 0,0,0,0,0,0,0,0, # c8 - cf + 0,0,0,0,0,0,0,0, # d0 - d7 + 0,0,0,0,0,0,0,0, # d8 - df + 0,0,0,0,0,0,0,0, # e0 - e7 + 0,0,0,0,0,0,0,0, # e8 - ef + 0,0,0,0,0,0,0,0, # f0 - f7 + 0,0,0,0,0,0,4,5 # f8 - ff +) + +UCS2BE_st = ( + 5, 7, 7,eError, 4, 3,eError,eError,#00-07 + eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,#08-0f + eItsMe,eItsMe, 6, 6, 6, 6,eError,eError,#10-17 + 6, 6, 6, 6, 6,eItsMe, 6, 6,#18-1f + 6, 6, 6, 6, 5, 7, 7,eError,#20-27 + 5, 8, 6, 6,eError, 6, 6, 6,#28-2f + 6, 6, 6, 6,eError,eError,eStart,eStart #30-37 +) UCS2BECharLenTable = (2, 2, 2, 0, 2, 2) @@ -390,48 +445,50 @@ UCS2BESMModel = {'classTable': UCS2BE_cls, # UCS2-LE -UCS2LE_cls = ( \ - 0,0,0,0,0,0,0,0, # 00 - 07 - 0,0,1,0,0,2,0,0, # 08 - 0f - 0,0,0,0,0,0,0,0, # 10 - 17 - 0,0,0,3,0,0,0,0, # 18 - 1f - 0,0,0,0,0,0,0,0, # 20 - 27 - 0,3,3,3,3,3,0,0, # 28 - 2f - 0,0,0,0,0,0,0,0, # 30 - 37 - 0,0,0,0,0,0,0,0, # 38 - 3f - 0,0,0,0,0,0,0,0, # 40 - 47 - 0,0,0,0,0,0,0,0, # 48 - 4f - 0,0,0,0,0,0,0,0, # 50 - 57 - 0,0,0,0,0,0,0,0, # 58 - 5f - 0,0,0,0,0,0,0,0, # 60 - 67 - 0,0,0,0,0,0,0,0, # 68 - 6f - 0,0,0,0,0,0,0,0, # 70 - 77 - 0,0,0,0,0,0,0,0, # 78 - 7f - 0,0,0,0,0,0,0,0, # 80 - 87 - 0,0,0,0,0,0,0,0, # 88 - 8f - 0,0,0,0,0,0,0,0, # 90 - 97 - 0,0,0,0,0,0,0,0, # 98 - 9f - 0,0,0,0,0,0,0,0, # a0 - a7 - 0,0,0,0,0,0,0,0, # a8 - af - 0,0,0,0,0,0,0,0, # b0 - b7 - 0,0,0,0,0,0,0,0, # b8 - bf - 0,0,0,0,0,0,0,0, # c0 - c7 - 0,0,0,0,0,0,0,0, # c8 - cf - 0,0,0,0,0,0,0,0, # d0 - d7 - 0,0,0,0,0,0,0,0, # d8 - df - 0,0,0,0,0,0,0,0, # e0 - e7 - 0,0,0,0,0,0,0,0, # e8 - ef - 0,0,0,0,0,0,0,0, # f0 - f7 - 0,0,0,0,0,0,4,5) # f8 - ff - -UCS2LE_st = ( \ - 6, 6, 7, 6, 4, 3,eError,eError,#00-07 - eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,#08-0f - eItsMe,eItsMe, 5, 5, 5,eError,eItsMe,eError,#10-17 - 5, 5, 5,eError, 5,eError, 6, 6,#18-1f - 7, 6, 8, 8, 5, 5, 5,eError,#20-27 - 5, 5, 5,eError,eError,eError, 5, 5,#28-2f - 5, 5, 5,eError, 5,eError,eStart,eStart)#30-37 +UCS2LE_cls = ( + 0,0,0,0,0,0,0,0, # 00 - 07 + 0,0,1,0,0,2,0,0, # 08 - 0f + 0,0,0,0,0,0,0,0, # 10 - 17 + 0,0,0,3,0,0,0,0, # 18 - 1f + 0,0,0,0,0,0,0,0, # 20 - 27 + 0,3,3,3,3,3,0,0, # 28 - 2f + 0,0,0,0,0,0,0,0, # 30 - 37 + 0,0,0,0,0,0,0,0, # 38 - 3f + 0,0,0,0,0,0,0,0, # 40 - 47 + 0,0,0,0,0,0,0,0, # 48 - 4f + 0,0,0,0,0,0,0,0, # 50 - 57 + 0,0,0,0,0,0,0,0, # 58 - 5f + 0,0,0,0,0,0,0,0, # 60 - 67 + 0,0,0,0,0,0,0,0, # 68 - 6f + 0,0,0,0,0,0,0,0, # 70 - 77 + 0,0,0,0,0,0,0,0, # 78 - 7f + 0,0,0,0,0,0,0,0, # 80 - 87 + 0,0,0,0,0,0,0,0, # 88 - 8f + 0,0,0,0,0,0,0,0, # 90 - 97 + 0,0,0,0,0,0,0,0, # 98 - 9f + 0,0,0,0,0,0,0,0, # a0 - a7 + 0,0,0,0,0,0,0,0, # a8 - af + 0,0,0,0,0,0,0,0, # b0 - b7 + 0,0,0,0,0,0,0,0, # b8 - bf + 0,0,0,0,0,0,0,0, # c0 - c7 + 0,0,0,0,0,0,0,0, # c8 - cf + 0,0,0,0,0,0,0,0, # d0 - d7 + 0,0,0,0,0,0,0,0, # d8 - df + 0,0,0,0,0,0,0,0, # e0 - e7 + 0,0,0,0,0,0,0,0, # e8 - ef + 0,0,0,0,0,0,0,0, # f0 - f7 + 0,0,0,0,0,0,4,5 # f8 - ff +) + +UCS2LE_st = ( + 6, 6, 7, 6, 4, 3,eError,eError,#00-07 + eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,#08-0f + eItsMe,eItsMe, 5, 5, 5,eError,eItsMe,eError,#10-17 + 5, 5, 5,eError, 5,eError, 6, 6,#18-1f + 7, 6, 8, 8, 5, 5, 5,eError,#20-27 + 5, 5, 5,eError,eError,eError, 5, 5,#28-2f + 5, 5, 5,eError, 5,eError,eStart,eStart #30-37 +) UCS2LECharLenTable = (2, 2, 2, 2, 2, 2) @@ -443,67 +500,69 @@ UCS2LESMModel = {'classTable': UCS2LE_cls, # UTF-8 -UTF8_cls = ( \ +UTF8_cls = ( 1,1,1,1,1,1,1,1, # 00 - 07 #allow 0x00 as a legal value - 1,1,1,1,1,1,0,0, # 08 - 0f - 1,1,1,1,1,1,1,1, # 10 - 17 - 1,1,1,0,1,1,1,1, # 18 - 1f - 1,1,1,1,1,1,1,1, # 20 - 27 - 1,1,1,1,1,1,1,1, # 28 - 2f - 1,1,1,1,1,1,1,1, # 30 - 37 - 1,1,1,1,1,1,1,1, # 38 - 3f - 1,1,1,1,1,1,1,1, # 40 - 47 - 1,1,1,1,1,1,1,1, # 48 - 4f - 1,1,1,1,1,1,1,1, # 50 - 57 - 1,1,1,1,1,1,1,1, # 58 - 5f - 1,1,1,1,1,1,1,1, # 60 - 67 - 1,1,1,1,1,1,1,1, # 68 - 6f - 1,1,1,1,1,1,1,1, # 70 - 77 - 1,1,1,1,1,1,1,1, # 78 - 7f - 2,2,2,2,3,3,3,3, # 80 - 87 - 4,4,4,4,4,4,4,4, # 88 - 8f - 4,4,4,4,4,4,4,4, # 90 - 97 - 4,4,4,4,4,4,4,4, # 98 - 9f - 5,5,5,5,5,5,5,5, # a0 - a7 - 5,5,5,5,5,5,5,5, # a8 - af - 5,5,5,5,5,5,5,5, # b0 - b7 - 5,5,5,5,5,5,5,5, # b8 - bf - 0,0,6,6,6,6,6,6, # c0 - c7 - 6,6,6,6,6,6,6,6, # c8 - cf - 6,6,6,6,6,6,6,6, # d0 - d7 - 6,6,6,6,6,6,6,6, # d8 - df - 7,8,8,8,8,8,8,8, # e0 - e7 - 8,8,8,8,8,9,8,8, # e8 - ef - 10,11,11,11,11,11,11,11, # f0 - f7 - 12,13,13,13,14,15,0,0) # f8 - ff - -UTF8_st = ( \ - eError,eStart,eError,eError,eError,eError, 12, 10,#00-07 - 9, 11, 8, 7, 6, 5, 4, 3,#08-0f - eError,eError,eError,eError,eError,eError,eError,eError,#10-17 - eError,eError,eError,eError,eError,eError,eError,eError,#18-1f - eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,#20-27 - eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,#28-2f - eError,eError, 5, 5, 5, 5,eError,eError,#30-37 - eError,eError,eError,eError,eError,eError,eError,eError,#38-3f - eError,eError,eError, 5, 5, 5,eError,eError,#40-47 - eError,eError,eError,eError,eError,eError,eError,eError,#48-4f - eError,eError, 7, 7, 7, 7,eError,eError,#50-57 - eError,eError,eError,eError,eError,eError,eError,eError,#58-5f - eError,eError,eError,eError, 7, 7,eError,eError,#60-67 - eError,eError,eError,eError,eError,eError,eError,eError,#68-6f - eError,eError, 9, 9, 9, 9,eError,eError,#70-77 - eError,eError,eError,eError,eError,eError,eError,eError,#78-7f - eError,eError,eError,eError,eError, 9,eError,eError,#80-87 - eError,eError,eError,eError,eError,eError,eError,eError,#88-8f - eError,eError, 12, 12, 12, 12,eError,eError,#90-97 - eError,eError,eError,eError,eError,eError,eError,eError,#98-9f - eError,eError,eError,eError,eError, 12,eError,eError,#a0-a7 - eError,eError,eError,eError,eError,eError,eError,eError,#a8-af - eError,eError, 12, 12, 12,eError,eError,eError,#b0-b7 - eError,eError,eError,eError,eError,eError,eError,eError,#b8-bf - eError,eError,eStart,eStart,eStart,eStart,eError,eError,#c0-c7 - eError,eError,eError,eError,eError,eError,eError,eError)#c8-cf + 1,1,1,1,1,1,0,0, # 08 - 0f + 1,1,1,1,1,1,1,1, # 10 - 17 + 1,1,1,0,1,1,1,1, # 18 - 1f + 1,1,1,1,1,1,1,1, # 20 - 27 + 1,1,1,1,1,1,1,1, # 28 - 2f + 1,1,1,1,1,1,1,1, # 30 - 37 + 1,1,1,1,1,1,1,1, # 38 - 3f + 1,1,1,1,1,1,1,1, # 40 - 47 + 1,1,1,1,1,1,1,1, # 48 - 4f + 1,1,1,1,1,1,1,1, # 50 - 57 + 1,1,1,1,1,1,1,1, # 58 - 5f + 1,1,1,1,1,1,1,1, # 60 - 67 + 1,1,1,1,1,1,1,1, # 68 - 6f + 1,1,1,1,1,1,1,1, # 70 - 77 + 1,1,1,1,1,1,1,1, # 78 - 7f + 2,2,2,2,3,3,3,3, # 80 - 87 + 4,4,4,4,4,4,4,4, # 88 - 8f + 4,4,4,4,4,4,4,4, # 90 - 97 + 4,4,4,4,4,4,4,4, # 98 - 9f + 5,5,5,5,5,5,5,5, # a0 - a7 + 5,5,5,5,5,5,5,5, # a8 - af + 5,5,5,5,5,5,5,5, # b0 - b7 + 5,5,5,5,5,5,5,5, # b8 - bf + 0,0,6,6,6,6,6,6, # c0 - c7 + 6,6,6,6,6,6,6,6, # c8 - cf + 6,6,6,6,6,6,6,6, # d0 - d7 + 6,6,6,6,6,6,6,6, # d8 - df + 7,8,8,8,8,8,8,8, # e0 - e7 + 8,8,8,8,8,9,8,8, # e8 - ef + 10,11,11,11,11,11,11,11, # f0 - f7 + 12,13,13,13,14,15,0,0 # f8 - ff +) + +UTF8_st = ( + eError,eStart,eError,eError,eError,eError, 12, 10,#00-07 + 9, 11, 8, 7, 6, 5, 4, 3,#08-0f + eError,eError,eError,eError,eError,eError,eError,eError,#10-17 + eError,eError,eError,eError,eError,eError,eError,eError,#18-1f + eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,#20-27 + eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,#28-2f + eError,eError, 5, 5, 5, 5,eError,eError,#30-37 + eError,eError,eError,eError,eError,eError,eError,eError,#38-3f + eError,eError,eError, 5, 5, 5,eError,eError,#40-47 + eError,eError,eError,eError,eError,eError,eError,eError,#48-4f + eError,eError, 7, 7, 7, 7,eError,eError,#50-57 + eError,eError,eError,eError,eError,eError,eError,eError,#58-5f + eError,eError,eError,eError, 7, 7,eError,eError,#60-67 + eError,eError,eError,eError,eError,eError,eError,eError,#68-6f + eError,eError, 9, 9, 9, 9,eError,eError,#70-77 + eError,eError,eError,eError,eError,eError,eError,eError,#78-7f + eError,eError,eError,eError,eError, 9,eError,eError,#80-87 + eError,eError,eError,eError,eError,eError,eError,eError,#88-8f + eError,eError, 12, 12, 12, 12,eError,eError,#90-97 + eError,eError,eError,eError,eError,eError,eError,eError,#98-9f + eError,eError,eError,eError,eError, 12,eError,eError,#a0-a7 + eError,eError,eError,eError,eError,eError,eError,eError,#a8-af + eError,eError, 12, 12, 12,eError,eError,eError,#b0-b7 + eError,eError,eError,eError,eError,eError,eError,eError,#b8-bf + eError,eError,eStart,eStart,eStart,eStart,eError,eError,#c0-c7 + eError,eError,eError,eError,eError,eError,eError,eError #c8-cf +) UTF8CharLenTable = (0, 1, 0, 0, 0, 0, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6) @@ -512,3 +571,5 @@ UTF8SMModel = {'classTable': UTF8_cls, 'stateTable': UTF8_st, 'charLenTable': UTF8CharLenTable, 'name': 'UTF-8'} + +# flake8: noqa diff --git a/libs/chardet/sbcharsetprober.py b/libs/chardet/sbcharsetprober.py index da07116..37291bd 100755 --- a/libs/chardet/sbcharsetprober.py +++ b/libs/chardet/sbcharsetprober.py @@ -14,20 +14,22 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants, sys -from charsetprober import CharSetProber +import sys +from . import constants +from .charsetprober import CharSetProber +from .compat import wrap_ord SAMPLE_SIZE = 64 SB_ENOUGH_REL_THRESHOLD = 1024 @@ -37,22 +39,27 @@ SYMBOL_CAT_ORDER = 250 NUMBER_OF_SEQ_CAT = 4 POSITIVE_CAT = NUMBER_OF_SEQ_CAT - 1 #NEGATIVE_CAT = 0 - + + class SingleByteCharSetProber(CharSetProber): - def __init__(self, model, reversed=constants.False, nameProber=None): + def __init__(self, model, reversed=False, nameProber=None): CharSetProber.__init__(self) self._mModel = model - self._mReversed = reversed # TRUE if we need to reverse every pair in the model lookup - self._mNameProber = nameProber # Optional auxiliary prober for name decision + # TRUE if we need to reverse every pair in the model lookup + self._mReversed = reversed + # Optional auxiliary prober for name decision + self._mNameProber = nameProber self.reset() def reset(self): CharSetProber.reset(self) - self._mLastOrder = 255 # char order of last character + # char order of last character + self._mLastOrder = 255 self._mSeqCounters = [0] * NUMBER_OF_SEQ_CAT self._mTotalSeqs = 0 self._mTotalChar = 0 - self._mFreqChar = 0 # characters that fall in our sampling range + # characters that fall in our sampling range + self._mFreqChar = 0 def get_charset_name(self): if self._mNameProber: @@ -67,7 +74,7 @@ class SingleByteCharSetProber(CharSetProber): if not aLen: return self.get_state() for c in aBuf: - order = self._mModel['charToOrderMap'][ord(c)] + order = self._mModel['charToOrderMap'][wrap_ord(c)] if order < SYMBOL_CAT_ORDER: self._mTotalChar += 1 if order < SAMPLE_SIZE: @@ -75,9 +82,12 @@ class SingleByteCharSetProber(CharSetProber): if self._mLastOrder < SAMPLE_SIZE: self._mTotalSeqs += 1 if not self._mReversed: - self._mSeqCounters[self._mModel['precedenceMatrix'][(self._mLastOrder * SAMPLE_SIZE) + order]] += 1 - else: # reverse the order of the letters in the lookup - self._mSeqCounters[self._mModel['precedenceMatrix'][(order * SAMPLE_SIZE) + self._mLastOrder]] += 1 + i = (self._mLastOrder * SAMPLE_SIZE) + order + model = self._mModel['precedenceMatrix'][i] + else: # reverse the order of the letters in the lookup + i = (order * SAMPLE_SIZE) + self._mLastOrder + model = self._mModel['precedenceMatrix'][i] + self._mSeqCounters[model] += 1 self._mLastOrder = order if self.get_state() == constants.eDetecting: @@ -85,11 +95,16 @@ class SingleByteCharSetProber(CharSetProber): cf = self.get_confidence() if cf > POSITIVE_SHORTCUT_THRESHOLD: if constants._debug: - sys.stderr.write('%s confidence = %s, we have a winner\n' % (self._mModel['charsetName'], cf)) + sys.stderr.write('%s confidence = %s, we have a' + 'winner\n' % + (self._mModel['charsetName'], cf)) self._mState = constants.eFoundIt elif cf < NEGATIVE_SHORTCUT_THRESHOLD: if constants._debug: - sys.stderr.write('%s confidence = %s, below negative shortcut threshhold %s\n' % (self._mModel['charsetName'], cf, NEGATIVE_SHORTCUT_THRESHOLD)) + sys.stderr.write('%s confidence = %s, below negative' + 'shortcut threshhold %s\n' % + (self._mModel['charsetName'], cf, + NEGATIVE_SHORTCUT_THRESHOLD)) self._mState = constants.eNotMe return self.get_state() @@ -97,9 +112,8 @@ class SingleByteCharSetProber(CharSetProber): def get_confidence(self): r = 0.01 if self._mTotalSeqs > 0: -# print self._mSeqCounters[POSITIVE_CAT], self._mTotalSeqs, self._mModel['mTypicalPositiveRatio'] - r = (1.0 * self._mSeqCounters[POSITIVE_CAT]) / self._mTotalSeqs / self._mModel['mTypicalPositiveRatio'] -# print r, self._mFreqChar, self._mTotalChar + r = ((1.0 * self._mSeqCounters[POSITIVE_CAT]) / self._mTotalSeqs + / self._mModel['mTypicalPositiveRatio']) r = r * self._mFreqChar / self._mTotalChar if r >= 1.0: r = 0.99 diff --git a/libs/chardet/sbcsgroupprober.py b/libs/chardet/sbcsgroupprober.py index d19160c..1b6196c 100755 --- a/libs/chardet/sbcsgroupprober.py +++ b/libs/chardet/sbcsgroupprober.py @@ -14,33 +14,35 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants, sys -from charsetgroupprober import CharSetGroupProber -from sbcharsetprober import SingleByteCharSetProber -from langcyrillicmodel import Win1251CyrillicModel, Koi8rModel, Latin5CyrillicModel, MacCyrillicModel, Ibm866Model, Ibm855Model -from langgreekmodel import Latin7GreekModel, Win1253GreekModel -from langbulgarianmodel import Latin5BulgarianModel, Win1251BulgarianModel -from langhungarianmodel import Latin2HungarianModel, Win1250HungarianModel -from langthaimodel import TIS620ThaiModel -from langhebrewmodel import Win1255HebrewModel -from hebrewprober import HebrewProber +from .charsetgroupprober import CharSetGroupProber +from .sbcharsetprober import SingleByteCharSetProber +from .langcyrillicmodel import (Win1251CyrillicModel, Koi8rModel, + Latin5CyrillicModel, MacCyrillicModel, + Ibm866Model, Ibm855Model) +from .langgreekmodel import Latin7GreekModel, Win1253GreekModel +from .langbulgarianmodel import Latin5BulgarianModel, Win1251BulgarianModel +from .langhungarianmodel import Latin2HungarianModel, Win1250HungarianModel +from .langthaimodel import TIS620ThaiModel +from .langhebrewmodel import Win1255HebrewModel +from .hebrewprober import HebrewProber + class SBCSGroupProber(CharSetGroupProber): def __init__(self): CharSetGroupProber.__init__(self) - self._mProbers = [ \ + self._mProbers = [ SingleByteCharSetProber(Win1251CyrillicModel), SingleByteCharSetProber(Koi8rModel), SingleByteCharSetProber(Latin5CyrillicModel), @@ -54,11 +56,14 @@ class SBCSGroupProber(CharSetGroupProber): SingleByteCharSetProber(Latin2HungarianModel), SingleByteCharSetProber(Win1250HungarianModel), SingleByteCharSetProber(TIS620ThaiModel), - ] + ] hebrewProber = HebrewProber() - logicalHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, constants.False, hebrewProber) - visualHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, constants.True, hebrewProber) + logicalHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, + False, hebrewProber) + visualHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, True, + hebrewProber) hebrewProber.set_model_probers(logicalHebrewProber, visualHebrewProber) - self._mProbers.extend([hebrewProber, logicalHebrewProber, visualHebrewProber]) + self._mProbers.extend([hebrewProber, logicalHebrewProber, + visualHebrewProber]) self.reset() diff --git a/libs/chardet/sjisprober.py b/libs/chardet/sjisprober.py index fea2690..b173614 100755 --- a/libs/chardet/sjisprober.py +++ b/libs/chardet/sjisprober.py @@ -13,25 +13,26 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -from mbcharsetprober import MultiByteCharSetProber -from codingstatemachine import CodingStateMachine -from chardistribution import SJISDistributionAnalysis -from jpcntx import SJISContextAnalysis -from mbcssm import SJISSMModel -import constants, sys -from constants import eStart, eError, eItsMe +import sys +from .mbcharsetprober import MultiByteCharSetProber +from .codingstatemachine import CodingStateMachine +from .chardistribution import SJISDistributionAnalysis +from .jpcntx import SJISContextAnalysis +from .mbcssm import SJISSMModel +from . import constants + class SJISProber(MultiByteCharSetProber): def __init__(self): @@ -44,7 +45,7 @@ class SJISProber(MultiByteCharSetProber): def reset(self): MultiByteCharSetProber.reset(self) self._mContextAnalyzer.reset() - + def get_charset_name(self): return "SHIFT_JIS" @@ -52,29 +53,34 @@ class SJISProber(MultiByteCharSetProber): aLen = len(aBuf) for i in range(0, aLen): codingState = self._mCodingSM.next_state(aBuf[i]) - if codingState == eError: + if codingState == constants.eError: if constants._debug: - sys.stderr.write(self.get_charset_name() + ' prober hit error at byte ' + str(i) + '\n') + sys.stderr.write(self.get_charset_name() + + ' prober hit error at byte ' + str(i) + + '\n') self._mState = constants.eNotMe break - elif codingState == eItsMe: + elif codingState == constants.eItsMe: self._mState = constants.eFoundIt break - elif codingState == eStart: + elif codingState == constants.eStart: charLen = self._mCodingSM.get_current_charlen() if i == 0: self._mLastChar[1] = aBuf[0] - self._mContextAnalyzer.feed(self._mLastChar[2 - charLen :], charLen) + self._mContextAnalyzer.feed(self._mLastChar[2 - charLen:], + charLen) self._mDistributionAnalyzer.feed(self._mLastChar, charLen) else: - self._mContextAnalyzer.feed(aBuf[i + 1 - charLen : i + 3 - charLen], charLen) - self._mDistributionAnalyzer.feed(aBuf[i - 1 : i + 1], charLen) - + self._mContextAnalyzer.feed(aBuf[i + 1 - charLen:i + 3 + - charLen], charLen) + self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], + charLen) + self._mLastChar[0] = aBuf[aLen - 1] - + if self.get_state() == constants.eDetecting: - if self._mContextAnalyzer.got_enough_data() and \ - (self.get_confidence() > constants.SHORTCUT_THRESHOLD): + if (self._mContextAnalyzer.got_enough_data() and + (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): self._mState = constants.eFoundIt return self.get_state() diff --git a/libs/chardet/universaldetector.py b/libs/chardet/universaldetector.py index 809df22..9a03ad3 100755 --- a/libs/chardet/universaldetector.py +++ b/libs/chardet/universaldetector.py @@ -14,23 +14,25 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants, sys -from latin1prober import Latin1Prober # windows-1252 -from mbcsgroupprober import MBCSGroupProber # multi-byte character sets -from sbcsgroupprober import SBCSGroupProber # single-byte character sets -from escprober import EscCharSetProber # ISO-2122, etc. +from . import constants +import sys +import codecs +from .latin1prober import Latin1Prober # windows-1252 +from .mbcsgroupprober import MBCSGroupProber # multi-byte character sets +from .sbcsgroupprober import SBCSGroupProber # single-byte character sets +from .escprober import EscCharSetProber # ISO-2122, etc. import re MINIMUM_THRESHOLD = 0.20 @@ -38,68 +40,78 @@ ePureAscii = 0 eEscAscii = 1 eHighbyte = 2 + class UniversalDetector: def __init__(self): - self._highBitDetector = re.compile(r'[\x80-\xFF]') - self._escDetector = re.compile(r'(\033|~{)') + self._highBitDetector = re.compile(b'[\x80-\xFF]') + self._escDetector = re.compile(b'(\033|~{)') self._mEscCharSetProber = None self._mCharSetProbers = [] self.reset() def reset(self): self.result = {'encoding': None, 'confidence': 0.0} - self.done = constants.False - self._mStart = constants.True - self._mGotData = constants.False + self.done = False + self._mStart = True + self._mGotData = False self._mInputState = ePureAscii - self._mLastChar = '' + self._mLastChar = b'' if self._mEscCharSetProber: self._mEscCharSetProber.reset() for prober in self._mCharSetProbers: prober.reset() def feed(self, aBuf): - if self.done: return + if self.done: + return aLen = len(aBuf) - if not aLen: return - + if not aLen: + return + if not self._mGotData: # If the data starts with BOM, we know it is UTF - if aBuf[:3] == '\xEF\xBB\xBF': + if aBuf[:3] == codecs.BOM: # EF BB BF UTF-8 with BOM self.result = {'encoding': "UTF-8", 'confidence': 1.0} - elif aBuf[:4] == '\xFF\xFE\x00\x00': + elif aBuf[:4] == codecs.BOM_UTF32_LE: # FF FE 00 00 UTF-32, little-endian BOM self.result = {'encoding': "UTF-32LE", 'confidence': 1.0} - elif aBuf[:4] == '\x00\x00\xFE\xFF': + elif aBuf[:4] == codecs.BOM_UTF32_BE: # 00 00 FE FF UTF-32, big-endian BOM self.result = {'encoding': "UTF-32BE", 'confidence': 1.0} - elif aBuf[:4] == '\xFE\xFF\x00\x00': + elif aBuf[:4] == b'\xFE\xFF\x00\x00': # FE FF 00 00 UCS-4, unusual octet order BOM (3412) - self.result = {'encoding': "X-ISO-10646-UCS-4-3412", 'confidence': 1.0} - elif aBuf[:4] == '\x00\x00\xFF\xFE': + self.result = { + 'encoding': "X-ISO-10646-UCS-4-3412", + 'confidence': 1.0 + } + elif aBuf[:4] == b'\x00\x00\xFF\xFE': # 00 00 FF FE UCS-4, unusual octet order BOM (2143) - self.result = {'encoding': "X-ISO-10646-UCS-4-2143", 'confidence': 1.0} - elif aBuf[:2] == '\xFF\xFE': + self.result = { + 'encoding': "X-ISO-10646-UCS-4-2143", + 'confidence': 1.0 + } + elif aBuf[:2] == codecs.BOM_LE: # FF FE UTF-16, little endian BOM self.result = {'encoding': "UTF-16LE", 'confidence': 1.0} - elif aBuf[:2] == '\xFE\xFF': + elif aBuf[:2] == codecs.BOM_BE: # FE FF UTF-16, big endian BOM self.result = {'encoding': "UTF-16BE", 'confidence': 1.0} - self._mGotData = constants.True + self._mGotData = True if self.result['encoding'] and (self.result['confidence'] > 0.0): - self.done = constants.True + self.done = True return if self._mInputState == ePureAscii: if self._highBitDetector.search(aBuf): self._mInputState = eHighbyte - elif (self._mInputState == ePureAscii) and self._escDetector.search(self._mLastChar + aBuf): + elif ((self._mInputState == ePureAscii) and + self._escDetector.search(self._mLastChar + aBuf)): self._mInputState = eEscAscii - self._mLastChar = aBuf[-1] + self._mLastChar = aBuf[-1:] if self._mInputState == eEscAscii: if not self._mEscCharSetProber: @@ -107,25 +119,27 @@ class UniversalDetector: if self._mEscCharSetProber.feed(aBuf) == constants.eFoundIt: self.result = {'encoding': self._mEscCharSetProber.get_charset_name(), 'confidence': self._mEscCharSetProber.get_confidence()} - self.done = constants.True + self.done = True elif self._mInputState == eHighbyte: if not self._mCharSetProbers: - self._mCharSetProbers = [MBCSGroupProber(), SBCSGroupProber(), Latin1Prober()] + self._mCharSetProbers = [MBCSGroupProber(), SBCSGroupProber(), + Latin1Prober()] for prober in self._mCharSetProbers: if prober.feed(aBuf) == constants.eFoundIt: self.result = {'encoding': prober.get_charset_name(), 'confidence': prober.get_confidence()} - self.done = constants.True + self.done = True break def close(self): - if self.done: return + if self.done: + return if not self._mGotData: if constants._debug: sys.stderr.write('no data received!\n') return - self.done = constants.True - + self.done = True + if self._mInputState == ePureAscii: self.result = {'encoding': 'ascii', 'confidence': 1.0} return self.result @@ -135,7 +149,8 @@ class UniversalDetector: maxProberConfidence = 0.0 maxProber = None for prober in self._mCharSetProbers: - if not prober: continue + if not prober: + continue proberConfidence = prober.get_confidence() if proberConfidence > maxProberConfidence: maxProberConfidence = proberConfidence @@ -148,7 +163,8 @@ class UniversalDetector: if constants._debug: sys.stderr.write('no probers hit minimum threshhold\n') for prober in self._mCharSetProbers[0].mProbers: - if not prober: continue - sys.stderr.write('%s confidence = %s\n' % \ - (prober.get_charset_name(), \ + if not prober: + continue + sys.stderr.write('%s confidence = %s\n' % + (prober.get_charset_name(), prober.get_confidence())) diff --git a/libs/chardet/utf8prober.py b/libs/chardet/utf8prober.py index c1792bb..1c0bb5d 100755 --- a/libs/chardet/utf8prober.py +++ b/libs/chardet/utf8prober.py @@ -13,26 +13,26 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -import constants, sys -from constants import eStart, eError, eItsMe -from charsetprober import CharSetProber -from codingstatemachine import CodingStateMachine -from mbcssm import UTF8SMModel +from . import constants +from .charsetprober import CharSetProber +from .codingstatemachine import CodingStateMachine +from .mbcssm import UTF8SMModel ONE_CHAR_PROB = 0.5 + class UTF8Prober(CharSetProber): def __init__(self): CharSetProber.__init__(self) @@ -50,13 +50,13 @@ class UTF8Prober(CharSetProber): def feed(self, aBuf): for c in aBuf: codingState = self._mCodingSM.next_state(c) - if codingState == eError: + if codingState == constants.eError: self._mState = constants.eNotMe break - elif codingState == eItsMe: + elif codingState == constants.eItsMe: self._mState = constants.eFoundIt break - elif codingState == eStart: + elif codingState == constants.eStart: if self._mCodingSM.get_current_charlen() >= 2: self._mNumOfMBChar += 1 From e1d4df79378a075ead388f06d8c2bbc2029914b1 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 5 Oct 2014 18:46:16 +0200 Subject: [PATCH 181/202] Give api encoding error log --- couchpotato/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/api.py b/couchpotato/api.py index b21cfeb..cd01197 100644 --- a/couchpotato/api.py +++ b/couchpotato/api.py @@ -143,6 +143,8 @@ class ApiHandler(RequestHandler): else: self.write(result) self.finish() + except UnicodeDecodeError: + log.error('Failed proper encode: %s', traceback.format_exc()) except: log.debug('Failed doing request, probably already closed: %s', (traceback.format_exc())) try: self.finish({'success': False, 'error': 'Failed returning results'}) From 14d636d098cb9061c413336ff4f47caac0c54fea Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 5 Oct 2014 18:46:44 +0200 Subject: [PATCH 182/202] Return image filepath in unicode --- couchpotato/core/media/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index 549ed0d..7a178b8 100755 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -95,7 +95,7 @@ class MediaBase(Plugin): if file_type not in existing_files or len(existing_files.get(file_type, [])) == 0: file_path = fireEvent('file.download', url = image, single = True) if file_path: - existing_files[file_type] = [file_path] + existing_files[file_type] = [toUnicode(file_path)] break else: break From c9638ec3fa17d03490e5da16649da5efc8d83888 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 5 Oct 2014 18:47:30 +0200 Subject: [PATCH 183/202] Encode templ download destination --- couchpotato/core/plugins/file.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/file.py b/couchpotato/core/plugins/file.py index db6787d..56c5230 100644 --- a/couchpotato/core/plugins/file.py +++ b/couchpotato/core/plugins/file.py @@ -4,7 +4,7 @@ import traceback from couchpotato import get_db from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent -from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.encoding import toUnicode, ss, sp from couchpotato.core.helpers.variable import md5, getExt, isSubFolder from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin @@ -59,7 +59,7 @@ class FileManager(Plugin): log.error('Failed removing unused file: %s', traceback.format_exc()) def showCacheFile(self, route, **kwargs): - Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), route), StaticFileHandler, {'path': Env.get('cache_dir')})]) + Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), route), StaticFileHandler, {'path': toUnicode(Env.get('cache_dir'))})]) def download(self, url = '', dest = None, overwrite = False, urlopen_kwargs = None): if not urlopen_kwargs: urlopen_kwargs = {} @@ -68,7 +68,9 @@ class FileManager(Plugin): urlopen_kwargs['stream'] = True if not dest: # to Cache - dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) + dest = os.path.join(Env.get('cache_dir'), ss('%s.%s' % (md5(url), getExt(url)))) + + dest = sp(dest) if not overwrite and os.path.isfile(dest): return dest From ba47d7eea762c10e085f78bb79f038eca5d33a8a Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 7 Oct 2014 11:46:42 +0200 Subject: [PATCH 184/202] Use torrent-duplicate if returned from Transmission fix #4014 --- couchpotato/core/downloaders/transmission.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/downloaders/transmission.py b/couchpotato/core/downloaders/transmission.py index d6112a9..0361330 100644 --- a/couchpotato/core/downloaders/transmission.py +++ b/couchpotato/core/downloaders/transmission.py @@ -78,12 +78,14 @@ class Transmission(DownloaderBase): log.error('Failed sending torrent to Transmission') return False + data = remote_torrent.get('torrent-added') or remote_torrent.get('torrent-duplicate') + # Change settings of added torrents if torrent_params: - self.trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params) + self.trpc.set_torrent(data['hashString'], torrent_params) log.info('Torrent sent to Transmission successfully.') - return self.downloadReturnId(remote_torrent['torrent-added']['hashString']) + return self.downloadReturnId(data['hashString']) def test(self): if self.connect() and self.trpc.get_session(): From d4a4bd40a8a4f0bf5a84aa07686b21382cac31e0 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 7 Oct 2014 20:14:17 +0200 Subject: [PATCH 185/202] Always return version info --- couchpotato/core/_base/updater/main.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index 7730d3b..27fbd8b 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -205,19 +205,28 @@ class GitUpdater(BaseUpdater): def getVersion(self): if not self.version: + + hash = None + date = None + branch = self.branch + try: output = self.repo.getHead() # Yes, please log.debug('Git version output: %s', output.hash) - self.version = { - 'repr': 'git:(%s:%s % s) %s (%s)' % (self.repo_user, self.repo_name, self.repo.getCurrentBranch().name or self.branch, output.hash[:8], datetime.fromtimestamp(output.getDate())), - 'hash': output.hash[:8], - 'date': output.getDate(), - 'type': 'git', - 'branch': self.repo.getCurrentBranch().name - } + + hash = output.hash[:8] + date = output.getDate() + branch = self.repo.getCurrentBranch().name except Exception as e: log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s', e) - return 'No GIT' + + self.version = { + 'repr': 'git:(%s:%s % s) %s (%s)' % (self.repo_user, self.repo_name, branch, hash or 'unknown_hash', datetime.fromtimestamp(date) if date else 'unknown_date'), + 'hash': hash, + 'date': date, + 'type': 'git', + 'branch': branch + } return self.version From 2104cb283949cd8b065f0d4ed4ea2dba1d1559b2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 7 Oct 2014 20:30:51 +0200 Subject: [PATCH 186/202] Always try to return version string --- couchpotato/core/_base/_core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/_base/_core.py b/couchpotato/core/_base/_core.py index 6bfb441..78d5471 100644 --- a/couchpotato/core/_base/_core.py +++ b/couchpotato/core/_base/_core.py @@ -181,13 +181,13 @@ class Core(Plugin): return '%sapi/%s' % (self.createBaseUrl(), Env.setting('api_key')) def version(self): - ver = fireEvent('updater.info', single = True) + ver = fireEvent('updater.info', single = True) or {'version': {}} if os.name == 'nt': platf = 'windows' elif 'Darwin' in platform.platform(): platf = 'osx' else: platf = 'linux' - return '%s - %s-%s - v2' % (platf, ver.get('version')['type'], ver.get('version')['hash']) + return '%s - %s-%s - v2' % (platf, ver.get('version').get('type') or 'unknown', ver.get('version').get('hash') or 'unknown') def versionView(self, **kwargs): return { From 4cdb9bc81d518a020f40d64c808e6bf8c47e0cd2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 6 Oct 2014 16:43:46 +0200 Subject: [PATCH 187/202] Remove tmdb3 dependency --- .../core/media/movie/providers/info/themoviedb.py | 136 +++++++++++---------- 1 file changed, 74 insertions(+), 62 deletions(-) diff --git a/couchpotato/core/media/movie/providers/info/themoviedb.py b/couchpotato/core/media/movie/providers/info/themoviedb.py index ac1daec..2073383 100644 --- a/couchpotato/core/media/movie/providers/info/themoviedb.py +++ b/couchpotato/core/media/movie/providers/info/themoviedb.py @@ -1,11 +1,11 @@ import traceback +import time -from couchpotato.core.event import addEvent -from couchpotato.core.helpers.encoding import simplifyString, toUnicode, ss +from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.helpers.encoding import simplifyString, toUnicode, ss, tryUrlencode from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.media.movie.providers.base import MovieProvider -import tmdb3 log = CPLog(__name__) @@ -13,15 +13,24 @@ autoload = 'TheMovieDb' class TheMovieDb(MovieProvider): - MAX_EXTRATHUMBS = 4 + + http_time_between_calls = .3 + + configuration = { + 'images': { + 'secure_base_url': 'https://image.tmdb.org/t/p/', + }, + } def __init__(self): addEvent('movie.info', self.getInfo, priority = 3) addEvent('movie.info_by_tmdb', self.getInfo) + addEvent('app.load', self.config) - # Configure TMDB settings - tmdb3.set_key(self.conf('api_key')) - tmdb3.set_cache('null') + def config(self): + configuration = self.request('configuration') + if configuration: + self.configuration = configuration def search(self, q, limit = 12): """ Find movie by name """ @@ -31,14 +40,19 @@ class TheMovieDb(MovieProvider): search_string = simplifyString(q) cache_key = 'tmdb.cache.%s.%s' % (search_string, limit) - results = self.getCache(cache_key) + results = None #self.getCache(cache_key) if not results: log.debug('Searching for movie: %s', q) raw = None try: - raw = tmdb3.searchMovie(search_string) + + #name_year = fireEvent('scanner.name_year', q, single = True) + + raw = self.request('search/movie', { + 'query': q + }, return_key = 'results') except: log.error('Failed searching TMDB for "%s": %s', (search_string, traceback.format_exc())) @@ -69,39 +83,27 @@ class TheMovieDb(MovieProvider): if not identifier: return {} - cache_key = 'tmdb.cache.%s%s' % (identifier, '.ex' if extended else '') - result = self.getCache(cache_key) - - if not result: - try: - log.debug('Getting info: %s', cache_key) - # noinspection PyArgumentList - movie = tmdb3.Movie(identifier) - try: exists = movie.title is not None - except: exists = False - - if exists: - result = self.parseMovie(movie, extended = extended) - self.setCache(cache_key, result) - else: - result = {} - except: - log.error('Failed getting info for %s: %s', (identifier, traceback.format_exc())) + result = self.parseMovie({ + 'id': identifier + }, extended = extended) return result def parseMovie(self, movie, extended = True): - cache_key = 'tmdb.cache.%s%s' % (movie.id, '.ex' if extended else '') - movie_data = self.getCache(cache_key) + cache_key = 'tmdb.cache.%s%s' % (movie.get('id'), '.ex' if extended else '') + movie_data = None #self.getCache(cache_key) if not movie_data: + # Full data + movie = self.request('movie/%s' % movie.get('id')) + # Images poster = self.getImage(movie, type = 'poster', size = 'w154') poster_original = self.getImage(movie, type = 'poster', size = 'original') backdrop_original = self.getImage(movie, type = 'backdrop', size = 'original') - extra_thumbs = self.getMultImages(movie, type = 'backdrops', size = 'original', n = self.MAX_EXTRATHUMBS, skipfirst = True) + extra_thumbs = self.getMultImages(movie, type = 'backdrops', size = 'original') images = { 'poster': [poster] if poster else [], @@ -114,39 +116,43 @@ class TheMovieDb(MovieProvider): # Genres try: - genres = [genre.name for genre in movie.genres] + genres = [genre.get('name') for genre in movie.get('genres', [])] except: genres = [] # 1900 is the same as None - year = str(movie.releasedate or '')[:4] - if not movie.releasedate or year == '1900' or year.lower() == 'none': + year = str(movie.get('release_date') or '')[:4] + if not movie.get('release_date') or year == '1900' or year.lower() == 'none': year = None # Gather actors data actors = {} if extended: - for cast_item in movie.cast: + + # Full data + cast = self.request('movie/%s/casts' % movie.get('id'), return_key = 'cast') + + for cast_item in cast: try: - actors[toUnicode(cast_item.name)] = toUnicode(cast_item.character) - images['actors'][toUnicode(cast_item.name)] = self.getImage(cast_item, type = 'profile', size = 'original') + actors[toUnicode(cast_item.get('name'))] = toUnicode(cast_item.get('character')) + images['actors'][toUnicode(cast_item.get('name'))] = self.getImage(cast_item, type = 'profile', size = 'original') except: log.debug('Error getting cast info for %s: %s', (cast_item, traceback.format_exc())) movie_data = { 'type': 'movie', 'via_tmdb': True, - 'tmdb_id': movie.id, - 'titles': [toUnicode(movie.title)], - 'original_title': movie.originaltitle, + 'tmdb_id': movie.get('id'), + 'titles': [toUnicode(movie.get('title'))], + 'original_title': movie.get('original_title'), 'images': images, - 'imdb': movie.imdb, - 'runtime': movie.runtime, - 'released': str(movie.releasedate), + 'imdb': movie.get('imdb_id'), + 'runtime': movie.get('runtime'), + 'released': str(movie.get('release_date')), 'year': tryInt(year, None), - 'plot': movie.overview, + 'plot': movie.get('overview'), 'genres': genres, - 'collection': getattr(movie.collection, 'name', None), + 'collection': getattr(movie.get('belongs_to_collection'), 'name', None), 'actor_roles': actors } @@ -157,8 +163,12 @@ class TheMovieDb(MovieProvider): movie_data['titles'].append(movie_data['original_title']) if extended: - for alt in movie.alternate_titles: - alt_name = alt.title + + # Full data + alternate_titles = self.request('movie/%s/alternative_titles' % movie.get('id'), return_key = 'titles') + + for alt in alternate_titles: + alt_name = alt.get('title') if alt_name and alt_name not in movie_data['titles'] and alt_name.lower() != 'none' and alt_name is not None: movie_data['titles'].append(alt_name) @@ -171,36 +181,38 @@ class TheMovieDb(MovieProvider): image_url = '' try: - image_url = getattr(movie, type).geturl(size = size) + path = movie.get('%s_path' % type) + image_url = '%s%s%s' % (self.configuration['images']['secure_base_url'], size, path) except: log.debug('Failed getting %s.%s for "%s"', (type, size, ss(str(movie)))) return image_url - def getMultImages(self, movie, type = 'backdrops', size = 'original', n = -1, skipfirst = False): - """ - If n < 0, return all images. Otherwise return n images. - If n > len(getattr(movie, type)), then return all images. - If skipfirst is True, then it will skip getattr(movie, type)[0]. This - is because backdrops[0] is typically backdrop. - """ + def getMultImages(self, movie, type = 'backdrops', size = 'original'): image_urls = [] try: - images = getattr(movie, type) - if n < 0 or n > len(images): - num_images = len(images) - else: - num_images = n - for i in range(int(skipfirst), num_images + int(skipfirst)): - image_urls.append(images[i].geturl(size = size)) + # Full data + images = self.request('movie/%s/images' % movie.get('id'), return_key = type) + for image in images[1:5]: + image_urls.append(self.getImage(image, 'file', size)) except: - log.debug('Failed getting %i %s.%s for "%s"', (n, type, size, ss(str(movie)))) + log.debug('Failed getting %s.%s for "%s"', (type, size, ss(str(movie)))) return image_urls + def request(self, call = '', params = {}, return_key = None): + params = tryUrlencode(params) + url = 'http://api.themoviedb.org/3/%s?api_key=%s%s' % (call, self.conf('api_key'), '&%s' % params if params else '') + data = self.getJsonData(url, cache_timeout = 0) + + if data and return_key and data.get(return_key): + data = data.get(return_key) + + return data + def isDisabled(self): if self.conf('api_key') == '': log.error('No API key provided.') From 38a5d967dd843eee614412538f69f958028b3d01 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 6 Oct 2014 16:44:22 +0200 Subject: [PATCH 188/202] Remove tmdb3 lib --- libs/tmdb3/__init__.py | 12 - libs/tmdb3/cache.py | 130 ------ libs/tmdb3/cache_engine.py | 84 ---- libs/tmdb3/cache_file.py | 400 ------------------- libs/tmdb3/cache_null.py | 27 -- libs/tmdb3/locales.py | 642 ----------------------------- libs/tmdb3/pager.py | 116 ------ libs/tmdb3/request.py | 167 -------- libs/tmdb3/tmdb_api.py | 910 ------------------------------------------ libs/tmdb3/tmdb_auth.py | 138 ------- libs/tmdb3/tmdb_exceptions.py | 107 ----- libs/tmdb3/util.py | 403 ------------------- 12 files changed, 3136 deletions(-) delete mode 100755 libs/tmdb3/__init__.py delete mode 100755 libs/tmdb3/cache.py delete mode 100755 libs/tmdb3/cache_engine.py delete mode 100755 libs/tmdb3/cache_file.py delete mode 100755 libs/tmdb3/cache_null.py delete mode 100755 libs/tmdb3/locales.py delete mode 100755 libs/tmdb3/pager.py delete mode 100755 libs/tmdb3/request.py delete mode 100755 libs/tmdb3/tmdb_api.py delete mode 100755 libs/tmdb3/tmdb_auth.py delete mode 100755 libs/tmdb3/tmdb_exceptions.py delete mode 100755 libs/tmdb3/util.py diff --git a/libs/tmdb3/__init__.py b/libs/tmdb3/__init__.py deleted file mode 100755 index d5e35b3..0000000 --- a/libs/tmdb3/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python - -from tmdb_api import Configuration, searchMovie, searchMovieWithYear, \ - searchPerson, searchStudio, searchList, searchCollection, \ - searchSeries, Person, Movie, Collection, Genre, List, \ - Series, Studio, Network, Episode, Season, __version__ -from request import set_key, set_cache -from locales import get_locale, set_locale -from tmdb_auth import get_session, set_session -from cache_engine import CacheEngine -from tmdb_exceptions import * - diff --git a/libs/tmdb3/cache.py b/libs/tmdb3/cache.py deleted file mode 100755 index 463d7a2..0000000 --- a/libs/tmdb3/cache.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -#----------------------- -# Name: cache.py -# Python Library -# Author: Raymond Wagner -# Purpose: Caching framework to store TMDb API results -#----------------------- - -import time -import os - -from tmdb_exceptions import * -from cache_engine import Engines - -import cache_null -import cache_file - - -class Cache(object): - """ - This class implements a cache framework, allowing selecting of a - pluggable engine. The framework stores data in a key/value manner, - along with a lifetime, after which data will be expired and - pulled fresh next time it is requested from the cache. - - This class defines a wrapper to be used with query functions. The - wrapper will automatically cache the inputs and outputs of the - wrapped function, pulling the output from local storage for - subsequent calls with those inputs. - """ - def __init__(self, engine=None, *args, **kwargs): - self._engine = None - self._data = {} - self._age = 0 - self.configure(engine, *args, **kwargs) - - def _import(self, data=None): - if data is None: - data = self._engine.get(self._age) - for obj in sorted(data, key=lambda x: x.creation): - if not obj.expired: - self._data[obj.key] = obj - self._age = max(self._age, obj.creation) - - def _expire(self): - for k, v in self._data.items(): - if v.expired: - del self._data[k] - - def configure(self, engine, *args, **kwargs): - if engine is None: - engine = 'file' - elif engine not in Engines: - raise TMDBCacheError("Invalid cache engine specified: "+engine) - self._engine = Engines[engine](self) - self._engine.configure(*args, **kwargs) - - def put(self, key, data, lifetime=60*60*12): - # pull existing data, so cache will be fresh when written back out - if self._engine is None: - raise TMDBCacheError("No cache engine configured") - self._expire() - self._import(self._engine.put(key, data, lifetime)) - - def get(self, key): - if self._engine is None: - raise TMDBCacheError("No cache engine configured") - self._expire() - if key not in self._data: - self._import() - try: - return self._data[key].data - except: - return None - - def cached(self, callback): - """ - Returns a decorator that uses a callback to specify the key to use - for caching the responses from the decorated function. - """ - return self.Cached(self, callback) - - class Cached( object ): - def __init__(self, cache, callback, func=None, inst=None): - self.cache = cache - self.callback = callback - self.func = func - self.inst = inst - - if func: - self.__module__ = func.__module__ - self.__name__ = func.__name__ - self.__doc__ = func.__doc__ - - def __call__(self, *args, **kwargs): - if self.func is None: - # decorator is waiting to be given a function - if len(kwargs) or (len(args) != 1): - raise TMDBCacheError( - 'Cache.Cached decorator must be called a single ' + - 'callable argument before it be used.') - elif args[0] is None: - raise TMDBCacheError( - 'Cache.Cached decorator called before being given ' + - 'a function to wrap.') - elif not callable(args[0]): - raise TMDBCacheError( - 'Cache.Cached must be provided a callable object.') - return self.__class__(self.cache, self.callback, args[0]) - elif self.inst.lifetime == 0: - # lifetime of zero means never cache - return self.func(*args, **kwargs) - else: - key = self.callback() - data = self.cache.get(key) - if data is None: - data = self.func(*args, **kwargs) - if hasattr(self.inst, 'lifetime'): - self.cache.put(key, data, self.inst.lifetime) - else: - self.cache.put(key, data) - return data - - def __get__(self, inst, owner): - if inst is None: - return self - func = self.func.__get__(inst, owner) - callback = self.callback.__get__(inst, owner) - return self.__class__(self.cache, callback, func, inst) diff --git a/libs/tmdb3/cache_engine.py b/libs/tmdb3/cache_engine.py deleted file mode 100755 index 1101955..0000000 --- a/libs/tmdb3/cache_engine.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -#----------------------- -# Name: cache_engine.py -# Python Library -# Author: Raymond Wagner -# Purpose: Base cache engine class for collecting registered engines -#----------------------- - -import time -from weakref import ref - - -class Engines(object): - """ - Static collector for engines to register against. - """ - def __init__(self): - self._engines = {} - - def register(self, engine): - self._engines[engine.__name__] = engine - self._engines[engine.name] = engine - - def __getitem__(self, key): - return self._engines[key] - - def __contains__(self, key): - return self._engines.__contains__(key) - -Engines = Engines() - - -class CacheEngineType(type): - """ - Cache Engine Metaclass that registers new engines against the cache - for named selection and use. - """ - def __init__(cls, name, bases, attrs): - super(CacheEngineType, cls).__init__(name, bases, attrs) - if name != 'CacheEngine': - # skip base class - Engines.register(cls) - - -class CacheEngine(object): - __metaclass__ = CacheEngineType - name = 'unspecified' - - def __init__(self, parent): - self.parent = ref(parent) - - def configure(self): - raise RuntimeError - def get(self, date): - raise RuntimeError - def put(self, key, value, lifetime): - raise RuntimeError - def expire(self, key): - raise RuntimeError - - -class CacheObject(object): - """ - Cache object class, containing one stored record. - """ - - def __init__(self, key, data, lifetime=0, creation=None): - self.key = key - self.data = data - self.lifetime = lifetime - self.creation = creation if creation is not None else time.time() - - def __len__(self): - return len(self.data) - - @property - def expired(self): - return self.remaining == 0 - - @property - def remaining(self): - return max((self.creation + self.lifetime) - time.time(), 0) - diff --git a/libs/tmdb3/cache_file.py b/libs/tmdb3/cache_file.py deleted file mode 100755 index 4e96581..0000000 --- a/libs/tmdb3/cache_file.py +++ /dev/null @@ -1,400 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -#----------------------- -# Name: cache_file.py -# Python Library -# Author: Raymond Wagner -# Purpose: Persistant file-backed cache using /tmp/ to share data -# using flock or msvcrt.locking to allow safe concurrent -# access. -#----------------------- - -import struct -import errno -import json -import time -import os -import io - -from cStringIO import StringIO - -from tmdb_exceptions import * -from cache_engine import CacheEngine, CacheObject - -#################### -# Cache File Format -#------------------ -# cache version (2) unsigned short -# slot count (2) unsigned short -# slot 0: timestamp (8) double -# slot 0: lifetime (4) unsigned int -# slot 0: seek point (4) unsigned int -# slot 1: timestamp -# slot 1: lifetime index slots are IDd by their query date and -# slot 1: seek point are filled incrementally forwards. lifetime -# .... is how long after query date before the item -# .... expires, and seek point is the location of the -# slot N-2: timestamp start of data for that entry. 256 empty slots -# slot N-2: lifetime are pre-allocated, allowing fast updates. -# slot N-2: seek point when all slots are filled, the cache file is -# slot N-1: timestamp rewritten from scrach to add more slots. -# slot N-1: lifetime -# slot N-1: seek point -# block 1 (?) ASCII -# block 2 -# .... blocks are just simple ASCII text, generated -# .... as independent objects by the JSON encoder -# block N-2 -# block N-1 -# -#################### - - -def _donothing(*args, **kwargs): - pass - -try: - import fcntl - class Flock(object): - """ - Context manager to flock file for the duration the object - exists. Referenced file will be automatically unflocked as the - interpreter exits the context. - Supports an optional callback to process the error and optionally - suppress it. - """ - LOCK_EX = fcntl.LOCK_EX - LOCK_SH = fcntl.LOCK_SH - - def __init__(self, fileobj, operation, callback=None): - self.fileobj = fileobj - self.operation = operation - self.callback = callback - - def __enter__(self): - fcntl.flock(self.fileobj, self.operation) - - def __exit__(self, exc_type, exc_value, exc_tb): - suppress = False - if callable(self.callback): - suppress = self.callback(exc_type, exc_value, exc_tb) - fcntl.flock(self.fileobj, fcntl.LOCK_UN) - return suppress - - def parse_filename(filename): - if '$' in filename: - # replace any environmental variables - filename = os.path.expandvars(filename) - if filename.startswith('~'): - # check for home directory - return os.path.expanduser(filename) - elif filename.startswith('/'): - # check for absolute path - return filename - # return path with temp directory prepended - return '/tmp/' + filename - -except ImportError: - import msvcrt - class Flock( object ): - LOCK_EX = msvcrt.LK_LOCK - LOCK_SH = msvcrt.LK_LOCK - - def __init__(self, fileobj, operation, callback=None): - self.fileobj = fileobj - self.operation = operation - self.callback = callback - - def __enter__(self): - self.size = os.path.getsize(self.fileobj.name) - msvcrt.locking(self.fileobj.fileno(), self.operation, self.size) - - def __exit__(self, exc_type, exc_value, exc_tb): - suppress = False - if callable(self.callback): - suppress = self.callback(exc_type, exc_value, exc_tb) - msvcrt.locking(self.fileobj.fileno(), msvcrt.LK_UNLCK, self.size) - return suppress - - def parse_filename(filename): - if '%' in filename: - # replace any environmental variables - filename = os.path.expandvars(filename) - if filename.startswith('~'): - # check for home directory - return os.path.expanduser(filename) - elif (ord(filename[0]) in (range(65, 91) + range(99, 123))) \ - and (filename[1:3] == ':\\'): - # check for absolute drive path (e.g. C:\...) - return filename - elif (filename.count('\\') >= 3) and (filename.startswith('\\\\')): - # check for absolute UNC path (e.g. \\server\...) - return filename - # return path with temp directory prepended - return os.path.expandvars(os.path.join('%TEMP%', filename)) - - -class FileCacheObject(CacheObject): - _struct = struct.Struct('dII') # double and two ints - # timestamp, lifetime, position - - @classmethod - def fromFile(cls, fd): - dat = cls._struct.unpack(fd.read(cls._struct.size)) - obj = cls(None, None, dat[1], dat[0]) - obj.position = dat[2] - return obj - - def __init__(self, *args, **kwargs): - self._key = None - self._data = None - self._size = None - self._buff = StringIO() - super(FileCacheObject, self).__init__(*args, **kwargs) - - @property - def size(self): - if self._size is None: - self._buff.seek(0, 2) - size = self._buff.tell() - if size == 0: - if (self._key is None) or (self._data is None): - raise RuntimeError - json.dump([self.key, self.data], self._buff) - self._size = self._buff.tell() - self._size = size - return self._size - - @size.setter - def size(self, value): - self._size = value - - @property - def key(self): - if self._key is None: - try: - self._key, self._data = json.loads(self._buff.getvalue()) - except: - pass - return self._key - - @key.setter - def key(self, value): - self._key = value - - @property - def data(self): - if self._data is None: - self._key, self._data = json.loads(self._buff.getvalue()) - return self._data - - @data.setter - def data(self, value): - self._data = value - - def load(self, fd): - fd.seek(self.position) - self._buff.seek(0) - self._buff.write(fd.read(self.size)) - - def dumpslot(self, fd): - pos = fd.tell() - fd.write(self._struct.pack(self.creation, self.lifetime, self.position)) - - def dumpdata(self, fd): - self.size - fd.seek(self.position) - fd.write(self._buff.getvalue()) - - -class FileEngine( CacheEngine ): - """Simple file-backed engine.""" - name = 'file' - _struct = struct.Struct('HH') # two shorts for version and count - _version = 2 - - def __init__(self, parent): - super(FileEngine, self).__init__(parent) - self.configure(None) - - def configure(self, filename, preallocate=256): - self.preallocate = preallocate - self.cachefile = filename - self.size = 0 - self.free = 0 - self.age = 0 - - def _init_cache(self): - # only run this once - self._init_cache = _donothing - - if self.cachefile is None: - raise TMDBCacheError("No cache filename given.") - self.cachefile = parse_filename(self.cachefile) - - try: - # attempt to read existing cache at filename - # handle any errors that occur - self._open('r+b') - # seems to have read fine, make sure we have write access - if not os.access(self.cachefile, os.W_OK): - raise TMDBCacheWriteError(self.cachefile) - - except IOError as e: - if e.errno == errno.ENOENT: - # file does not exist, create a new one - try: - self._open('w+b') - self._write([]) - except IOError as e: - if e.errno == errno.ENOENT: - # directory does not exist - raise TMDBCacheDirectoryError(self.cachefile) - elif e.errno == errno.EACCES: - # user does not have rights to create new file - raise TMDBCacheWriteError(self.cachefile) - else: - # let the unhandled error continue through - raise - elif e.errno == errno.EACCES: - # file exists, but we do not have permission to access it - raise TMDBCacheReadError(self.cachefile) - else: - # let the unhandled error continue through - raise - - def get(self, date): - self._init_cache() - self._open('r+b') - - with Flock(self.cachefd, Flock.LOCK_SH): - # return any new objects in the cache - return self._read(date) - - def put(self, key, value, lifetime): - self._init_cache() - self._open('r+b') - - with Flock(self.cachefd, Flock.LOCK_EX): - newobjs = self._read(self.age) - newobjs.append(FileCacheObject(key, value, lifetime)) - - # this will cause a new file object to be opened with the proper - # access mode, however the Flock should keep the old object open - # and properly locked - self._open('r+b') - self._write(newobjs) - return newobjs - - def _open(self, mode='r+b'): - # enforce binary operation - try: - if self.cachefd.mode == mode: - # already opened in requested mode, nothing to do - self.cachefd.seek(0) - return - except: - pass # catch issue of no cachefile yet opened - self.cachefd = io.open(self.cachefile, mode) - - def _read(self, date): - try: - self.cachefd.seek(0) - version, count = self._struct.unpack(\ - self.cachefd.read(self._struct.size)) - if version != self._version: - # old version, break out and well rewrite when finished - raise Exception - - self.size = count - cache = [] - while count: - # loop through storage definitions - obj = FileCacheObject.fromFile(self.cachefd) - cache.append(obj) - count -= 1 - - except: - # failed to read information, so just discard it and return empty - self.size = 0 - self.free = 0 - return [] - - # get end of file - self.cachefd.seek(0, 2) - position = self.cachefd.tell() - newobjs = [] - emptycount = 0 - - # walk backward through all, collecting new content and populating size - while len(cache): - obj = cache.pop() - if obj.creation == 0: - # unused slot, skip - emptycount += 1 - elif obj.expired: - # object has passed expiration date, no sense processing - continue - elif obj.creation > date: - # used slot with new data, process - obj.size, position = position - obj.position, obj.position - newobjs.append(obj) - # update age - self.age = max(self.age, obj.creation) - elif len(newobjs): - # end of new data, break - break - - # walk forward and load new content - for obj in newobjs: - obj.load(self.cachefd) - - self.free = emptycount - return newobjs - - def _write(self, data): - if self.free and (self.size != self.free): - # we only care about the last data point, since the rest are - # already stored in the file - data = data[-1] - - # determine write position of data in cache - self.cachefd.seek(0, 2) - end = self.cachefd.tell() - data.position = end - - # write incremental update to free slot - self.cachefd.seek(4 + 16*(self.size-self.free)) - data.dumpslot(self.cachefd) - data.dumpdata(self.cachefd) - - else: - # rewrite cache file from scratch - # pull data from parent cache - data.extend(self.parent()._data.values()) - data.sort(key=lambda x: x.creation) - # write header - size = len(data) + self.preallocate - self.cachefd.seek(0) - self.cachefd.truncate() - self.cachefd.write(self._struct.pack(self._version, size)) - # write storage slot definitions - prev = None - for d in data: - if prev == None: - d.position = 4 + 16*size - else: - d.position = prev.position + prev.size - d.dumpslot(self.cachefd) - prev = d - # fill in allocated slots - for i in range(2**8): - self.cachefd.write(FileCacheObject._struct.pack(0, 0, 0)) - # write stored data - for d in data: - d.dumpdata(self.cachefd) - - self.cachefd.flush() - - def expire(self, key): - pass diff --git a/libs/tmdb3/cache_null.py b/libs/tmdb3/cache_null.py deleted file mode 100755 index 8c360da..0000000 --- a/libs/tmdb3/cache_null.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -#----------------------- -# Name: cache_null.py -# Python Library -# Author: Raymond Wagner -# Purpose: Null caching engine for debugging purposes -#----------------------- - -from cache_engine import CacheEngine - - -class NullEngine(CacheEngine): - """Non-caching engine for debugging.""" - name = 'null' - - def configure(self): - pass - - def get(self, date): - return [] - - def put(self, key, value, lifetime): - return [] - - def expire(self, key): - pass diff --git a/libs/tmdb3/locales.py b/libs/tmdb3/locales.py deleted file mode 100755 index 0ef0310..0000000 --- a/libs/tmdb3/locales.py +++ /dev/null @@ -1,642 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -#----------------------- -# Name: locales.py Stores locale information for filtering results -# Python Library -# Author: Raymond Wagner -#----------------------- - -from tmdb_exceptions import * -import locale - -syslocale = None - - -class LocaleBase(object): - __slots__ = ['__immutable'] - _stored = {} - fallthrough = False - - def __init__(self, *keys): - for key in keys: - self._stored[key.lower()] = self - self.__immutable = True - - def __setattr__(self, key, value): - if getattr(self, '__immutable', False): - raise NotImplementedError(self.__class__.__name__ + - ' does not support modification.') - super(LocaleBase, self).__setattr__(key, value) - - def __delattr__(self, key): - if getattr(self, '__immutable', False): - raise NotImplementedError(self.__class__.__name__ + - ' does not support modification.') - super(LocaleBase, self).__delattr__(key) - - def __lt__(self, other): - return (id(self) != id(other)) and (str(self) > str(other)) - - def __gt__(self, other): - return (id(self) != id(other)) and (str(self) < str(other)) - - def __eq__(self, other): - return (id(self) == id(other)) or (str(self) == str(other)) - - @classmethod - def getstored(cls, key): - if key is None: - return None - try: - return cls._stored[key.lower()] - except: - raise TMDBLocaleError("'{0}' is not a known valid {1} code."\ - .format(key, cls.__name__)) - - -class Language(LocaleBase): - __slots__ = ['ISO639_1', 'ISO639_2', 'ISO639_2B', 'englishname', - 'nativename'] - _stored = {} - - def __init__(self, iso1, iso2, ename): - self.ISO639_1 = iso1 - self.ISO639_2 = iso2 -# self.ISO639_2B = iso2b - self.englishname = ename -# self.nativename = nname - super(Language, self).__init__(iso1, iso2) - - def __str__(self): - return self.ISO639_1 - - def __repr__(self): - return u"".format(self) - - -class Country(LocaleBase): - __slots__ = ['alpha2', 'name'] - _stored = {} - - def __init__(self, alpha2, name): - self.alpha2 = alpha2 - self.name = name - super(Country, self).__init__(alpha2) - - def __str__(self): - return self.alpha2 - - def __repr__(self): - return u"".format(self) - - -class Locale(LocaleBase): - __slots__ = ['language', 'country', 'encoding'] - - def __init__(self, language, country, encoding): - self.language = Language.getstored(language) - self.country = Country.getstored(country) - self.encoding = encoding if encoding else 'latin-1' - - def __str__(self): - return u"{0}_{1}".format(self.language, self.country) - - def __repr__(self): - return u"".format(self) - - def encode(self, dat): - """Encode using system default encoding for network/file output.""" - try: - return dat.encode(self.encoding) - except AttributeError: - # not a string type, pass along - return dat - except UnicodeDecodeError: - # just return unmodified and hope for the best - return dat - - def decode(self, dat): - """Decode to system default encoding for internal use.""" - try: - return dat.decode(self.encoding) - except AttributeError: - # not a string type, pass along - return dat - except UnicodeEncodeError: - # just return unmodified and hope for the best - return dat - - -def set_locale(language=None, country=None, fallthrough=False): - global syslocale - LocaleBase.fallthrough = fallthrough - - sysloc, sysenc = locale.getdefaultlocale() - - if (not language) or (not country): - dat = None - if syslocale is not None: - dat = (str(syslocale.language), str(syslocale.country)) - else: - if (sysloc is None) or ('_' not in sysloc): - dat = ('en', 'US') - else: - dat = sysloc.split('_') - if language is None: - language = dat[0] - if country is None: - country = dat[1] - - syslocale = Locale(language, country, sysenc) - - -def get_locale(language=-1, country=-1): - """Output locale using provided attributes, or return system locale.""" - global syslocale - # pull existing stored values - if syslocale is None: - loc = Locale(None, None, locale.getdefaultlocale()[1]) - else: - loc = syslocale - - # both options are default, return stored values - if language == country == -1: - return loc - - # supplement default option with stored values - if language == -1: - language = loc.language - elif country == -1: - country = loc.country - return Locale(language, country, loc.encoding) - -######## AUTOGENERATED LANGUAGE AND COUNTRY DATA BELOW HERE ######### - -Language("ab", "abk", u"Abkhazian") -Language("aa", "aar", u"Afar") -Language("af", "afr", u"Afrikaans") -Language("ak", "aka", u"Akan") -Language("sq", "alb/sqi", u"Albanian") -Language("am", "amh", u"Amharic") -Language("ar", "ara", u"Arabic") -Language("an", "arg", u"Aragonese") -Language("hy", "arm/hye", u"Armenian") -Language("as", "asm", u"Assamese") -Language("av", "ava", u"Avaric") -Language("ae", "ave", u"Avestan") -Language("ay", "aym", u"Aymara") -Language("az", "aze", u"Azerbaijani") -Language("bm", "bam", u"Bambara") -Language("ba", "bak", u"Bashkir") -Language("eu", "baq/eus", u"Basque") -Language("be", "bel", u"Belarusian") -Language("bn", "ben", u"Bengali") -Language("bh", "bih", u"Bihari languages") -Language("bi", "bis", u"Bislama") -Language("nb", "nob", u"Bokmål, Norwegian") -Language("bs", "bos", u"Bosnian") -Language("br", "bre", u"Breton") -Language("bg", "bul", u"Bulgarian") -Language("my", "bur/mya", u"Burmese") -Language("es", "spa", u"Castilian") -Language("ca", "cat", u"Catalan") -Language("km", "khm", u"Central Khmer") -Language("ch", "cha", u"Chamorro") -Language("ce", "che", u"Chechen") -Language("ny", "nya", u"Chewa") -Language("ny", "nya", u"Chichewa") -Language("zh", "chi/zho", u"Chinese") -Language("za", "zha", u"Chuang") -Language("cu", "chu", u"Church Slavic") -Language("cu", "chu", u"Church Slavonic") -Language("cv", "chv", u"Chuvash") -Language("kw", "cor", u"Cornish") -Language("co", "cos", u"Corsican") -Language("cr", "cre", u"Cree") -Language("hr", "hrv", u"Croatian") -Language("cs", "cze/ces", u"Czech") -Language("da", "dan", u"Danish") -Language("dv", "div", u"Dhivehi") -Language("dv", "div", u"Divehi") -Language("nl", "dut/nld", u"Dutch") -Language("dz", "dzo", u"Dzongkha") -Language("en", "eng", u"English") -Language("eo", "epo", u"Esperanto") -Language("et", "est", u"Estonian") -Language("ee", "ewe", u"Ewe") -Language("fo", "fao", u"Faroese") -Language("fj", "fij", u"Fijian") -Language("fi", "fin", u"Finnish") -Language("nl", "dut/nld", u"Flemish") -Language("fr", "fre/fra", u"French") -Language("ff", "ful", u"Fulah") -Language("gd", "gla", u"Gaelic") -Language("gl", "glg", u"Galician") -Language("lg", "lug", u"Ganda") -Language("ka", "geo/kat", u"Georgian") -Language("de", "ger/deu", u"German") -Language("ki", "kik", u"Gikuyu") -Language("el", "gre/ell", u"Greek, Modern (1453-)") -Language("kl", "kal", u"Greenlandic") -Language("gn", "grn", u"Guarani") -Language("gu", "guj", u"Gujarati") -Language("ht", "hat", u"Haitian") -Language("ht", "hat", u"Haitian Creole") -Language("ha", "hau", u"Hausa") -Language("he", "heb", u"Hebrew") -Language("hz", "her", u"Herero") -Language("hi", "hin", u"Hindi") -Language("ho", "hmo", u"Hiri Motu") -Language("hu", "hun", u"Hungarian") -Language("is", "ice/isl", u"Icelandic") -Language("io", "ido", u"Ido") -Language("ig", "ibo", u"Igbo") -Language("id", "ind", u"Indonesian") -Language("ia", "ina", u"Interlingua (International Auxiliary Language Association)") -Language("ie", "ile", u"Interlingue") -Language("iu", "iku", u"Inuktitut") -Language("ik", "ipk", u"Inupiaq") -Language("ga", "gle", u"Irish") -Language("it", "ita", u"Italian") -Language("ja", "jpn", u"Japanese") -Language("jv", "jav", u"Javanese") -Language("kl", "kal", u"Kalaallisut") -Language("kn", "kan", u"Kannada") -Language("kr", "kau", u"Kanuri") -Language("ks", "kas", u"Kashmiri") -Language("kk", "kaz", u"Kazakh") -Language("ki", "kik", u"Kikuyu") -Language("rw", "kin", u"Kinyarwanda") -Language("ky", "kir", u"Kirghiz") -Language("kv", "kom", u"Komi") -Language("kg", "kon", u"Kongo") -Language("ko", "kor", u"Korean") -Language("kj", "kua", u"Kuanyama") -Language("ku", "kur", u"Kurdish") -Language("kj", "kua", u"Kwanyama") -Language("ky", "kir", u"Kyrgyz") -Language("lo", "lao", u"Lao") -Language("la", "lat", u"Latin") -Language("lv", "lav", u"Latvian") -Language("lb", "ltz", u"Letzeburgesch") -Language("li", "lim", u"Limburgan") -Language("li", "lim", u"Limburger") -Language("li", "lim", u"Limburgish") -Language("ln", "lin", u"Lingala") -Language("lt", "lit", u"Lithuanian") -Language("lu", "lub", u"Luba-Katanga") -Language("lb", "ltz", u"Luxembourgish") -Language("mk", "mac/mkd", u"Macedonian") -Language("mg", "mlg", u"Malagasy") -Language("ms", "may/msa", u"Malay") -Language("ml", "mal", u"Malayalam") -Language("dv", "div", u"Maldivian") -Language("mt", "mlt", u"Maltese") -Language("gv", "glv", u"Manx") -Language("mi", "mao/mri", u"Maori") -Language("mr", "mar", u"Marathi") -Language("mh", "mah", u"Marshallese") -Language("ro", "rum/ron", u"Moldavian") -Language("ro", "rum/ron", u"Moldovan") -Language("mn", "mon", u"Mongolian") -Language("na", "nau", u"Nauru") -Language("nv", "nav", u"Navaho") -Language("nv", "nav", u"Navajo") -Language("nd", "nde", u"Ndebele, North") -Language("nr", "nbl", u"Ndebele, South") -Language("ng", "ndo", u"Ndonga") -Language("ne", "nep", u"Nepali") -Language("nd", "nde", u"North Ndebele") -Language("se", "sme", u"Northern Sami") -Language("no", "nor", u"Norwegian") -Language("nb", "nob", u"Norwegian Bokmål") -Language("nn", "nno", u"Norwegian Nynorsk") -Language("ii", "iii", u"Nuosu") -Language("ny", "nya", u"Nyanja") -Language("nn", "nno", u"Nynorsk, Norwegian") -Language("ie", "ile", u"Occidental") -Language("oc", "oci", u"Occitan (post 1500)") -Language("oj", "oji", u"Ojibwa") -Language("cu", "chu", u"Old Bulgarian") -Language("cu", "chu", u"Old Church Slavonic") -Language("cu", "chu", u"Old Slavonic") -Language("or", "ori", u"Oriya") -Language("om", "orm", u"Oromo") -Language("os", "oss", u"Ossetian") -Language("os", "oss", u"Ossetic") -Language("pi", "pli", u"Pali") -Language("pa", "pan", u"Panjabi") -Language("ps", "pus", u"Pashto") -Language("fa", "per/fas", u"Persian") -Language("pl", "pol", u"Polish") -Language("pt", "por", u"Portuguese") -Language("pa", "pan", u"Punjabi") -Language("ps", "pus", u"Pushto") -Language("qu", "que", u"Quechua") -Language("ro", "rum/ron", u"Romanian") -Language("rm", "roh", u"Romansh") -Language("rn", "run", u"Rundi") -Language("ru", "rus", u"Russian") -Language("sm", "smo", u"Samoan") -Language("sg", "sag", u"Sango") -Language("sa", "san", u"Sanskrit") -Language("sc", "srd", u"Sardinian") -Language("gd", "gla", u"Scottish Gaelic") -Language("sr", "srp", u"Serbian") -Language("sn", "sna", u"Shona") -Language("ii", "iii", u"Sichuan Yi") -Language("sd", "snd", u"Sindhi") -Language("si", "sin", u"Sinhala") -Language("si", "sin", u"Sinhalese") -Language("sk", "slo/slk", u"Slovak") -Language("sl", "slv", u"Slovenian") -Language("so", "som", u"Somali") -Language("st", "sot", u"Sotho, Southern") -Language("nr", "nbl", u"South Ndebele") -Language("es", "spa", u"Spanish") -Language("su", "sun", u"Sundanese") -Language("sw", "swa", u"Swahili") -Language("ss", "ssw", u"Swati") -Language("sv", "swe", u"Swedish") -Language("tl", "tgl", u"Tagalog") -Language("ty", "tah", u"Tahitian") -Language("tg", "tgk", u"Tajik") -Language("ta", "tam", u"Tamil") -Language("tt", "tat", u"Tatar") -Language("te", "tel", u"Telugu") -Language("th", "tha", u"Thai") -Language("bo", "tib/bod", u"Tibetan") -Language("ti", "tir", u"Tigrinya") -Language("to", "ton", u"Tonga (Tonga Islands)") -Language("ts", "tso", u"Tsonga") -Language("tn", "tsn", u"Tswana") -Language("tr", "tur", u"Turkish") -Language("tk", "tuk", u"Turkmen") -Language("tw", "twi", u"Twi") -Language("ug", "uig", u"Uighur") -Language("uk", "ukr", u"Ukrainian") -Language("ur", "urd", u"Urdu") -Language("ug", "uig", u"Uyghur") -Language("uz", "uzb", u"Uzbek") -Language("ca", "cat", u"Valencian") -Language("ve", "ven", u"Venda") -Language("vi", "vie", u"Vietnamese") -Language("vo", "vol", u"Volapük") -Language("wa", "wln", u"Walloon") -Language("cy", "wel/cym", u"Welsh") -Language("fy", "fry", u"Western Frisian") -Language("wo", "wol", u"Wolof") -Language("xh", "xho", u"Xhosa") -Language("yi", "yid", u"Yiddish") -Language("yo", "yor", u"Yoruba") -Language("za", "zha", u"Zhuang") -Language("zu", "zul", u"Zulu") -Country("AF", u"AFGHANISTAN") -Country("AX", u"ÅLAND ISLANDS") -Country("AL", u"ALBANIA") -Country("DZ", u"ALGERIA") -Country("AS", u"AMERICAN SAMOA") -Country("AD", u"ANDORRA") -Country("AO", u"ANGOLA") -Country("AI", u"ANGUILLA") -Country("AQ", u"ANTARCTICA") -Country("AG", u"ANTIGUA AND BARBUDA") -Country("AR", u"ARGENTINA") -Country("AM", u"ARMENIA") -Country("AW", u"ARUBA") -Country("AU", u"AUSTRALIA") -Country("AT", u"AUSTRIA") -Country("AZ", u"AZERBAIJAN") -Country("BS", u"BAHAMAS") -Country("BH", u"BAHRAIN") -Country("BD", u"BANGLADESH") -Country("BB", u"BARBADOS") -Country("BY", u"BELARUS") -Country("BE", u"BELGIUM") -Country("BZ", u"BELIZE") -Country("BJ", u"BENIN") -Country("BM", u"BERMUDA") -Country("BT", u"BHUTAN") -Country("BO", u"BOLIVIA, PLURINATIONAL STATE OF") -Country("BQ", u"BONAIRE, SINT EUSTATIUS AND SABA") -Country("BA", u"BOSNIA AND HERZEGOVINA") -Country("BW", u"BOTSWANA") -Country("BV", u"BOUVET ISLAND") -Country("BR", u"BRAZIL") -Country("IO", u"BRITISH INDIAN OCEAN TERRITORY") -Country("BN", u"BRUNEI DARUSSALAM") -Country("BG", u"BULGARIA") -Country("BF", u"BURKINA FASO") -Country("BI", u"BURUNDI") -Country("KH", u"CAMBODIA") -Country("CM", u"CAMEROON") -Country("CA", u"CANADA") -Country("CV", u"CAPE VERDE") -Country("KY", u"CAYMAN ISLANDS") -Country("CF", u"CENTRAL AFRICAN REPUBLIC") -Country("TD", u"CHAD") -Country("CL", u"CHILE") -Country("CN", u"CHINA") -Country("CX", u"CHRISTMAS ISLAND") -Country("CC", u"COCOS (KEELING) ISLANDS") -Country("CO", u"COLOMBIA") -Country("KM", u"COMOROS") -Country("CG", u"CONGO") -Country("CD", u"CONGO, THE DEMOCRATIC REPUBLIC OF THE") -Country("CK", u"COOK ISLANDS") -Country("CR", u"COSTA RICA") -Country("CI", u"CÔTE D'IVOIRE") -Country("HR", u"CROATIA") -Country("CU", u"CUBA") -Country("CW", u"CURAÇAO") -Country("CY", u"CYPRUS") -Country("CZ", u"CZECH REPUBLIC") -Country("DK", u"DENMARK") -Country("DJ", u"DJIBOUTI") -Country("DM", u"DOMINICA") -Country("DO", u"DOMINICAN REPUBLIC") -Country("EC", u"ECUADOR") -Country("EG", u"EGYPT") -Country("SV", u"EL SALVADOR") -Country("GQ", u"EQUATORIAL GUINEA") -Country("ER", u"ERITREA") -Country("EE", u"ESTONIA") -Country("ET", u"ETHIOPIA") -Country("FK", u"FALKLAND ISLANDS (MALVINAS)") -Country("FO", u"FAROE ISLANDS") -Country("FJ", u"FIJI") -Country("FI", u"FINLAND") -Country("FR", u"FRANCE") -Country("GF", u"FRENCH GUIANA") -Country("PF", u"FRENCH POLYNESIA") -Country("TF", u"FRENCH SOUTHERN TERRITORIES") -Country("GA", u"GABON") -Country("GM", u"GAMBIA") -Country("GE", u"GEORGIA") -Country("DE", u"GERMANY") -Country("GH", u"GHANA") -Country("GI", u"GIBRALTAR") -Country("GR", u"GREECE") -Country("GL", u"GREENLAND") -Country("GD", u"GRENADA") -Country("GP", u"GUADELOUPE") -Country("GU", u"GUAM") -Country("GT", u"GUATEMALA") -Country("GG", u"GUERNSEY") -Country("GN", u"GUINEA") -Country("GW", u"GUINEA-BISSAU") -Country("GY", u"GUYANA") -Country("HT", u"HAITI") -Country("HM", u"HEARD ISLAND AND MCDONALD ISLANDS") -Country("VA", u"HOLY SEE (VATICAN CITY STATE)") -Country("HN", u"HONDURAS") -Country("HK", u"HONG KONG") -Country("HU", u"HUNGARY") -Country("IS", u"ICELAND") -Country("IN", u"INDIA") -Country("ID", u"INDONESIA") -Country("IR", u"IRAN, ISLAMIC REPUBLIC OF") -Country("IQ", u"IRAQ") -Country("IE", u"IRELAND") -Country("IM", u"ISLE OF MAN") -Country("IL", u"ISRAEL") -Country("IT", u"ITALY") -Country("JM", u"JAMAICA") -Country("JP", u"JAPAN") -Country("JE", u"JERSEY") -Country("JO", u"JORDAN") -Country("KZ", u"KAZAKHSTAN") -Country("KE", u"KENYA") -Country("KI", u"KIRIBATI") -Country("KP", u"KOREA, DEMOCRATIC PEOPLE'S REPUBLIC OF") -Country("KR", u"KOREA, REPUBLIC OF") -Country("KW", u"KUWAIT") -Country("KG", u"KYRGYZSTAN") -Country("LA", u"LAO PEOPLE'S DEMOCRATIC REPUBLIC") -Country("LV", u"LATVIA") -Country("LB", u"LEBANON") -Country("LS", u"LESOTHO") -Country("LR", u"LIBERIA") -Country("LY", u"LIBYA") -Country("LI", u"LIECHTENSTEIN") -Country("LT", u"LITHUANIA") -Country("LU", u"LUXEMBOURG") -Country("MO", u"MACAO") -Country("MK", u"MACEDONIA, THE FORMER YUGOSLAV REPUBLIC OF") -Country("MG", u"MADAGASCAR") -Country("MW", u"MALAWI") -Country("MY", u"MALAYSIA") -Country("MV", u"MALDIVES") -Country("ML", u"MALI") -Country("MT", u"MALTA") -Country("MH", u"MARSHALL ISLANDS") -Country("MQ", u"MARTINIQUE") -Country("MR", u"MAURITANIA") -Country("MU", u"MAURITIUS") -Country("YT", u"MAYOTTE") -Country("MX", u"MEXICO") -Country("FM", u"MICRONESIA, FEDERATED STATES OF") -Country("MD", u"MOLDOVA, REPUBLIC OF") -Country("MC", u"MONACO") -Country("MN", u"MONGOLIA") -Country("ME", u"MONTENEGRO") -Country("MS", u"MONTSERRAT") -Country("MA", u"MOROCCO") -Country("MZ", u"MOZAMBIQUE") -Country("MM", u"MYANMAR") -Country("NA", u"NAMIBIA") -Country("NR", u"NAURU") -Country("NP", u"NEPAL") -Country("NL", u"NETHERLANDS") -Country("NC", u"NEW CALEDONIA") -Country("NZ", u"NEW ZEALAND") -Country("NI", u"NICARAGUA") -Country("NE", u"NIGER") -Country("NG", u"NIGERIA") -Country("NU", u"NIUE") -Country("NF", u"NORFOLK ISLAND") -Country("MP", u"NORTHERN MARIANA ISLANDS") -Country("NO", u"NORWAY") -Country("OM", u"OMAN") -Country("PK", u"PAKISTAN") -Country("PW", u"PALAU") -Country("PS", u"PALESTINIAN TERRITORY, OCCUPIED") -Country("PA", u"PANAMA") -Country("PG", u"PAPUA NEW GUINEA") -Country("PY", u"PARAGUAY") -Country("PE", u"PERU") -Country("PH", u"PHILIPPINES") -Country("PN", u"PITCAIRN") -Country("PL", u"POLAND") -Country("PT", u"PORTUGAL") -Country("PR", u"PUERTO RICO") -Country("QA", u"QATAR") -Country("RE", u"RÉUNION") -Country("RO", u"ROMANIA") -Country("RU", u"RUSSIAN FEDERATION") -Country("RW", u"RWANDA") -Country("BL", u"SAINT BARTHÉLEMY") -Country("SH", u"SAINT HELENA, ASCENSION AND TRISTAN DA CUNHA") -Country("KN", u"SAINT KITTS AND NEVIS") -Country("LC", u"SAINT LUCIA") -Country("MF", u"SAINT MARTIN (FRENCH PART)") -Country("PM", u"SAINT PIERRE AND MIQUELON") -Country("VC", u"SAINT VINCENT AND THE GRENADINES") -Country("WS", u"SAMOA") -Country("SM", u"SAN MARINO") -Country("ST", u"SAO TOME AND PRINCIPE") -Country("SA", u"SAUDI ARABIA") -Country("SN", u"SENEGAL") -Country("RS", u"SERBIA") -Country("SC", u"SEYCHELLES") -Country("SL", u"SIERRA LEONE") -Country("SG", u"SINGAPORE") -Country("SX", u"SINT MAARTEN (DUTCH PART)") -Country("SK", u"SLOVAKIA") -Country("SI", u"SLOVENIA") -Country("SB", u"SOLOMON ISLANDS") -Country("SO", u"SOMALIA") -Country("ZA", u"SOUTH AFRICA") -Country("GS", u"SOUTH GEORGIA AND THE SOUTH SANDWICH ISLANDS") -Country("SS", u"SOUTH SUDAN") -Country("ES", u"SPAIN") -Country("LK", u"SRI LANKA") -Country("SD", u"SUDAN") -Country("SR", u"SURINAME") -Country("SJ", u"SVALBARD AND JAN MAYEN") -Country("SZ", u"SWAZILAND") -Country("SE", u"SWEDEN") -Country("CH", u"SWITZERLAND") -Country("SY", u"SYRIAN ARAB REPUBLIC") -Country("TW", u"TAIWAN, PROVINCE OF CHINA") -Country("TJ", u"TAJIKISTAN") -Country("TZ", u"TANZANIA, UNITED REPUBLIC OF") -Country("TH", u"THAILAND") -Country("TL", u"TIMOR-LESTE") -Country("TG", u"TOGO") -Country("TK", u"TOKELAU") -Country("TO", u"TONGA") -Country("TT", u"TRINIDAD AND TOBAGO") -Country("TN", u"TUNISIA") -Country("TR", u"TURKEY") -Country("TM", u"TURKMENISTAN") -Country("TC", u"TURKS AND CAICOS ISLANDS") -Country("TV", u"TUVALU") -Country("UG", u"UGANDA") -Country("UA", u"UKRAINE") -Country("AE", u"UNITED ARAB EMIRATES") -Country("GB", u"UNITED KINGDOM") -Country("US", u"UNITED STATES") -Country("UM", u"UNITED STATES MINOR OUTLYING ISLANDS") -Country("UY", u"URUGUAY") -Country("UZ", u"UZBEKISTAN") -Country("VU", u"VANUATU") -Country("VE", u"VENEZUELA, BOLIVARIAN REPUBLIC OF") -Country("VN", u"VIET NAM") -Country("VG", u"VIRGIN ISLANDS, BRITISH") -Country("VI", u"VIRGIN ISLANDS, U.S.") -Country("WF", u"WALLIS AND FUTUNA") -Country("EH", u"WESTERN SAHARA") -Country("YE", u"YEMEN") -Country("ZM", u"ZAMBIA") -Country("ZW", u"ZIMBABWE") diff --git a/libs/tmdb3/pager.py b/libs/tmdb3/pager.py deleted file mode 100755 index ebcb9d2..0000000 --- a/libs/tmdb3/pager.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -#----------------------- -# Name: pager.py List-like structure designed for handling paged results -# Python Library -# Author: Raymond Wagner -#----------------------- - -from collections import Sequence, Iterator - - -class PagedIterator(Iterator): - def __init__(self, parent): - self._parent = parent - self._index = -1 - self._len = len(parent) - - def __iter__(self): - return self - - def next(self): - self._index += 1 - if self._index == self._len: - raise StopIteration - return self._parent[self._index] - - -class UnpagedData(object): - def copy(self): - return self.__class__() - - def __mul__(self, other): - return (self.copy() for a in range(other)) - - def __rmul__(self, other): - return (self.copy() for a in range(other)) - - -class PagedList(Sequence): - """ - List-like object, with support for automatically grabbing - additional pages from a data source. - """ - _iter_class = None - - def __iter__(self): - if self._iter_class is None: - self._iter_class = type(self.__class__.__name__ + 'Iterator', - (PagedIterator,), {}) - return self._iter_class(self) - - def __len__(self): - try: - return self._len - except: - return len(self._data) - - def __init__(self, iterable, pagesize=20): - self._data = list(iterable) - self._pagesize = pagesize - - def __getitem__(self, index): - if isinstance(index, slice): - return [self[x] for x in xrange(*index.indices(len(self)))] - if index >= len(self): - raise IndexError("list index outside range") - if (index >= len(self._data)) \ - or isinstance(self._data[index], UnpagedData): - self._populatepage(index/self._pagesize + 1) - return self._data[index] - - def __setitem__(self, index, value): - raise NotImplementedError - - def __delitem__(self, index): - raise NotImplementedError - - def __contains__(self, item): - raise NotImplementedError - - def _populatepage(self, page): - pagestart = (page-1) * self._pagesize - if len(self._data) < pagestart: - self._data.extend(UnpagedData()*(pagestart-len(self._data))) - if len(self._data) == pagestart: - self._data.extend(self._getpage(page)) - else: - for data in self._getpage(page): - self._data[pagestart] = data - pagestart += 1 - - def _getpage(self, page): - raise NotImplementedError("PagedList._getpage() must be provided " + - "by subclass") - - -class PagedRequest(PagedList): - """ - Derived PageList that provides a list-like object with automatic - paging intended for use with search requests. - """ - def __init__(self, request, handler=None): - self._request = request - if handler: - self._handler = handler - super(PagedRequest, self).__init__(self._getpage(1), 20) - - def _getpage(self, page): - req = self._request.new(page=page) - res = req.readJSON() - self._len = res['total_results'] - for item in res['results']: - if item is None: - yield None - else: - yield self._handler(item) diff --git a/libs/tmdb3/request.py b/libs/tmdb3/request.py deleted file mode 100755 index 2d51dcd..0000000 --- a/libs/tmdb3/request.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -#----------------------- -# Name: tmdb_request.py -# Python Library -# Author: Raymond Wagner -# Purpose: Wrapped urllib2.Request class pre-configured for accessing the -# TMDb v3 API -#----------------------- - -from tmdb_exceptions import * -from locales import get_locale -from cache import Cache - -from urllib import urlencode -import urllib2 -import json -import os - -DEBUG = False -cache = Cache(filename='pytmdb3.cache') - -#DEBUG = True -#cache = Cache(engine='null') - - -def set_key(key): - """ - Specify the API key to use retrieving data from themoviedb.org. - This key must be set before any calls will function. - """ - if len(key) != 32: - raise TMDBKeyInvalid("Specified API key must be 128-bit hex") - try: - int(key, 16) - except: - raise TMDBKeyInvalid("Specified API key must be 128-bit hex") - Request._api_key = key - - -def set_cache(engine=None, *args, **kwargs): - """Specify caching engine and properties.""" - cache.configure(engine, *args, **kwargs) - - -class Request(urllib2.Request): - _api_key = None - _base_url = "http://api.themoviedb.org/3/" - - @property - def api_key(self): - if self._api_key is None: - raise TMDBKeyMissing("API key must be specified before " + - "requests can be made") - return self._api_key - - def __init__(self, url, **kwargs): - """ - Return a request object, using specified API path and - arguments. - """ - kwargs['api_key'] = self.api_key - self._url = url.lstrip('/') - self._kwargs = dict([(kwa, kwv) for kwa, kwv in kwargs.items() - if kwv is not None]) - - locale = get_locale() - kwargs = {} - for k, v in self._kwargs.items(): - kwargs[k] = locale.encode(v) - url = '{0}{1}?{2}'\ - .format(self._base_url, self._url, urlencode(kwargs)) - - urllib2.Request.__init__(self, url) - self.add_header('Accept', 'application/json') - self.lifetime = 3600 # 1hr - - def new(self, **kwargs): - """ - Create a new instance of the request, with tweaked arguments. - """ - args = dict(self._kwargs) - for k, v in kwargs.items(): - if v is None: - if k in args: - del args[k] - else: - args[k] = v - obj = self.__class__(self._url, **args) - obj.lifetime = self.lifetime - return obj - - def add_data(self, data): - """Provide data to be sent with POST.""" - urllib2.Request.add_data(self, urlencode(data)) - - def open(self): - """Open a file object to the specified URL.""" - try: - if DEBUG: - print 'loading '+self.get_full_url() - if self.has_data(): - print ' '+self.get_data() - return urllib2.urlopen(self) - except urllib2.HTTPError, e: - raise TMDBHTTPError(e) - - def read(self): - """Return result from specified URL as a string.""" - return self.open().read() - - @cache.cached(urllib2.Request.get_full_url) - def readJSON(self): - """Parse result from specified URL as JSON data.""" - url = self.get_full_url() - try: - # catch HTTP error from open() - data = json.load(self.open()) - except TMDBHTTPError, e: - try: - # try to load whatever was returned - data = json.loads(e.response) - except: - # cannot parse json, just raise existing error - raise e - else: - # response parsed, try to raise error from TMDB - handle_status(data, url) - # no error from TMDB, just raise existing error - raise e - handle_status(data, url) - if DEBUG: - import pprint - pprint.PrettyPrinter().pprint(data) - return data - -status_handlers = { - 1: None, - 2: TMDBRequestInvalid('Invalid service - This service does not exist.'), - 3: TMDBRequestError('Authentication Failed - You do not have ' + - 'permissions to access this service.'), - 4: TMDBRequestInvalid("Invalid format - This service doesn't exist " + - 'in that format.'), - 5: TMDBRequestInvalid('Invalid parameters - Your request parameters ' + - 'are incorrect.'), - 6: TMDBRequestInvalid('Invalid id - The pre-requisite id is invalid ' + - 'or not found.'), - 7: TMDBKeyInvalid('Invalid API key - You must be granted a valid key.'), - 8: TMDBRequestError('Duplicate entry - The data you tried to submit ' + - 'already exists.'), - 9: TMDBOffline('This service is tempirarily offline. Try again later.'), - 10: TMDBKeyRevoked('Suspended API key - Access to your account has been ' + - 'suspended, contact TMDB.'), - 11: TMDBError('Internal error - Something went wrong. Contact TMDb.'), - 12: None, - 13: None, - 14: TMDBRequestError('Authentication Failed.'), - 15: TMDBError('Failed'), - 16: TMDBError('Device Denied'), - 17: TMDBError('Session Denied')} - -def handle_status(data, query): - status = status_handlers[data.get('status_code', 1)] - if status is not None: - status.tmdberrno = data['status_code'] - status.query = query - raise status diff --git a/libs/tmdb3/tmdb_api.py b/libs/tmdb3/tmdb_api.py deleted file mode 100755 index 1c8fabd..0000000 --- a/libs/tmdb3/tmdb_api.py +++ /dev/null @@ -1,910 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -#----------------------- -# Name: tmdb_api.py Simple-to-use Python interface to TMDB's API v3 -# Python Library -# Author: Raymond Wagner -# Purpose: This Python library is intended to provide a series of classes -# and methods for search and retrieval of text metadata and image -# URLs from TMDB. -# Preliminary API specifications can be found at -# http://help.themoviedb.org/kb/api/about-3 -# License: Creative Commons GNU GPL v2 -# (http://creativecommons.org/licenses/GPL/2.0/) -#----------------------- - -__title__ = ("tmdb_api - Simple-to-use Python interface to TMDB's API v3 " + - "(www.themoviedb.org)") -__author__ = "Raymond Wagner" -__purpose__ = """ -This Python library is intended to provide a series of classes and methods -for search and retrieval of text metadata and image URLs from TMDB. -Preliminary API specifications can be found at -http://help.themoviedb.org/kb/api/about-3""" - -__version__ = "v0.7.0" -# 0.1.0 Initial development -# 0.2.0 Add caching mechanism for API queries -# 0.2.1 Temporary work around for broken search paging -# 0.3.0 Rework backend machinery for managing OO interface to results -# 0.3.1 Add collection support -# 0.3.2 Remove MythTV key from results.py -# 0.3.3 Add functional language support -# 0.3.4 Re-enable search paging -# 0.3.5 Add methods for grabbing current, popular, and top rated movies -# 0.3.6 Rework paging mechanism -# 0.3.7 Generalize caching mechanism, and allow controllability -# 0.4.0 Add full locale support (language and country) and optional fall through -# 0.4.1 Add custom classmethod for dealing with IMDB movie IDs -# 0.4.2 Improve cache file selection for Windows systems -# 0.4.3 Add a few missed Person properties -# 0.4.4 Add support for additional Studio information -# 0.4.5 Add locale fallthrough for images and alternate titles -# 0.4.6 Add slice support for search results -# 0.5.0 Rework cache framework and improve file cache performance -# 0.6.0 Add user authentication support -# 0.6.1 Add adult filtering for people searches -# 0.6.2 Add similar movie search for Movie objects -# 0.6.3 Add Studio search -# 0.6.4 Add Genre list and associated Movie search -# 0.6.5 Prevent data from being blanked out by subsequent queries -# 0.6.6 Turn date processing errors into mutable warnings -# 0.6.7 Add support for searching by year -# 0.6.8 Add support for collection images -# 0.6.9 Correct Movie image language filtering -# 0.6.10 Add upcoming movie classmethod -# 0.6.11 Fix URL for top rated Movie query -# 0.6.12 Add support for Movie watchlist query and editing -# 0.6.13 Fix URL for rating Movies -# 0.6.14 Add support for Lists -# 0.6.15 Add ability to search Collections -# 0.6.16 Make absent primary images return None (previously u'') -# 0.6.17 Add userrating/votes to Image, add overview to Collection, remove -# releasedate sorting from Collection Movies -# 0.7.0 Add support for television series data - -from request import set_key, Request -from util import Datapoint, Datalist, Datadict, Element, NameRepr, SearchRepr -from pager import PagedRequest -from locales import get_locale, set_locale -from tmdb_auth import get_session, set_session -from tmdb_exceptions import * - -import json -import urllib -import urllib2 -import datetime - -DEBUG = False - - -def process_date(datestr): - try: - return datetime.date(*[int(x) for x in datestr.split('-')]) - except (TypeError, ValueError): - import sys - import warnings - import traceback - _,_,tb = sys.exc_info() - f,l,_,_ = traceback.extract_tb(tb)[-1] - warnings.warn_explicit(('"{0}" is not a supported date format. ' + - 'Please fix upstream data at ' + - 'http://www.themoviedb.org.' - ).format(datestr), Warning, f, l) - return None - - -class Configuration(Element): - images = Datapoint('images') - - def _populate(self): - return Request('configuration') - -Configuration = Configuration() - - -class Account(NameRepr, Element): - def _populate(self): - return Request('account', session_id=self._session.sessionid) - - id = Datapoint('id') - adult = Datapoint('include_adult') - country = Datapoint('iso_3166_1') - language = Datapoint('iso_639_1') - name = Datapoint('name') - username = Datapoint('username') - - @property - def locale(self): - return get_locale(self.language, self.country) - - -def searchMovie(query, locale=None, adult=False, year=None): - kwargs = {'query': query, 'include_adult': adult} - if year is not None: - try: - kwargs['year'] = year.year - except AttributeError: - kwargs['year'] = year - return MovieSearchResult(Request('search/movie', **kwargs), locale=locale) - - -def searchMovieWithYear(query, locale=None, adult=False): - year = None - if (len(query) > 6) and (query[-1] == ')') and (query[-6] == '('): - # simple syntax check, no need for regular expression - try: - year = int(query[-5:-1]) - except ValueError: - pass - else: - if 1885 < year < 2050: - # strip out year from search - query = query[:-7] - else: - # sanity check on resolved year failed, pass through - year = None - return searchMovie(query, locale, adult, year) - - -class MovieSearchResult(SearchRepr, PagedRequest): - """Stores a list of search matches.""" - _name = None - def __init__(self, request, locale=None): - if locale is None: - locale = get_locale() - super(MovieSearchResult, self).__init__( - request.new(language=locale.language), - lambda x: Movie(raw=x, locale=locale)) - -def searchSeries(query, first_air_date_year=None, search_type=None, locale=None): - return SeriesSearchResult( - Request('search/tv', query=query, first_air_date_year=first_air_date_year, search_type=search_type), - locale=locale) - - -class SeriesSearchResult(SearchRepr, PagedRequest): - """Stores a list of search matches.""" - _name = None - def __init__(self, request, locale=None): - if locale is None: - locale = get_locale() - super(SeriesSearchResult, self).__init__( - request.new(language=locale.language), - lambda x: Series(raw=x, locale=locale)) - -def searchPerson(query, adult=False): - return PeopleSearchResult(Request('search/person', query=query, - include_adult=adult)) - - -class PeopleSearchResult(SearchRepr, PagedRequest): - """Stores a list of search matches.""" - _name = None - def __init__(self, request): - super(PeopleSearchResult, self).__init__( - request, lambda x: Person(raw=x)) - - -def searchStudio(query): - return StudioSearchResult(Request('search/company', query=query)) - - -class StudioSearchResult(SearchRepr, PagedRequest): - """Stores a list of search matches.""" - _name = None - def __init__(self, request): - super(StudioSearchResult, self).__init__( - request, lambda x: Studio(raw=x)) - - -def searchList(query, adult=False): - ListSearchResult(Request('search/list', query=query, include_adult=adult)) - - -class ListSearchResult(SearchRepr, PagedRequest): - """Stores a list of search matches.""" - _name = None - def __init__(self, request): - super(ListSearchResult, self).__init__( - request, lambda x: List(raw=x)) - - -def searchCollection(query, locale=None): - return CollectionSearchResult(Request('search/collection', query=query), - locale=locale) - - -class CollectionSearchResult(SearchRepr, PagedRequest): - """Stores a list of search matches.""" - _name=None - def __init__(self, request, locale=None): - if locale is None: - locale = get_locale() - super(CollectionSearchResult, self).__init__( - request.new(language=locale.language), - lambda x: Collection(raw=x, locale=locale)) - - -class Image(Element): - filename = Datapoint('file_path', initarg=1, - handler=lambda x: x.lstrip('/')) - aspectratio = Datapoint('aspect_ratio') - height = Datapoint('height') - width = Datapoint('width') - language = Datapoint('iso_639_1') - userrating = Datapoint('vote_average') - votes = Datapoint('vote_count') - - def sizes(self): - return ['original'] - - def geturl(self, size='original'): - if size not in self.sizes(): - raise TMDBImageSizeError - url = Configuration.images['secure_base_url'].rstrip('/') - return url+'/{0}/{1}'.format(size, self.filename) - - # sort preferring locale's language, but keep remaining ordering consistent - def __lt__(self, other): - if not isinstance(other, Image): - return False - return (self.language == self._locale.language) \ - and (self.language != other.language) - - def __gt__(self, other): - if not isinstance(other, Image): - return True - return (self.language != other.language) \ - and (other.language == self._locale.language) - - # direct match for comparison - def __eq__(self, other): - if not isinstance(other, Image): - return False - return self.filename == other.filename - - # special handling for boolean to see if exists - def __nonzero__(self): - if len(self.filename) == 0: - return False - return True - - def __repr__(self): - # BASE62 encoded filename, no need to worry about unicode - return u"<{0.__class__.__name__} '{0.filename}'>".format(self) - - -class Backdrop(Image): - def sizes(self): - return Configuration.images['backdrop_sizes'] - - -class Poster(Image): - def sizes(self): - return Configuration.images['poster_sizes'] - - -class Profile(Image): - def sizes(self): - return Configuration.images['profile_sizes'] - - -class Logo(Image): - def sizes(self): - return Configuration.images['logo_sizes'] - - -class AlternateTitle(Element): - country = Datapoint('iso_3166_1') - title = Datapoint('title') - - # sort preferring locale's country, but keep remaining ordering consistent - def __lt__(self, other): - return (self.country == self._locale.country) \ - and (self.country != other.country) - - def __gt__(self, other): - return (self.country != other.country) \ - and (other.country == self._locale.country) - - def __eq__(self, other): - return self.country == other.country - - def __repr__(self): - return u"<{0.__class__.__name__} '{0.title}' ({0.country})>"\ - .format(self).encode('utf-8') - - -class Person(Element): - id = Datapoint('id', initarg=1) - name = Datapoint('name') - biography = Datapoint('biography') - dayofbirth = Datapoint('birthday', default=None, handler=process_date) - dayofdeath = Datapoint('deathday', default=None, handler=process_date) - homepage = Datapoint('homepage') - birthplace = Datapoint('place_of_birth') - profile = Datapoint('profile_path', handler=Profile, - raw=False, default=None) - adult = Datapoint('adult') - aliases = Datalist('also_known_as') - - def __repr__(self): - return u"<{0.__class__.__name__} '{0.name}'>"\ - .format(self).encode('utf-8') - - def _populate(self): - return Request('person/{0}'.format(self.id)) - - def _populate_credits(self): - return Request('person/{0}/credits'.format(self.id), - language=self._locale.language) - def _populate_images(self): - return Request('person/{0}/images'.format(self.id)) - - roles = Datalist('cast', handler=lambda x: ReverseCast(raw=x), - poller=_populate_credits) - crew = Datalist('crew', handler=lambda x: ReverseCrew(raw=x), - poller=_populate_credits) - profiles = Datalist('profiles', handler=Profile, poller=_populate_images) - - -class Cast(Person): - character = Datapoint('character') - order = Datapoint('order') - - def __repr__(self): - return u"<{0.__class__.__name__} '{0.name}' as '{0.character}'>"\ - .format(self).encode('utf-8') - - -class Crew(Person): - job = Datapoint('job') - department = Datapoint('department') - - def __repr__(self): - return u"<{0.__class__.__name__} '{0.name}','{0.job}'>"\ - .format(self).encode('utf-8') - - -class Keyword(Element): - id = Datapoint('id') - name = Datapoint('name') - - def __repr__(self): - return u"<{0.__class__.__name__} {0.name}>"\ - .format(self).encode('utf-8') - - -class Release(Element): - certification = Datapoint('certification') - country = Datapoint('iso_3166_1') - releasedate = Datapoint('release_date', handler=process_date) - def __repr__(self): - return u"<{0.__class__.__name__} {0.country}, {0.releasedate}>"\ - .format(self).encode('utf-8') - - -class Trailer(Element): - name = Datapoint('name') - size = Datapoint('size') - source = Datapoint('source') - - -class YoutubeTrailer(Trailer): - def geturl(self): - return "http://www.youtube.com/watch?v={0}".format(self.source) - - def __repr__(self): - # modified BASE64 encoding, no need to worry about unicode - return u"<{0.__class__.__name__} '{0.name}'>".format(self) - - -class AppleTrailer(Element): - name = Datapoint('name') - sources = Datadict('sources', handler=Trailer, attr='size') - - def sizes(self): - return self.sources.keys() - - def geturl(self, size=None): - if size is None: - # sort assuming ###p format for now, take largest resolution - size = str(sorted( - [int(size[:-1]) for size in self.sources] - )[-1]) + 'p' - return self.sources[size].source - - def __repr__(self): - return u"<{0.__class__.__name__} '{0.name}'>".format(self) - - -class Translation(Element): - name = Datapoint('name') - language = Datapoint('iso_639_1') - englishname = Datapoint('english_name') - - def __repr__(self): - return u"<{0.__class__.__name__} '{0.name}' ({0.language})>"\ - .format(self).encode('utf-8') - - -class Genre(NameRepr, Element): - id = Datapoint('id') - name = Datapoint('name') - - def _populate_movies(self): - return Request('genre/{0}/movies'.format(self.id), \ - language=self._locale.language) - - @property - def movies(self): - if 'movies' not in self._data: - search = MovieSearchResult(self._populate_movies(), \ - locale=self._locale) - search._name = "{0.name} Movies".format(self) - self._data['movies'] = search - return self._data['movies'] - - @classmethod - def getAll(cls, locale=None): - class GenreList(Element): - genres = Datalist('genres', handler=Genre) - - def _populate(self): - return Request('genre/list', language=self._locale.language) - return GenreList(locale=locale).genres - - -class Studio(NameRepr, Element): - id = Datapoint('id', initarg=1) - name = Datapoint('name') - description = Datapoint('description') - headquarters = Datapoint('headquarters') - logo = Datapoint('logo_path', handler=Logo, raw=False, default=None) - # FIXME: manage not-yet-defined handlers in a way that will propogate - # locale information properly - parent = Datapoint('parent_company', handler=lambda x: Studio(raw=x)) - - def _populate(self): - return Request('company/{0}'.format(self.id)) - - def _populate_movies(self): - return Request('company/{0}/movies'.format(self.id), - language=self._locale.language) - - # FIXME: add a cleaner way of adding types with no additional processing - @property - def movies(self): - if 'movies' not in self._data: - search = MovieSearchResult(self._populate_movies(), - locale=self._locale) - search._name = "{0.name} Movies".format(self) - self._data['movies'] = search - return self._data['movies'] - - -class Country(NameRepr, Element): - code = Datapoint('iso_3166_1') - name = Datapoint('name') - - -class Language(NameRepr, Element): - code = Datapoint('iso_639_1') - name = Datapoint('name') - - -class Movie(Element): - @classmethod - def latest(cls): - req = Request('latest/movie') - req.lifetime = 600 - return cls(raw=req.readJSON()) - - @classmethod - def nowplaying(cls, locale=None): - res = MovieSearchResult(Request('movie/now-playing'), locale=locale) - res._name = 'Now Playing' - return res - - @classmethod - def mostpopular(cls, locale=None): - res = MovieSearchResult(Request('movie/popular'), locale=locale) - res._name = 'Popular' - return res - - @classmethod - def toprated(cls, locale=None): - res = MovieSearchResult(Request('movie/top_rated'), locale=locale) - res._name = 'Top Rated' - return res - - @classmethod - def upcoming(cls, locale=None): - res = MovieSearchResult(Request('movie/upcoming'), locale=locale) - res._name = 'Upcoming' - return res - - @classmethod - def favorites(cls, session=None): - if session is None: - session = get_session() - account = Account(session=session) - res = MovieSearchResult( - Request('account/{0}/favorite_movies'.format(account.id), - session_id=session.sessionid)) - res._name = "Favorites" - return res - - @classmethod - def ratedmovies(cls, session=None): - if session is None: - session = get_session() - account = Account(session=session) - res = MovieSearchResult( - Request('account/{0}/rated_movies'.format(account.id), - session_id=session.sessionid)) - res._name = "Movies You Rated" - return res - - @classmethod - def watchlist(cls, session=None): - if session is None: - session = get_session() - account = Account(session=session) - res = MovieSearchResult( - Request('account/{0}/movie_watchlist'.format(account.id), - session_id=session.sessionid)) - res._name = "Movies You're Watching" - return res - - @classmethod - def fromIMDB(cls, imdbid, locale=None): - try: - # assume string - if not imdbid.startswith('tt'): - imdbid = "tt{0:0>7}".format(imdbid) - except AttributeError: - # assume integer - imdbid = "tt{0:0>7}".format(imdbid) - if locale is None: - locale = get_locale() - movie = cls(imdbid, locale=locale) - movie._populate() - return movie - - id = Datapoint('id', initarg=1) - title = Datapoint('title') - originaltitle = Datapoint('original_title') - tagline = Datapoint('tagline') - overview = Datapoint('overview') - runtime = Datapoint('runtime') - budget = Datapoint('budget') - revenue = Datapoint('revenue') - releasedate = Datapoint('release_date', handler=process_date) - homepage = Datapoint('homepage') - imdb = Datapoint('imdb_id') - - backdrop = Datapoint('backdrop_path', handler=Backdrop, - raw=False, default=None) - poster = Datapoint('poster_path', handler=Poster, - raw=False, default=None) - - popularity = Datapoint('popularity') - userrating = Datapoint('vote_average') - votes = Datapoint('vote_count') - - adult = Datapoint('adult') - collection = Datapoint('belongs_to_collection', handler=lambda x: \ - Collection(raw=x)) - genres = Datalist('genres', handler=Genre) - studios = Datalist('production_companies', handler=Studio) - countries = Datalist('production_countries', handler=Country) - languages = Datalist('spoken_languages', handler=Language) - - def _populate(self): - return Request('movie/{0}'.format(self.id), \ - language=self._locale.language) - - def _populate_titles(self): - kwargs = {} - if not self._locale.fallthrough: - kwargs['country'] = self._locale.country - return Request('movie/{0}/alternative_titles'.format(self.id), - **kwargs) - - def _populate_cast(self): - return Request('movie/{0}/casts'.format(self.id)) - - def _populate_images(self): - kwargs = {} - if not self._locale.fallthrough: - kwargs['language'] = self._locale.language - return Request('movie/{0}/images'.format(self.id), **kwargs) - - def _populate_keywords(self): - return Request('movie/{0}/keywords'.format(self.id)) - - def _populate_releases(self): - return Request('movie/{0}/releases'.format(self.id)) - - def _populate_trailers(self): - return Request('movie/{0}/trailers'.format(self.id), - language=self._locale.language) - - def _populate_translations(self): - return Request('movie/{0}/translations'.format(self.id)) - - alternate_titles = Datalist('titles', handler=AlternateTitle, \ - poller=_populate_titles, sort=True) - - # FIXME: this data point will need to be changed to 'credits' at some point - cast = Datalist('cast', handler=Cast, - poller=_populate_cast, sort='order') - - crew = Datalist('crew', handler=Crew, poller=_populate_cast) - backdrops = Datalist('backdrops', handler=Backdrop, - poller=_populate_images, sort=True) - posters = Datalist('posters', handler=Poster, - poller=_populate_images, sort=True) - keywords = Datalist('keywords', handler=Keyword, - poller=_populate_keywords) - releases = Datadict('countries', handler=Release, - poller=_populate_releases, attr='country') - youtube_trailers = Datalist('youtube', handler=YoutubeTrailer, - poller=_populate_trailers) - apple_trailers = Datalist('quicktime', handler=AppleTrailer, - poller=_populate_trailers) - translations = Datalist('translations', handler=Translation, - poller=_populate_translations) - - def setFavorite(self, value): - req = Request('account/{0}/favorite'.format( - Account(session=self._session).id), - session_id=self._session.sessionid) - req.add_data({'movie_id': self.id, - 'favorite': str(bool(value)).lower()}) - req.lifetime = 0 - req.readJSON() - - def setRating(self, value): - if not (0 <= value <= 10): - raise TMDBError("Ratings must be between '0' and '10'.") - req = Request('movie/{0}/rating'.format(self.id), - session_id=self._session.sessionid) - req.lifetime = 0 - req.add_data({'value':value}) - req.readJSON() - - def setWatchlist(self, value): - req = Request('account/{0}/movie_watchlist'.format( - Account(session=self._session).id), - session_id=self._session.sessionid) - req.lifetime = 0 - req.add_data({'movie_id': self.id, - 'movie_watchlist': str(bool(value)).lower()}) - req.readJSON() - - def getSimilar(self): - return self.similar - - @property - def similar(self): - res = MovieSearchResult(Request( - 'movie/{0}/similar_movies'.format(self.id)), - locale=self._locale) - res._name = 'Similar to {0}'.format(self._printable_name()) - return res - - @property - def lists(self): - res = ListSearchResult(Request('movie/{0}/lists'.format(self.id))) - res._name = "Lists containing {0}".format(self._printable_name()) - return res - - def _printable_name(self): - if self.title is not None: - s = u"'{0}'".format(self.title) - elif self.originaltitle is not None: - s = u"'{0}'".format(self.originaltitle) - else: - s = u"'No Title'" - if self.releasedate: - s = u"{0} ({1})".format(s, self.releasedate.year) - return s - - def __repr__(self): - return u"<{0} {1}>".format(self.__class__.__name__, - self._printable_name()).encode('utf-8') - - -class ReverseCast( Movie ): - character = Datapoint('character') - - def __repr__(self): - return (u"<{0.__class__.__name__} '{0.character}' on {1}>" - .format(self, self._printable_name()).encode('utf-8')) - - -class ReverseCrew( Movie ): - department = Datapoint('department') - job = Datapoint('job') - - def __repr__(self): - return (u"<{0.__class__.__name__} '{0.job}' for {1}>" - .format(self, self._printable_name()).encode('utf-8')) - - -class Collection(NameRepr, Element): - id = Datapoint('id', initarg=1) - name = Datapoint('name') - backdrop = Datapoint('backdrop_path', handler=Backdrop, \ - raw=False, default=None) - poster = Datapoint('poster_path', handler=Poster, raw=False, default=None) - members = Datalist('parts', handler=Movie) - overview = Datapoint('overview') - - def _populate(self): - return Request('collection/{0}'.format(self.id), - language=self._locale.language) - - def _populate_images(self): - kwargs = {} - if not self._locale.fallthrough: - kwargs['language'] = self._locale.language - return Request('collection/{0}/images'.format(self.id), **kwargs) - - backdrops = Datalist('backdrops', handler=Backdrop, - poller=_populate_images, sort=True) - posters = Datalist('posters', handler=Poster, - poller=_populate_images, sort=True) - -class List(NameRepr, Element): - id = Datapoint('id', initarg=1) - name = Datapoint('name') - author = Datapoint('created_by') - description = Datapoint('description') - favorites = Datapoint('favorite_count') - language = Datapoint('iso_639_1') - count = Datapoint('item_count') - poster = Datapoint('poster_path', handler=Poster, raw=False, default=None) - members = Datalist('items', handler=Movie) - - def _populate(self): - return Request('list/{0}'.format(self.id)) - -class Network(NameRepr,Element): - id = Datapoint('id', initarg=1) - name = Datapoint('name') - -class Episode(NameRepr, Element): - episode_number = Datapoint('episode_number', initarg=3) - season_number = Datapoint('season_number', initarg=2) - series_id = Datapoint('series_id', initarg=1) - air_date = Datapoint('air_date', handler=process_date) - overview = Datapoint('overview') - name = Datapoint('name') - userrating = Datapoint('vote_average') - votes = Datapoint('vote_count') - id = Datapoint('id') - production_code = Datapoint('production_code') - still = Datapoint('still_path', handler=Backdrop, raw=False, default=None) - - def _populate(self): - return Request('tv/{0}/season/{1}/episode/{2}'.format(self.series_id, self.season_number, self.episode_number), - language=self._locale.language) - - def _populate_cast(self): - return Request('tv/{0}/season/{1}/episode/{2}/credits'.format( - self.series_id, self.season_number, self.episode_number), - language=self._locale.language) - - def _populate_external_ids(self): - return Request('tv/{0}/season/{1}/episode/{2}/external_ids'.format( - self.series_id, self.season_number, self.episode_number)) - - def _populate_images(self): - kwargs = {} - if not self._locale.fallthrough: - kwargs['language'] = self._locale.language - return Request('tv/{0}/season/{1}/episode/{2}/images'.format( - self.series_id, self.season_number, self.episode_number), **kwargs) - - cast = Datalist('cast', handler=Cast, - poller=_populate_cast, sort='order') - guest_stars = Datalist('guest_stars', handler=Cast, - poller=_populate_cast, sort='order') - crew = Datalist('crew', handler=Crew, poller=_populate_cast) - imdb_id = Datapoint('imdb_id', poller=_populate_external_ids) - freebase_id = Datapoint('freebase_id', poller=_populate_external_ids) - freebase_mid = Datapoint('freebase_mid', poller=_populate_external_ids) - tvdb_id = Datapoint('tvdb_id', poller=_populate_external_ids) - tvrage_id = Datapoint('tvrage_id', poller=_populate_external_ids) - stills = Datalist('stills', handler=Backdrop, poller=_populate_images, sort=True) - -class Season(NameRepr, Element): - season_number = Datapoint('season_number', initarg=2) - series_id = Datapoint('series_id', initarg=1) - id = Datapoint('id') - air_date = Datapoint('air_date', handler=process_date) - poster = Datapoint('poster_path', handler=Poster, raw=False, default=None) - overview = Datapoint('overview') - name = Datapoint('name') - episodes = Datadict('episodes', attr='episode_number', handler=Episode, - passthrough={'series_id': 'series_id', 'season_number': 'season_number'}) - - def _populate(self): - return Request('tv/{0}/season/{1}'.format(self.series_id, self.season_number), - language=self._locale.language) - - def _populate_images(self): - kwargs = {} - if not self._locale.fallthrough: - kwargs['language'] = self._locale.language - return Request('tv/{0}/season/{1}/images'.format(self.series_id, self.season_number), **kwargs) - - def _populate_external_ids(self): - return Request('tv/{0}/season/{1}/external_ids'.format(self.series_id, self.season_number)) - - posters = Datalist('posters', handler=Poster, - poller=_populate_images, sort=True) - - freebase_id = Datapoint('freebase_id', poller=_populate_external_ids) - freebase_mid = Datapoint('freebase_mid', poller=_populate_external_ids) - tvdb_id = Datapoint('tvdb_id', poller=_populate_external_ids) - tvrage_id = Datapoint('tvrage_id', poller=_populate_external_ids) - -class Series(NameRepr, Element): - id = Datapoint('id', initarg=1) - backdrop = Datapoint('backdrop_path', handler=Backdrop, raw=False, default=None) - authors = Datalist('created_by', handler=Person) - episode_run_times = Datalist('episode_run_time') - first_air_date = Datapoint('first_air_date', handler=process_date) - last_air_date = Datapoint('last_air_date', handler=process_date) - genres = Datalist('genres', handler=Genre) - homepage = Datapoint('homepage') - in_production = Datapoint('in_production') - languages = Datalist('languages') - origin_countries = Datalist('origin_country') - name = Datapoint('name') - original_name = Datapoint('original_name') - number_of_episodes = Datapoint('number_of_episodes') - number_of_seasons = Datapoint('number_of_seasons') - overview = Datapoint('overview') - popularity = Datapoint('popularity') - status = Datapoint('status') - userrating = Datapoint('vote_average') - votes = Datapoint('vote_count') - poster = Datapoint('poster_path', handler=Poster, raw=False, default=None) - networks = Datalist('networks', handler=Network) - seasons = Datadict('seasons', attr='season_number', handler=Season, passthrough={'id': 'series_id'}) - - def _populate(self): - return Request('tv/{0}'.format(self.id), - language=self._locale.language) - - def _populate_cast(self): - return Request('tv/{0}/credits'.format(self.id)) - - def _populate_images(self): - kwargs = {} - if not self._locale.fallthrough: - kwargs['language'] = self._locale.language - return Request('tv/{0}/images'.format(self.id), **kwargs) - - def _populate_external_ids(self): - return Request('tv/{0}/external_ids'.format(self.id)) - - cast = Datalist('cast', handler=Cast, - poller=_populate_cast, sort='order') - crew = Datalist('crew', handler=Crew, poller=_populate_cast) - backdrops = Datalist('backdrops', handler=Backdrop, - poller=_populate_images, sort=True) - posters = Datalist('posters', handler=Poster, - poller=_populate_images, sort=True) - - imdb_id = Datapoint('imdb_id', poller=_populate_external_ids) - freebase_id = Datapoint('freebase_id', poller=_populate_external_ids) - freebase_mid = Datapoint('freebase_mid', poller=_populate_external_ids) - tvdb_id = Datapoint('tvdb_id', poller=_populate_external_ids) - tvrage_id = Datapoint('tvrage_id', poller=_populate_external_ids) diff --git a/libs/tmdb3/tmdb_auth.py b/libs/tmdb3/tmdb_auth.py deleted file mode 100755 index b447b5a..0000000 --- a/libs/tmdb3/tmdb_auth.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -#----------------------- -# Name: tmdb_auth.py -# Python Library -# Author: Raymond Wagner -# Purpose: Provide authentication and session services for -# calls against the TMDB v3 API -#----------------------- - -from datetime import datetime as _pydatetime, \ - tzinfo as _pytzinfo -import re -class datetime(_pydatetime): - """Customized datetime class with ISO format parsing.""" - _reiso = re.compile('(?P[0-9]{4})' - '-(?P[0-9]{1,2})' - '-(?P[0-9]{1,2})' - '.' - '(?P[0-9]{2})' - ':(?P[0-9]{2})' - '(:(?P[0-9]{2}))?' - '(?PZ|' - '(?P[-+])' - '(?P[0-9]{1,2})' - '(:)?' - '(?P[0-9]{2})?' - ')?') - - class _tzinfo(_pytzinfo): - def __init__(self, direc='+', hr=0, min=0): - if direc == '-': - hr = -1*int(hr) - self._offset = timedelta(hours=int(hr), minutes=int(min)) - - def utcoffset(self, dt): - return self._offset - - def tzname(self, dt): - return '' - - def dst(self, dt): - return timedelta(0) - - @classmethod - def fromIso(cls, isotime, sep='T'): - match = cls._reiso.match(isotime) - if match is None: - raise TypeError("time data '%s' does not match ISO 8601 format" - % isotime) - - dt = [int(a) for a in match.groups()[:5]] - if match.group('sec') is not None: - dt.append(int(match.group('sec'))) - else: - dt.append(0) - if match.group('tz'): - if match.group('tz') == 'Z': - tz = cls._tzinfo() - elif match.group('tzmin'): - tz = cls._tzinfo(*match.group('tzdirec', 'tzhour', 'tzmin')) - else: - tz = cls._tzinfo(*match.group('tzdirec', 'tzhour')) - dt.append(0) - dt.append(tz) - return cls(*dt) - -from request import Request -from tmdb_exceptions import * - -syssession = None - - -def set_session(sessionid): - global syssession - syssession = Session(sessionid) - - -def get_session(sessionid=None): - global syssession - if sessionid: - return Session(sessionid) - elif syssession is not None: - return syssession - else: - return Session.new() - - -class Session(object): - @classmethod - def new(cls): - return cls(None) - - def __init__(self, sessionid): - self.sessionid = sessionid - - @property - def sessionid(self): - if self._sessionid is None: - if self._authtoken is None: - raise TMDBError("No Auth Token to produce Session for") - # TODO: check authtoken expiration against current time - req = Request('authentication/session/new', - request_token=self._authtoken) - req.lifetime = 0 - dat = req.readJSON() - if not dat['success']: - raise TMDBError("Session generation failed") - self._sessionid = dat['session_id'] - return self._sessionid - - @sessionid.setter - def sessionid(self, value): - self._sessionid = value - self._authtoken = None - self._authtokenexpiration = None - if value is None: - self.authenticated = False - else: - self.authenticated = True - - @property - def authtoken(self): - if self.authenticated: - raise TMDBError("Session is already authenticated") - if self._authtoken is None: - req = Request('authentication/token/new') - req.lifetime = 0 - dat = req.readJSON() - if not dat['success']: - raise TMDBError("Auth Token request failed") - self._authtoken = dat['request_token'] - self._authtokenexpiration = datetime.fromIso(dat['expires_at']) - return self._authtoken - - @property - def callbackurl(self): - return "http://www.themoviedb.org/authenticate/"+self._authtoken diff --git a/libs/tmdb3/tmdb_exceptions.py b/libs/tmdb3/tmdb_exceptions.py deleted file mode 100755 index f85fbcf..0000000 --- a/libs/tmdb3/tmdb_exceptions.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -#----------------------- -# Name: tmdb_exceptions.py Common exceptions used in tmdbv3 API library -# Python Library -# Author: Raymond Wagner -#----------------------- - - -class TMDBError(Exception): - Error = 0 - KeyError = 10 - KeyMissing = 20 - KeyInvalid = 30 - KeyRevoked = 40 - RequestError = 50 - RequestInvalid = 51 - PagingIssue = 60 - CacheError = 70 - CacheReadError = 71 - CacheWriteError = 72 - CacheDirectoryError = 73 - ImageSizeError = 80 - HTTPError = 90 - Offline = 100 - LocaleError = 110 - - def __init__(self, msg=None, errno=0): - self.errno = errno - if errno == 0: - self.errno = getattr(self, 'TMDB'+self.__class__.__name__, errno) - self.args = (msg,) - - -class TMDBKeyError(TMDBError): - pass - - -class TMDBKeyMissing(TMDBKeyError): - pass - - -class TMDBKeyInvalid(TMDBKeyError): - pass - - -class TMDBKeyRevoked(TMDBKeyInvalid): - pass - - -class TMDBRequestError(TMDBError): - pass - - -class TMDBRequestInvalid(TMDBRequestError): - pass - - -class TMDBPagingIssue(TMDBRequestError): - pass - - -class TMDBCacheError(TMDBRequestError): - pass - - -class TMDBCacheReadError(TMDBCacheError): - def __init__(self, filename): - super(TMDBCacheReadError, self).__init__( - "User does not have permission to access cache file: {0}."\ - .format(filename)) - self.filename = filename - - -class TMDBCacheWriteError(TMDBCacheError): - def __init__(self, filename): - super(TMDBCacheWriteError, self).__init__( - "User does not have permission to write cache file: {0}."\ - .format(filename)) - self.filename = filename - - -class TMDBCacheDirectoryError(TMDBCacheError): - def __init__(self, filename): - super(TMDBCacheDirectoryError, self).__init__( - "Directory containing cache file does not exist: {0}."\ - .format(filename)) - self.filename = filename - - -class TMDBImageSizeError(TMDBError ): - pass - - -class TMDBHTTPError(TMDBError): - def __init__(self, err): - self.httperrno = err.code - self.response = err.fp.read() - super(TMDBHTTPError, self).__init__(str(err)) - - -class TMDBOffline(TMDBError): - pass - - -class TMDBLocaleError(TMDBError): - pass diff --git a/libs/tmdb3/util.py b/libs/tmdb3/util.py deleted file mode 100755 index a0d2e28..0000000 --- a/libs/tmdb3/util.py +++ /dev/null @@ -1,403 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -#----------------------- -# Name: util.py Assorted utilities used in tmdb_api -# Python Library -# Author: Raymond Wagner -#----------------------- - -from copy import copy -from locales import get_locale -from tmdb_auth import get_session - - -class NameRepr(object): - """Mixin for __repr__ methods using 'name' attribute.""" - def __repr__(self): - return u"<{0.__class__.__name__} '{0.name}'>"\ - .format(self).encode('utf-8') - - -class SearchRepr(object): - """ - Mixin for __repr__ methods for classes with '_name' and - '_request' attributes. - """ - def __repr__(self): - name = self._name if self._name else self._request._kwargs['query'] - return u"".format(name).encode('utf-8') - - -class Poller(object): - """ - Wrapper for an optional callable to populate an Element derived - class with raw data, or data from a Request. - """ - def __init__(self, func, lookup, inst=None): - self.func = func - self.lookup = lookup - self.inst = inst - if func: - # with function, this allows polling data from the API - self.__doc__ = func.__doc__ - self.__name__ = func.__name__ - self.__module__ = func.__module__ - else: - # without function, this is just a dummy poller used for applying - # raw data to a new Element class with the lookup table - self.__name__ = '_populate' - - def __get__(self, inst, owner): - # normal decorator stuff - # return self for a class - # return instantiated copy of self for an object - if inst is None: - return self - func = None - if self.func: - func = self.func.__get__(inst, owner) - return self.__class__(func, self.lookup, inst) - - def __call__(self): - # retrieve data from callable function, and apply - if not callable(self.func): - raise RuntimeError('Poller object called without a source function') - req = self.func() - if ('language' in req._kwargs) or ('country' in req._kwargs) \ - and self.inst._locale.fallthrough: - # request specifies a locale filter, and fallthrough is enabled - # run a first pass with specified filter - if not self.apply(req.readJSON(), False): - return - # if first pass results in missed data, run a second pass to - # fill in the gaps - self.apply(req.new(language=None, country=None).readJSON()) - # re-apply the filtered first pass data over top the second - # unfiltered set. this is to work around the issue that the - # properties have no way of knowing when they should or - # should not overwrite existing data. the cache engine will - # take care of the duplicate query - self.apply(req.readJSON()) - - def apply(self, data, set_nones=True): - # apply data directly, bypassing callable function - unfilled = False - for k, v in self.lookup.items(): - if (k in data) and \ - ((data[k] is not None) if callable(self.func) else True): - # argument received data, populate it - setattr(self.inst, v, data[k]) - elif v in self.inst._data: - # argument did not receive data, but Element already contains - # some value, so skip this - continue - elif set_nones: - # argument did not receive data, so fill it with None - # to indicate such and prevent a repeat scan - setattr(self.inst, v, None) - else: - # argument does not need data, so ignore it allowing it to - # trigger a later poll. this is intended for use when - # initializing a class with raw data, or when performing a - # first pass through when performing locale fall through - unfilled = True - return unfilled - - -class Data(object): - """ - Basic response definition class - This maps to a single key in a JSON dictionary received from the API - """ - def __init__(self, field, initarg=None, handler=None, poller=None, - raw=True, default=u'', lang=None, passthrough={}): - """ - This defines how the dictionary value is to be processed by the - poller - field -- defines the dictionary key that filters what data - this uses - initarg -- (optional) specifies that this field must be - supplied when creating a new instance of the Element - class this definition is mapped to. Takes an integer - for the order it should be used in the input - arguments - handler -- (optional) callable used to process the received - value before being stored in the Element object. - poller -- (optional) callable to be used if data is requested - and this value has not yet been defined. the - callable should return a dictionary of data from a - JSON query. many definitions may share a single - poller, which will be and the data used to populate - all referenced definitions based off their defined - field - raw -- (optional) if the specified handler is an Element - class, the data will be passed into it using the - 'raw' keyword attribute. setting this to false - will force the data to instead be passed in as the - first argument - """ - self.field = field - self.initarg = initarg - self.poller = poller - self.raw = raw - self.default = default - self.sethandler(handler) - self.passthrough = passthrough - - def __get__(self, inst, owner): - if inst is None: - return self - if self.field not in inst._data: - if self.poller is None: - return None - self.poller.__get__(inst, owner)() - return inst._data[self.field] - - def __set__(self, inst, value): - if (value is not None) and (value != ''): - value = self.handler(value) - else: - value = self.default - if isinstance(value, Element): - value._locale = inst._locale - value._session = inst._session - - for source, dest in self.passthrough: - setattr(value, dest, getattr(inst, source)) - inst._data[self.field] = value - - def sethandler(self, handler): - # ensure handler is always callable, even for passthrough data - if handler is None: - self.handler = lambda x: x - elif isinstance(handler, ElementType) and self.raw: - self.handler = lambda x: handler(raw=x) - else: - self.handler = lambda x: handler(x) - - -class Datapoint(Data): - pass - - -class Datalist(Data): - """ - Response definition class for list data - This maps to a key in a JSON dictionary storing a list of data - """ - def __init__(self, field, handler=None, poller=None, sort=None, raw=True, passthrough={}): - """ - This defines how the dictionary value is to be processed by the - poller - field -- defines the dictionary key that filters what data - this uses - handler -- (optional) callable used to process the received - value before being stored in the Element object. - poller -- (optional) callable to be used if data is requested - and this value has not yet been defined. the - callable should return a dictionary of data from a - JSON query. many definitions may share a single - poller, which will be and the data used to populate - all referenced definitions based off their defined - field - sort -- (optional) name of attribute in resultant data to be - used to sort the list after processing. this - effectively requires a handler be defined to process - the data into something that has attributes - raw -- (optional) if the specified handler is an Element - class, the data will be passed into it using the - 'raw' keyword attribute. setting this to false will - force the data to instead be passed in as the first - argument - """ - super(Datalist, self).__init__(field, None, handler, poller, raw, passthrough=passthrough) - self.sort = sort - - def __set__(self, inst, value): - data = [] - if value: - for val in value: - val = self.handler(val) - if isinstance(val, Element): - val._locale = inst._locale - val._session = inst._session - - for source, dest in self.passthrough.items(): - setattr(val, dest, getattr(inst, source)) - - data.append(val) - if self.sort: - if self.sort is True: - data.sort() - else: - data.sort(key=lambda x: getattr(x, self.sort)) - inst._data[self.field] = data - - -class Datadict(Data): - """ - Response definition class for dictionary data - This maps to a key in a JSON dictionary storing a dictionary of data - """ - def __init__(self, field, handler=None, poller=None, raw=True, - key=None, attr=None, passthrough={}): - """ - This defines how the dictionary value is to be processed by the - poller - field -- defines the dictionary key that filters what data - this uses - handler -- (optional) callable used to process the received - value before being stored in the Element object. - poller -- (optional) callable to be used if data is requested - and this value has not yet been defined. the - callable should return a dictionary of data from a - JSON query. many definitions may share a single - poller, which will be and the data used to populate - all referenced definitions based off their defined - field - key -- (optional) name of key in resultant data to be used - as the key in the stored dictionary. if this is not - the field name from the source data is used instead - attr -- (optional) name of attribute in resultant data to be - used as the key in the stored dictionary. if this is - not the field name from the source data is used - instead - raw -- (optional) if the specified handler is an Element - class, the data will be passed into it using the - 'raw' keyword attribute. setting this to false will - force the data to instead be passed in as the first - argument - """ - if key and attr: - raise TypeError("`key` and `attr` cannot both be defined") - super(Datadict, self).__init__(field, None, handler, poller, raw, passthrough=passthrough) - if key: - self.getkey = lambda x: x[key] - elif attr: - self.getkey = lambda x: getattr(x, attr) - else: - raise TypeError("Datadict requires `key` or `attr` be defined " + - "for populating the dictionary") - - def __set__(self, inst, value): - data = {} - if value: - for val in value: - val = self.handler(val) - if isinstance(val, Element): - val._locale = inst._locale - val._session = inst._session - - for source, dest in self.passthrough.items(): - setattr(val, dest, getattr(inst, source)) - - data[self.getkey(val)] = val - inst._data[self.field] = data - -class ElementType( type ): - """ - MetaClass used to pre-process Element-derived classes and set up the - Data definitions - """ - def __new__(mcs, name, bases, attrs): - # any Data or Poller object defined in parent classes must be cloned - # and processed in this class to function properly - # scan through available bases for all such definitions and insert - # a copy into this class's attributes - # run in reverse order so higher priority values overwrite lower ones - data = {} - pollers = {'_populate':None} - - for base in reversed(bases): - if isinstance(base, mcs): - for k, attr in base.__dict__.items(): - if isinstance(attr, Data): - # extract copies of each defined Data element from - # parent classes - attr = copy(attr) - attr.poller = attr.poller.func - data[k] = attr - elif isinstance(attr, Poller): - # extract copies of each defined Poller function - # from parent classes - pollers[k] = attr.func - for k, attr in attrs.items(): - if isinstance(attr, Data): - data[k] = attr - if '_populate' in attrs: - pollers['_populate'] = attrs['_populate'] - - # process all defined Data attribues, testing for use as an initial - # argument, and building a list of what Pollers are used to populate - # which Data points - pollermap = dict([(k, []) for k in pollers]) - initargs = [] - for k, v in data.items(): - v.name = k - if v.initarg: - initargs.append(v) - if v.poller: - pn = v.poller.__name__ - if pn not in pollermap: - pollermap[pn] = [] - if pn not in pollers: - pollers[pn] = v.poller - pollermap[pn].append(v) - else: - pollermap['_populate'].append(v) - - # wrap each used poller function with a Poller class, and push into - # the new class attributes - for k, v in pollermap.items(): - if len(v) == 0: - continue - lookup = dict([(attr.field, attr.name) for attr in v]) - poller = Poller(pollers[k], lookup) - attrs[k] = poller - # backfill wrapped Poller into each mapped Data object, and ensure - # the data elements are defined for this new class - for attr in v: - attr.poller = poller - attrs[attr.name] = attr - - # build sorted list of arguments used for intialization - attrs['_InitArgs'] = tuple( - [a.name for a in sorted(initargs, key=lambda x: x.initarg)]) - return type.__new__(mcs, name, bases, attrs) - - def __call__(cls, *args, **kwargs): - obj = cls.__new__(cls) - if ('locale' in kwargs) and (kwargs['locale'] is not None): - obj._locale = kwargs['locale'] - else: - obj._locale = get_locale() - - if 'session' in kwargs: - obj._session = kwargs['session'] - else: - obj._session = get_session() - - obj._data = {} - if 'raw' in kwargs: - # if 'raw' keyword is supplied, create populate object manually - if len(args) != 0: - raise TypeError( - '__init__() takes exactly 2 arguments (1 given)') - obj._populate.apply(kwargs['raw'], False) - else: - # if not, the number of input arguments must exactly match that - # defined by the Data definitions - if len(args) != len(cls._InitArgs): - raise TypeError( - '__init__() takes exactly {0} arguments ({1} given)'\ - .format(len(cls._InitArgs)+1, len(args)+1)) - for a, v in zip(cls._InitArgs, args): - setattr(obj, a, v) - - obj.__init__() - return obj - - -class Element( object ): - __metaclass__ = ElementType - _lang = 'en' From 06293dc0a26da82726563d72f5efc3be2ce78607 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 7 Oct 2014 22:50:22 +0200 Subject: [PATCH 189/202] Simplify tmdb provider --- .../core/media/movie/providers/info/themoviedb.py | 229 ++++++++++----------- 1 file changed, 108 insertions(+), 121 deletions(-) diff --git a/couchpotato/core/media/movie/providers/info/themoviedb.py b/couchpotato/core/media/movie/providers/info/themoviedb.py index 2073383..d1dcd78 100644 --- a/couchpotato/core/media/movie/providers/info/themoviedb.py +++ b/couchpotato/core/media/movie/providers/info/themoviedb.py @@ -1,8 +1,7 @@ import traceback -import time from couchpotato.core.event import addEvent, fireEvent -from couchpotato.core.helpers.encoding import simplifyString, toUnicode, ss, tryUrlencode +from couchpotato.core.helpers.encoding import toUnicode, ss, tryUrlencode from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.media.movie.providers.base import MovieProvider @@ -14,7 +13,7 @@ autoload = 'TheMovieDb' class TheMovieDb(MovieProvider): - http_time_between_calls = .3 + http_time_between_calls = .35 configuration = { 'images': { @@ -23,6 +22,8 @@ class TheMovieDb(MovieProvider): } def __init__(self): + addEvent('info.search', self.search, priority = 3) + addEvent('movie.search', self.search, priority = 3) addEvent('movie.info', self.getInfo, priority = 3) addEvent('movie.info_by_tmdb', self.getInfo) addEvent('app.load', self.config) @@ -32,49 +33,44 @@ class TheMovieDb(MovieProvider): if configuration: self.configuration = configuration - def search(self, q, limit = 12): + def search(self, q, limit = 3): """ Find movie by name """ if self.isDisabled(): return False - search_string = simplifyString(q) - cache_key = 'tmdb.cache.%s.%s' % (search_string, limit) - results = None #self.getCache(cache_key) + log.debug('Searching for movie: %s', q) - if not results: - log.debug('Searching for movie: %s', q) + raw = None + try: + name_year = fireEvent('scanner.name_year', q, single = True) + raw = self.request('search/movie', { + 'query': name_year.get('name', q), + 'year': name_year.get('year'), + 'search_type': 'ngram' if limit > 1 else 'phrase' + }, return_key = 'results') + except: + log.error('Failed searching TMDB for "%s": %s', (q, traceback.format_exc())) - raw = None + results = [] + if raw: try: + nr = 0 - #name_year = fireEvent('scanner.name_year', q, single = True) - - raw = self.request('search/movie', { - 'query': q - }, return_key = 'results') - except: - log.error('Failed searching TMDB for "%s": %s', (search_string, traceback.format_exc())) - - results = [] - if raw: - try: - nr = 0 + for movie in raw: + parsed_movie = self.parseMovie(movie, extended = False) + results.append(parsed_movie) - for movie in raw: - results.append(self.parseMovie(movie, extended = False)) + nr += 1 + if nr == limit: + break - nr += 1 - if nr == limit: - break + log.info('Found: %s', [result['titles'][0] + ' (' + str(result.get('year', 0)) + ')' for result in results]) - log.info('Found: %s', [result['titles'][0] + ' (' + str(result.get('year', 0)) + ')' for result in results]) - - self.setCache(cache_key, results) - return results - except SyntaxError as e: - log.error('Failed to parse XML response: %s', e) - return False + return results + except SyntaxError as e: + log.error('Failed to parse XML response: %s', e) + return False return results @@ -91,89 +87,81 @@ class TheMovieDb(MovieProvider): def parseMovie(self, movie, extended = True): - cache_key = 'tmdb.cache.%s%s' % (movie.get('id'), '.ex' if extended else '') - movie_data = None #self.getCache(cache_key) + # Do request, append other items + movie = self.request('movie/%s' % movie.get('id'), { + 'append_to_response': 'alternative_titles' + (',images,casts' if extended else '') + }) + + # Images + poster = self.getImage(movie, type = 'poster', size = 'w154') + poster_original = self.getImage(movie, type = 'poster', size = 'original') + backdrop_original = self.getImage(movie, type = 'backdrop', size = 'original') + extra_thumbs = self.getMultImages(movie, type = 'backdrops', size = 'original') if extended else [] + + images = { + 'poster': [poster] if poster else [], + #'backdrop': [backdrop] if backdrop else [], + 'poster_original': [poster_original] if poster_original else [], + 'backdrop_original': [backdrop_original] if backdrop_original else [], + 'actors': {}, + 'extra_thumbs': extra_thumbs + } + + # Genres + try: + genres = [genre.get('name') for genre in movie.get('genres', [])] + except: + genres = [] + + # 1900 is the same as None + year = str(movie.get('release_date') or '')[:4] + if not movie.get('release_date') or year == '1900' or year.lower() == 'none': + year = None - if not movie_data: + # Gather actors data + actors = {} + if extended: # Full data - movie = self.request('movie/%s' % movie.get('id')) - - # Images - poster = self.getImage(movie, type = 'poster', size = 'w154') - poster_original = self.getImage(movie, type = 'poster', size = 'original') - backdrop_original = self.getImage(movie, type = 'backdrop', size = 'original') - extra_thumbs = self.getMultImages(movie, type = 'backdrops', size = 'original') - - images = { - 'poster': [poster] if poster else [], - #'backdrop': [backdrop] if backdrop else [], - 'poster_original': [poster_original] if poster_original else [], - 'backdrop_original': [backdrop_original] if backdrop_original else [], - 'actors': {}, - 'extra_thumbs': extra_thumbs - } - - # Genres - try: - genres = [genre.get('name') for genre in movie.get('genres', [])] - except: - genres = [] - - # 1900 is the same as None - year = str(movie.get('release_date') or '')[:4] - if not movie.get('release_date') or year == '1900' or year.lower() == 'none': - year = None - - # Gather actors data - actors = {} - if extended: - - # Full data - cast = self.request('movie/%s/casts' % movie.get('id'), return_key = 'cast') - - for cast_item in cast: - try: - actors[toUnicode(cast_item.get('name'))] = toUnicode(cast_item.get('character')) - images['actors'][toUnicode(cast_item.get('name'))] = self.getImage(cast_item, type = 'profile', size = 'original') - except: - log.debug('Error getting cast info for %s: %s', (cast_item, traceback.format_exc())) - - movie_data = { - 'type': 'movie', - 'via_tmdb': True, - 'tmdb_id': movie.get('id'), - 'titles': [toUnicode(movie.get('title'))], - 'original_title': movie.get('original_title'), - 'images': images, - 'imdb': movie.get('imdb_id'), - 'runtime': movie.get('runtime'), - 'released': str(movie.get('release_date')), - 'year': tryInt(year, None), - 'plot': movie.get('overview'), - 'genres': genres, - 'collection': getattr(movie.get('belongs_to_collection'), 'name', None), - 'actor_roles': actors - } - - movie_data = dict((k, v) for k, v in movie_data.items() if v) - - # Add alternative names - if movie_data['original_title'] and movie_data['original_title'] not in movie_data['titles']: - movie_data['titles'].append(movie_data['original_title']) - - if extended: - - # Full data - alternate_titles = self.request('movie/%s/alternative_titles' % movie.get('id'), return_key = 'titles') - - for alt in alternate_titles: - alt_name = alt.get('title') - if alt_name and alt_name not in movie_data['titles'] and alt_name.lower() != 'none' and alt_name is not None: - movie_data['titles'].append(alt_name) - - # Cache movie parsed - self.setCache(cache_key, movie_data) + cast = movie.get('casts', {}).get('cast', []) + + for cast_item in cast: + try: + actors[toUnicode(cast_item.get('name'))] = toUnicode(cast_item.get('character')) + images['actors'][toUnicode(cast_item.get('name'))] = self.getImage(cast_item, type = 'profile', size = 'original') + except: + log.debug('Error getting cast info for %s: %s', (cast_item, traceback.format_exc())) + + movie_data = { + 'type': 'movie', + 'via_tmdb': True, + 'tmdb_id': movie.get('id'), + 'titles': [toUnicode(movie.get('title'))], + 'original_title': movie.get('original_title'), + 'images': images, + 'imdb': movie.get('imdb_id'), + 'runtime': movie.get('runtime'), + 'released': str(movie.get('release_date')), + 'year': tryInt(year, None), + 'plot': movie.get('overview'), + 'genres': genres, + 'collection': getattr(movie.get('belongs_to_collection'), 'name', None), + 'actor_roles': actors + } + + movie_data = dict((k, v) for k, v in movie_data.items() if v) + + # Add alternative names + if movie_data['original_title'] and movie_data['original_title'] not in movie_data['titles']: + movie_data['titles'].append(movie_data['original_title']) + + # Add alternative titles + alternate_titles = movie.get('alternative_titles', {}).get('titles', []) + + for alt in alternate_titles: + alt_name = alt.get('title') + if alt_name and alt_name not in movie_data['titles'] and alt_name.lower() != 'none' and alt_name is not None: + movie_data['titles'].append(alt_name) return movie_data @@ -192,23 +180,22 @@ class TheMovieDb(MovieProvider): image_urls = [] try: - - # Full data - images = self.request('movie/%s/images' % movie.get('id'), return_key = type) - for image in images[1:5]: + for image in movie.get('images', {}).get(type, [])[1:5]: image_urls.append(self.getImage(image, 'file', size)) - except: log.debug('Failed getting %s.%s for "%s"', (type, size, ss(str(movie)))) return image_urls def request(self, call = '', params = {}, return_key = None): + + params = dict((k, v) for k, v in params.items() if v) params = tryUrlencode(params) + url = 'http://api.themoviedb.org/3/%s?api_key=%s%s' % (call, self.conf('api_key'), '&%s' % params if params else '') - data = self.getJsonData(url, cache_timeout = 0) + data = self.getJsonData(url) - if data and return_key and data.get(return_key): + if data and return_key and return_key in data: data = data.get(return_key) return data From a763957334c18c6abe2e70a071a1384885cc4eb9 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 7 Oct 2014 22:53:57 +0200 Subject: [PATCH 190/202] Log minimum 1 second wait --- couchpotato/core/plugins/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 704aa62..ab16378 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -279,7 +279,7 @@ class Plugin(object): wait = (last_use - now) + self.http_time_between_calls if wait > 0: - log.debug('Waiting for %s, %d seconds', (self.getName(), wait)) + log.debug('Waiting for %s, %d seconds', (self.getName(), max(1, wait))) time.sleep(min(wait, 30)) def beforeCall(self, handler): From 9f6e4cc2fad0d11407e91fba16038c5e72577795 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 8 Oct 2014 20:46:11 +0200 Subject: [PATCH 191/202] Remove NotifyMyWP --- couchpotato/core/notifications/notifymywp.py | 68 -------------- libs/pynmwp/__init__.py | 134 --------------------------- 2 files changed, 202 deletions(-) delete mode 100644 couchpotato/core/notifications/notifymywp.py delete mode 100644 libs/pynmwp/__init__.py diff --git a/couchpotato/core/notifications/notifymywp.py b/couchpotato/core/notifications/notifymywp.py deleted file mode 100644 index 262fd8d..0000000 --- a/couchpotato/core/notifications/notifymywp.py +++ /dev/null @@ -1,68 +0,0 @@ -from couchpotato.core.helpers.variable import splitString -from couchpotato.core.logger import CPLog -from couchpotato.core.notifications.base import Notification -from pynmwp import PyNMWP -import six - -log = CPLog(__name__) - -autoload = 'NotifyMyWP' - - -class NotifyMyWP(Notification): - - def notify(self, message = '', data = None, listener = None): - if not data: data = {} - - keys = splitString(self.conf('api_key')) - p = PyNMWP(keys, self.conf('dev_key')) - - response = p.push(application = self.default_title, event = message, description = message, priority = self.conf('priority'), batch_mode = len(keys) > 1) - - for key in keys: - if not response[key]['Code'] == six.u('200'): - log.error('Could not send notification to NotifyMyWindowsPhone (%s). %s', (key, response[key]['message'])) - return False - - return response - - -config = [{ - 'name': 'notifymywp', - 'groups': [ - { - 'tab': 'notifications', - 'list': 'notification_providers', - 'name': 'notifymywp', - 'label': 'Windows Phone', - 'options': [ - { - 'name': 'enabled', - 'default': 0, - 'type': 'enabler', - }, - { - 'name': 'api_key', - 'description': 'Multiple keys seperated by a comma. Maximum of 5.' - }, - { - 'name': 'dev_key', - 'advanced': True, - }, - { - 'name': 'priority', - 'default': 0, - 'type': 'dropdown', - 'values': [('Very Low', -2), ('Moderate', -1), ('Normal', 0), ('High', 1), ('Emergency', 2)], - }, - { - 'name': 'on_snatch', - 'default': 0, - 'type': 'bool', - 'advanced': True, - 'description': 'Also send message when movie is snatched.', - }, - ], - } - ], -}] diff --git a/libs/pynmwp/__init__.py b/libs/pynmwp/__init__.py deleted file mode 100644 index de724b9..0000000 --- a/libs/pynmwp/__init__.py +++ /dev/null @@ -1,134 +0,0 @@ -from xml.dom.minidom import parseString -from httplib import HTTPSConnection -from urllib import urlencode - -__version__ = "0.1" - -API_SERVER = 'notifymywindowsphone.com' -ADD_PATH = '/publicapi/notify' - -USER_AGENT = "PyNMWP/v%s" % __version__ - -def uniq_preserve(seq): # Dave Kirby - # Order preserving - seen = set() - return [x for x in seq if x not in seen and not seen.add(x)] - -def uniq(seq): - # Not order preserving - return {}.fromkeys(seq).keys() - -class PyNMWP(object): - """PyNMWP(apikey=[], developerkey=None) -takes 2 optional arguments: - - (opt) apykey: might me a string containing 1 key or an array of keys - - (opt) developerkey: where you can store your developer key -""" - - def __init__(self, apikey = [], developerkey = None): - self._developerkey = None - self.developerkey(developerkey) - if apikey: - if type(apikey) == str: - apikey = [apikey] - self._apikey = uniq(apikey) - - def addkey(self, key): - "Add a key (register ?)" - if type(key) == str: - if not key in self._apikey: - self._apikey.append(key) - elif type(key) == list: - for k in key: - if not k in self._apikey: - self._apikey.append(k) - - def delkey(self, key): - "Removes a key (unregister ?)" - if type(key) == str: - if key in self._apikey: - self._apikey.remove(key) - elif type(key) == list: - for k in key: - if key in self._apikey: - self._apikey.remove(k) - - def developerkey(self, developerkey): - "Sets the developer key (and check it has the good length)" - if type(developerkey) == str and len(developerkey) == 48: - self._developerkey = developerkey - - def push(self, application = "", event = "", description = "", url = "", priority = 0, batch_mode = False): - """Pushes a message on the registered API keys. -takes 5 arguments: - - (req) application: application name [256] - - (req) event: event name [1000] - - (req) description: description [10000] - - (opt) url: url [512] - - (opt) priority: from -2 (lowest) to 2 (highest) (def:0) - - (opt) batch_mode: call API 5 by 5 (def:False) - -Warning: using batch_mode will return error only if all API keys are bad - cf: http://nma.usk.bz/api.php -""" - datas = { - 'application': application[:256].encode('utf8'), - 'event': event[:1024].encode('utf8'), - 'description': description[:10000].encode('utf8'), - 'priority': priority - } - - if url: - datas['url'] = url[:512] - - if self._developerkey: - datas['developerkey'] = self._developerkey - - results = {} - - if not batch_mode: - for key in self._apikey: - datas['apikey'] = key - res = self.callapi('POST', ADD_PATH, datas) - results[key] = res - else: - for i in range(0, len(self._apikey), 5): - datas['apikey'] = ",".join(self._apikey[i:i + 5]) - res = self.callapi('POST', ADD_PATH, datas) - results[datas['apikey']] = res - return results - - def callapi(self, method, path, args): - headers = { 'User-Agent': USER_AGENT } - if method == "POST": - headers['Content-type'] = "application/x-www-form-urlencoded" - http_handler = HTTPSConnection(API_SERVER) - http_handler.request(method, path, urlencode(args), headers) - resp = http_handler.getresponse() - - try: - res = self._parse_reponse(resp.read()) - except Exception, e: - res = {'type': "pynmwperror", - 'code': 600, - 'message': str(e) - } - pass - - return res - - def _parse_reponse(self, response): - root = parseString(response).firstChild - for elem in root.childNodes: - if elem.nodeType == elem.TEXT_NODE: continue - if elem.tagName == 'success': - res = dict(elem.attributes.items()) - res['message'] = "" - res['type'] = elem.tagName - return res - if elem.tagName == 'error': - res = dict(elem.attributes.items()) - res['message'] = elem.firstChild.nodeValue - res['type'] = elem.tagName - return res - From 259e2bc61ced9d8a65e1c64bb35f6747a2e77375 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 8 Oct 2014 21:58:34 +0200 Subject: [PATCH 192/202] Don't skip unpacking on manage scan --- couchpotato/core/plugins/manage.py | 2 +- couchpotato/core/plugins/scanner.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/manage.py b/couchpotato/core/plugins/manage.py index 75c550b..b0e1239 100755 --- a/couchpotato/core/plugins/manage.py +++ b/couchpotato/core/plugins/manage.py @@ -123,7 +123,7 @@ class Manage(Plugin): fireEvent('notify.frontend', type = 'manage.update', data = True, message = 'Scanning for movies in "%s"' % folder) onFound = self.createAddToLibrary(folder, added_identifiers) - fireEvent('scanner.scan', folder = folder, simple = True, newer_than = last_update if not full else 0, on_found = onFound, single = True) + fireEvent('scanner.scan', folder = folder, simple = True, newer_than = last_update if not full else 0, check_file_date = False, on_found = onFound, single = True) # Break if CP wants to shut down if self.shuttingDown(): diff --git a/couchpotato/core/plugins/scanner.py b/couchpotato/core/plugins/scanner.py index a7a5e88..aca9e64 100644 --- a/couchpotato/core/plugins/scanner.py +++ b/couchpotato/core/plugins/scanner.py @@ -131,7 +131,7 @@ class Scanner(Plugin): addEvent('scanner.name_year', self.getReleaseNameYear) addEvent('scanner.partnumber', self.getPartNumber) - def scan(self, folder = None, files = None, release_download = None, simple = False, newer_than = 0, return_ignored = True, on_found = None): + def scan(self, folder = None, files = None, release_download = None, simple = False, newer_than = 0, return_ignored = True, check_file_date = True, on_found = None): folder = sp(folder) @@ -145,7 +145,6 @@ class Scanner(Plugin): # Scan all files of the folder if no files are set if not files: - check_file_date = True try: files = [] for root, dirs, walk_files in os.walk(folder, followlinks=True): From 3c12a2c4bf8d25491b210d91bcfbe2b3ca349233 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 8 Oct 2014 22:35:56 +0200 Subject: [PATCH 193/202] Don't restatus movies to active when scanning manage section --- couchpotato/core/media/_base/media/main.py | 4 ++-- couchpotato/core/plugins/manage.py | 4 ++-- couchpotato/core/plugins/release/main.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index aa1ee38..bd42b9d 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -491,7 +491,7 @@ class MediaPlugin(MediaBase): } }) - def restatus(self, media_id, tag_recent = True): + def restatus(self, media_id, tag_recent = True, allowed_restatus = None): try: db = get_db() @@ -526,7 +526,7 @@ class MediaPlugin(MediaBase): m['status'] = previous_status # Only update when status has changed - if previous_status != m['status']: + if previous_status != m['status'] and (not allowed_restatus or m['status'] in allowed_restatus): db.update(m) # Tag media as recent diff --git a/couchpotato/core/plugins/manage.py b/couchpotato/core/plugins/manage.py index b0e1239..3c22ee3 100755 --- a/couchpotato/core/plugins/manage.py +++ b/couchpotato/core/plugins/manage.py @@ -218,8 +218,8 @@ class Manage(Plugin): if group['media'] and group['identifier']: added_identifiers.append(group['identifier']) - # Add it to release and update the info - fireEvent('release.add', group = group, update_info = False) + # Add it to release and update the info (only allow media restatus to done, not to active) + fireEvent('release.add', group = group, update_info = False, allowed_restatus = ['done']) fireEvent('movie.update', identifier = group['identifier'], on_complete = self.createAfterUpdate(folder, group['identifier'])) return addToLibrary diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 815cb40..a937eee 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -129,7 +129,7 @@ class Release(Plugin): if 'recent' in media.get('tags', []): fireEvent('media.untag', media.get('_id'), 'recent', single = True) - def add(self, group, update_info = True, update_id = None): + def add(self, group, update_info = True, update_id = None, allowed_restatus = None): try: db = get_db() @@ -187,7 +187,7 @@ class Release(Plugin): release['files'] = dict((k, [toUnicode(x) for x in v]) for k, v in group['files'].items() if v) db.update(release) - fireEvent('media.restatus', media['_id'], single = True) + fireEvent('media.restatus', media['_id'], allowed_restatus = allowed_restatus, single = True) return True except: From 7b6641d7096f39f4653733fac757a377ef29070c Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 8 Oct 2014 23:00:11 +0200 Subject: [PATCH 194/202] Never restatus "down" when adding release --- couchpotato/core/plugins/manage.py | 4 ++-- couchpotato/core/plugins/release/main.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/plugins/manage.py b/couchpotato/core/plugins/manage.py index 3c22ee3..b0e1239 100755 --- a/couchpotato/core/plugins/manage.py +++ b/couchpotato/core/plugins/manage.py @@ -218,8 +218,8 @@ class Manage(Plugin): if group['media'] and group['identifier']: added_identifiers.append(group['identifier']) - # Add it to release and update the info (only allow media restatus to done, not to active) - fireEvent('release.add', group = group, update_info = False, allowed_restatus = ['done']) + # Add it to release and update the info + fireEvent('release.add', group = group, update_info = False) fireEvent('movie.update', identifier = group['identifier'], on_complete = self.createAfterUpdate(folder, group['identifier'])) return addToLibrary diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index a937eee..452aa75 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -129,7 +129,7 @@ class Release(Plugin): if 'recent' in media.get('tags', []): fireEvent('media.untag', media.get('_id'), 'recent', single = True) - def add(self, group, update_info = True, update_id = None, allowed_restatus = None): + def add(self, group, update_info = True, update_id = None): try: db = get_db() @@ -187,7 +187,7 @@ class Release(Plugin): release['files'] = dict((k, [toUnicode(x) for x in v]) for k, v in group['files'].items() if v) db.update(release) - fireEvent('media.restatus', media['_id'], allowed_restatus = allowed_restatus, single = True) + fireEvent('media.restatus', media['_id'], allowed_restatus = ['done'], single = True) return True except: From cb48ca03df4af631c0b1dbb548ba7cbb54ef02b2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 8 Oct 2014 23:05:04 +0200 Subject: [PATCH 195/202] Remove profile when marking movies done --- couchpotato/core/media/_base/media/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index bd42b9d..01dc0f1 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -456,6 +456,11 @@ class MediaPlugin(MediaBase): deleted = True elif new_media_status: media['status'] = new_media_status + + # Remove profile (no use for in manage) + if new_media_status == 'done': + media['profile_id'] = None + db.update(media) fireEvent('media.untag', media['_id'], 'recent', single = True) From b6f288a522020f8e8335fdec78bdf9a001531b80 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 9 Oct 2014 23:10:28 +0200 Subject: [PATCH 196/202] Close request connection --- couchpotato/core/plugins/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index ab16378..5a90d92 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -196,7 +196,7 @@ class Plugin(object): headers['Host'] = headers.get('Host', None) headers['User-Agent'] = headers.get('User-Agent', self.user_agent) headers['Accept-encoding'] = headers.get('Accept-encoding', 'gzip') - headers['Connection'] = headers.get('Connection', 'keep-alive') + headers['Connection'] = headers.get('Connection', 'close') headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0') r = Env.get('http_opener') From c7ce18f8c274f055de864d9e6b4673d56592b50f Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 10 Oct 2014 15:13:58 +0200 Subject: [PATCH 197/202] Better error message for missing cd number --- couchpotato/core/plugins/renamer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index d6deba6..94e1002 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -222,8 +222,8 @@ class Renamer(Plugin): cd_keys = ['','', ''] if not any(x in folder_name for x in cd_keys) and not any(x in file_name for x in cd_keys): - log.error('Missing `cd` or `cd_nr` in the renamer. This will cause multi-file releases of being renamed to the same file.' - 'Force adding it') + log.error('Missing `cd` or `cd_nr` in the renamer. This will cause multi-file releases of being renamed to the same file. ' + 'Please add it in the renamer settings. Force adding it for now.') file_name = '%s %s' % ('', file_name) # Tag release folder as failed_rename in case no groups were found. This prevents check_snatched from removing the release from the downloader. From ca131073300d2daad9609e9ca883b6131b692026 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 10 Oct 2014 22:46:35 +0200 Subject: [PATCH 198/202] Ignore exceptions on removing db_backup stuff --- couchpotato/runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 36fc366..b780397 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -116,7 +116,8 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En # Delete non zip files if len(ints) != 1: - os.remove(os.path.join(root, backup_file)) + try: os.remove(os.path.join(root, backup_file)) + except: pass else: existing_backups.append((int(ints[0]), backup_file)) else: From b616af3a8397c38bbc101df72c5db5a177113bc8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 10 Oct 2014 23:16:58 +0200 Subject: [PATCH 199/202] Make minimum scoring editable fix #4042 --- couchpotato/core/media/movie/searcher.py | 3 ++- couchpotato/core/plugins/profile/main.py | 2 ++ couchpotato/core/plugins/profile/static/profile.css | 5 +++++ couchpotato/core/plugins/profile/static/profile.js | 12 +++++++++++- couchpotato/core/plugins/release/main.py | 4 ++-- 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index e44e8e6..3c26386 100755 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -166,7 +166,8 @@ class MovieSearcher(SearcherBase, MovieTypeBase): 'quality': q_identifier, 'finish': profile['finish'][index], 'wait_for': tryInt(profile['wait_for'][index]), - '3d': profile['3d'][index] if profile.get('3d') else False + '3d': profile['3d'][index] if profile.get('3d') else False, + 'minimum_score': profile.get('minimum_score', 1), } could_not_be_released = not self.couldBeReleased(q_identifier in pre_releases, release_dates, movie['info']['year']) diff --git a/couchpotato/core/plugins/profile/main.py b/couchpotato/core/plugins/profile/main.py index 489c34d..29bd6cb 100644 --- a/couchpotato/core/plugins/profile/main.py +++ b/couchpotato/core/plugins/profile/main.py @@ -86,6 +86,7 @@ class ProfilePlugin(Plugin): 'label': toUnicode(kwargs.get('label')), 'order': tryInt(kwargs.get('order', 999)), 'core': kwargs.get('core', False), + 'minimum_score': tryInt(kwargs.get('minimum_score', 1)), 'qualities': [], 'wait_for': [], 'stop_after': [], @@ -217,6 +218,7 @@ class ProfilePlugin(Plugin): 'label': toUnicode(profile.get('label')), 'order': order, 'qualities': profile.get('qualities'), + 'minimum_score': 1, 'finish': [], 'wait_for': [], 'stop_after': [], diff --git a/couchpotato/core/plugins/profile/static/profile.css b/couchpotato/core/plugins/profile/static/profile.css index edab831..df93944 100644 --- a/couchpotato/core/plugins/profile/static/profile.css +++ b/couchpotato/core/plugins/profile/static/profile.css @@ -51,6 +51,11 @@ margin: 0 5px !important; } + .profile .wait_for .minimum_score_input { + width: 40px !important; + text-align: left; + } + .profile .types { padding: 0; margin: 0 20px 0 -4px; diff --git a/couchpotato/core/plugins/profile/static/profile.js b/couchpotato/core/plugins/profile/static/profile.js index 89f1a69..35ad81b 100644 --- a/couchpotato/core/plugins/profile/static/profile.js +++ b/couchpotato/core/plugins/profile/static/profile.js @@ -53,12 +53,21 @@ var Profile = new Class({ }), new Element('span', {'text':'day(s) for a better quality '}), new Element('span.advanced', {'text':'and keep searching'}), + // "After a checked quality is found and downloaded, continue searching for even better quality releases for the entered number of days." new Element('input.inlay.xsmall.stop_after_input.advanced', { 'type':'text', 'value': data.stop_after && data.stop_after.length > 0 ? data.stop_after[0] : 0 }), - new Element('span.advanced', {'text':'day(s) for a better (checked) quality.'}) + new Element('span.advanced', {'text':'day(s) for a better (checked) quality.'}), + + // Minimum score of + new Element('span.advanced', {'html':'
Releases need a minimum score of'}), + new Element('input.advanced.inlay.xsmall.minimum_score_input', { + 'size': 4, + 'type':'text', + 'value': data.minimum_score || 1 + }) ) ); @@ -126,6 +135,7 @@ var Profile = new Class({ 'label' : self.el.getElement('.quality_label input').get('value'), 'wait_for' : self.el.getElement('.wait_for_input').get('value'), 'stop_after' : self.el.getElement('.stop_after_input').get('value'), + 'minimum_score' : self.el.getElement('.minimum_score_input').get('value'), 'types': [] }; diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 452aa75..375547c 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -389,8 +389,8 @@ class Release(Plugin): log.info('Ignored: %s', rel['name']) continue - if rel['score'] <= 0: - log.info('Ignored, score "%s" to low: %s', (rel['score'], rel['name'])) + if rel['score'] < quality_custom.get('minimum_score'): + log.info('Ignored, score "%s" to low, need at least "%s": %s', (rel['score'], quality_custom.get('minimum_score'), rel['name'])) continue if rel['size'] <= 50: From 65f0dc25d2dbb103a2ae4179a1bc28919cd26ad2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 11 Oct 2014 14:32:12 +0200 Subject: [PATCH 200/202] Allow 1080p in shitty quality releases --- couchpotato/core/plugins/quality/main.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index f3fbefe..96fb1a3 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -30,10 +30,10 @@ class QualityPlugin(Plugin): {'identifier': 'dvdr', 'size': (3000, 10000), 'median_size': 4500, 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']}, {'identifier': 'dvdrip', 'size': (600, 2400), 'median_size': 1500, 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, {'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr', 'webrip', ('web', 'rip')], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': []}, - {'identifier': 'r5', 'size': (600, 1000), 'median_size': 700, 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p'], 'ext':[]}, - {'identifier': 'tc', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p'], 'ext':[]}, - {'identifier': 'ts', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p'], 'ext':[]}, - {'identifier': 'cam', 'size': (600, 1000), 'median_size': 700, 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': ['720p'], 'ext':[]} + {'identifier': 'r5', 'size': (600, 1000), 'median_size': 700, 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p', '1080p'], 'ext':[]}, + {'identifier': 'tc', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p', '1080p'], 'ext':[]}, + {'identifier': 'ts', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p', '1080p'], 'ext':[]}, + {'identifier': 'cam', 'size': (600, 1000), 'median_size': 700, 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': ['720p', '1080p'], 'ext':[]} ] pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr'] threed_tags = { @@ -278,6 +278,8 @@ class QualityPlugin(Plugin): 'ext': 5, } + scored_on = [] + # Check alt and tags for tag_type in ['identifier', 'alternative', 'tags', 'label']: qualities = quality.get(tag_type, []) @@ -289,10 +291,13 @@ class QualityPlugin(Plugin): log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) score += points.get(tag_type) - if isinstance(alt, (str, unicode)) and ss(alt.lower()) in words: + if isinstance(alt, (str, unicode)) and ss(alt.lower()) in words and ss(alt.lower()) not in scored_on: log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) score += points.get(tag_type) + # Don't score twice on same tag + scored_on.append(ss(alt).lower()) + # Check extention for ext in quality.get('ext', []): if ext == extension: @@ -485,6 +490,7 @@ class QualityPlugin(Plugin): 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, 'Movie Name.2014.720p Web-Dl Aac2.0 h264-ReleaseGroup': {'size': 3800, 'quality': 'brrip'}, 'Movie Name.2014.720p.WEBRip.x264.AC3-ReleaseGroup': {'size': 3000, 'quality': 'scr'}, + 'Movie.Name.2014.1080p.HDCAM.-.ReleaseGroup': {'size': 5300, 'quality': 'cam'}, } correct = 0 From c9926802097059594cd298c5b69f9819718a6627 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 11 Oct 2014 15:35:56 +0200 Subject: [PATCH 201/202] Meta and Middle click not triggering new tab --- couchpotato/static/scripts/couchpotato.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js index 19b9f45..2da96a7 100644 --- a/couchpotato/static/scripts/couchpotato.js +++ b/couchpotato/static/scripts/couchpotato.js @@ -54,16 +54,22 @@ }, pushState: function(e){ - if((!e.meta && Browser.platform.mac) || (!e.control && !Browser.platform.mac)){ + var self = this; + + if((!e.meta && self.isMac()) || (!e.control && !self.isMac())){ (e).preventDefault(); var url = e.target.get('href'); - if(History.getPath() != url) + + // Middle click + if(e.event && e.event.button == 1) + window.open(url); + else if(History.getPath() != url) History.push(url); } }, isMac: function(){ - return Browser.platform.mac + return Browser.platform == 'mac' }, createLayout: function(){ @@ -325,11 +331,12 @@ }, openDerefered: function(e, el){ + var self = this; (e).stop(); var url = 'http://www.dereferer.org/?' + el.get('href'); - if(el.get('target') == '_blank' || (e.meta && Browser.platform.mac) || (e.control && !Browser.platform.mac)) + if(el.get('target') == '_blank' || (e.meta && self.isMac()) || (e.control && !self.isMac())) window.open(url); else window.location = url; From d31a2e2768adea48fd34a890ab7af46d14f358d1 Mon Sep 17 00:00:00 2001 From: Joshua McKinney Date: Sun, 12 Oct 2014 03:01:20 +1100 Subject: [PATCH 202/202] Fix default permissions on files to remove execute bits on files #4053 --- couchpotato/core/_base/_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/_base/_core.py b/couchpotato/core/_base/_core.py index 78d5471..320131b 100644 --- a/couchpotato/core/_base/_core.py +++ b/couchpotato/core/_base/_core.py @@ -290,7 +290,7 @@ config = [{ }, { 'name': 'permission_file', - 'default': '0755', + 'default': '0644', 'label': 'File CHMOD', 'description': 'See Folder CHMOD description, but for files', },