From c8f8539d776f03ae2357c70317e4520581024286 Mon Sep 17 00:00:00 2001 From: Prinz23 Date: Thu, 17 Jan 2019 23:09:57 +0000 Subject: [PATCH] Add hd/sd quality detection for x265 hevc (to use; remove x265 and hevc from global ignore list). Add sceneQuality tests to common_tests. Add quality determination test cases for singular x265 and hevc tags. Change refactor quality tests and reuse them for sceneQuality tests. --- CHANGES.md | 7 +- sickbeard/common.py | 79 ++++++++++----------- tests/common_tests.py | 189 ++++++++++++++++++++++++++++---------------------- 3 files changed, 152 insertions(+), 123 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 19a5cf0..91cceda 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,9 @@ -### 0.18.10 (2019-01-11 14:00:00 UTC) +### 0.18.11 (2019-01-17 23:10:00 UTC) + +* Add hd/sd quality detection for x265 hevc (to use; remove x265 and hevc from global ignore list) + + +### 0.18.10 (2019-01-11 14:00:00 UTC) * Fix using ampersand with find show search input diff --git a/sickbeard/common.py b/sickbeard/common.py index a71a861..362dc28 100644 --- a/sickbeard/common.py +++ b/sickbeard/common.py @@ -205,20 +205,20 @@ class Quality: from sickbeard import encodingKludge as ek name = ek.ek(os.path.basename, name) - check_name = (lambda quality_list, func=all: func([re.search(q, name, re.I) for q in quality_list])) + name_has = (lambda quality_list, func=all: func([re.search(q, name, re.I) for q in quality_list])) if anime: - sd_options = check_name(['360p', '480p', '848x480', 'XviD'], any) - dvd_options = check_name(['dvd', 'dvdrip'], any) - blue_ray_options = check_name(['bluray', 'blu-ray', 'BD'], any) + sd_options = name_has(['360p', '480p', '848x480', 'XviD'], any) + dvd_options = name_has(['dvd', 'dvdrip'], any) + blue_ray_options = name_has(['bluray', 'blu-ray', 'BD'], any) if sd_options and not dvd_options and not blue_ray_options: return Quality.SDTV if dvd_options: return Quality.SDDVD - hd_options = check_name(['720p', '1280x720', '960x720'], any) - full_hd = check_name(['1080p', '1920x1080'], any) + hd_options = name_has(['720p', '1280x720', '960x720'], any) + full_hd = name_has(['1080p', '1920x1080'], any) if not blue_ray_options: if hd_options and not full_hd: return Quality.HDTV @@ -237,48 +237,45 @@ class Quality: return Quality.HDTV return Quality.UNKNOWN - fmt = '((h.?|x)26[45]|vp9|av1)' + fmt = '((h.?|x)26[45]|vp9|av1|hevc)' webfmt = 'web.?(dl|rip|.%s)' % fmt rips = 'b[r|d]rip' hd_rips = 'blu.?ray|hddvd|%s' % rips - # for check in []: - if check_name(['(hdtv|pdtv|dsr|tvrip)([-]|.((aac|ac3|dd).?\d\.?\d.)*(xvid|%s))' % fmt]) \ - and not (check_name(['(720|1080|2160)[pi]']) or check_name(['hr.ws.pdtv.(x264|h.?264)'])): - return Quality.SDTV - if check_name([webfmt, 'xvid|%s' % fmt]) \ - and not check_name(['(720|1080|2160)[pi]']): - return Quality.SDTV - if check_name(['(dvd.?rip|%s)(.ws)?(.(xvid|divx|%s))?' % (rips, fmt)]) \ - and not check_name(['(720|1080|2160)[pi]']): - return Quality.SDDVD - if check_name(['(xvid|divx|480p)']) \ - and not (check_name(['(720|1080|2160)[pi]']) or check_name(['hr.ws.pdtv.%s' % fmt])): - return Quality.SDTV - if check_name(['720p', 'hdtv', fmt]) or check_name(['hr.ws.pdtv.%s' % fmt]) \ - and not check_name(['(1080|2160)[pi]']): - return Quality.HDTV - if check_name(['720p|1080i', 'hdtv', 'mpeg-?2']) or check_name(['1080[pi].hdtv', 'h.?264']): + if not name_has(['(720|1080|2160)[pi]|720hd']): + if name_has(['(dvd.?rip|%s)(.ws)?(.(xvid|divx|%s))?' % (rips, fmt)]): + return Quality.SDDVD + if (not name_has(['hr.ws.pdtv.(h.?|x)264']) + and (name_has(['(hdtv|pdtv|dsr|tvrip)([-]|.((aac|ac3|dd).?\d\.?\d.)*(xvid|%s))' % fmt]) + or name_has(['(xvid|divx|480p|hevc|x265)']))) \ + or name_has([webfmt, 'xvid|%s' % fmt]): + return Quality.SDTV + + if not name_has(['(1080|2160)[pi]']): + if name_has(['720p']): + if name_has([hd_rips, fmt]): + return Quality.HDBLURAY + if name_has([webfmt]) or name_has(['itunes', fmt]): + return Quality.HDWEBDL + if name_has([fmt]): + return Quality.HDTV + # p2p + if name_has(['720hd']) \ + or name_has(['hr.ws.pdtv.%s' % fmt]): + return Quality.HDTV + if name_has(['720p|1080i', 'hdtv', 'mpeg-?2']) or name_has(['1080[pi].hdtv', 'h.?264']): return Quality.RAWHDTV - if check_name(['1080p', 'hdtv', fmt]): - return Quality.FULLHDTV - if check_name(['720p', webfmt]) or check_name(['720p', 'itunes', fmt]): - return Quality.HDWEBDL - if check_name(['1080p', webfmt]) or check_name(['1080p', 'itunes', fmt]): - return Quality.FULLHDWEBDL - if check_name(['720p', hd_rips, fmt]): - return Quality.HDBLURAY - if check_name(['1080p', hd_rips, fmt]) \ - or (check_name(['1080[pi]', 'remux']) and not check_name(['hdtv'])): + if name_has(['1080[pi]', 'remux']) and not name_has(['hdtv']): return Quality.FULLHDBLURAY - if check_name(['2160p', webfmt]): + if name_has(['1080p']): + if name_has([hd_rips, fmt]) or name_has([hd_rips, 'avc|vc[ -.]?1']): + return Quality.FULLHDBLURAY + if name_has([webfmt]) or name_has(['itunes', fmt]): + return Quality.FULLHDWEBDL + if name_has([fmt]): + return Quality.FULLHDTV + if name_has(['2160p', webfmt]): return Quality.UHD4KWEB - # p2p - if check_name(['720HD']) \ - and not check_name(['(1080|2160)[pi]']): - return Quality.HDTV - if check_name(['1080p', hd_rips, 'avc|vc[-\s.]?1']): - return Quality.FULLHDBLURAY return Quality.UNKNOWN diff --git a/tests/common_tests.py b/tests/common_tests.py index a1063be..baade8f 100644 --- a/tests/common_tests.py +++ b/tests/common_tests.py @@ -1,4 +1,6 @@ import unittest +import warnings +warnings.filterwarnings('ignore', module=r'.*fuz.*', message='.*Sequence.*') import sys import os.path @@ -9,48 +11,8 @@ from sickbeard.common import Quality, wantedQualities from sickbeard.name_parser.parser import NameParser -class QualityTests(unittest.TestCase): - - def check_quality_names(self, quality, cases): - for fn in cases: - second = common.Quality.nameQuality(fn) - self.assertEqual(quality, second, 'fail %s != %s for case: %s' % (quality, second, fn)) - - def check_proper_level(self, cases, is_anime=False): - np = NameParser(False, indexer_lookup=False, try_scene_exceptions=False, testing=True) - for case, level in cases: - p = np.parse(case) - second = common.Quality.get_proper_level(p.extra_info_no_name(), p.version, is_anime) - self.assertEqual(level, second, 'fail %s != %s for case: %s' % (level, second, case)) - - def check_wantedquality_list(self, cases): - for show_quality, result in cases: - sq = common.Quality.combineQualities(*show_quality) - wd = common.wantedQualities() - _ = wd.get_wantedlist(sq, False, common.Quality.NONE, common.UNAIRED, manual=True) - for w, v in wd.iteritems(): - if w == sq: - for u, o in sorted(v.iteritems()): - self.assertEqual(o, result.get(u)) - - def check_wantedquality_get_wantedlist(self, cases): - for show_quality, result in cases: - sq = common.Quality.combineQualities(*show_quality) - wd = common.wantedQualities() - for case, wlist in result: - ka = {'qualities': sq} - ka.update(case) - res = wd.get_wantedlist(**ka) - self.assertEqual(res, wlist) - - # TODO: repack / proper ? air-by-date ? season rip? multi-ep? - - def test_SDTV(self): - - self.assertEqual(common.Quality.compositeStatus(common.DOWNLOADED, common.Quality.SDTV), - common.Quality.statusFromName('Test.Show.S01E02-GROUP.mkv')) - - self.check_quality_names(common.Quality.SDTV, [ +quality_tests = { + common.Quality.SDTV: [ 'Test.Show.S01E02.PDTV.XViD-GROUP', 'Test.Show.S01E02.PDTV.x264-GROUP', 'Test.Show.S01E02.HDTV.XViD-GROUP', @@ -78,10 +40,8 @@ class QualityTests(unittest.TestCase): 'Test.Show.S01E02.WEB.VP9-GROUP', 'Test.Show.S01E02.WEB.AV1-GROUP', 'Test.Show.S01E02.WEBRip.h264-GROUP', - 'Test.Show.S01E02.WEBRip.x264-GROUP']) - - def test_SDDVD(self): - self.check_quality_names(common.Quality.SDDVD, [ + 'Test.Show.S01E02.WEBRip.x264-GROUP'], + common.Quality.SDDVD: [ 'Test.Show.S01E02.DVDRiP.XViD-GROUP', 'Test.Show.S01E02.DVDRiP.DiVX-GROUP', 'Test.Show.S01E02.DVDRiP.x264-GROUP', @@ -95,31 +55,23 @@ class QualityTests(unittest.TestCase): 'Test.Show.S01E02.BDRIP.VP9-GROUP', 'Test.Show.S01E02.BDRIP.WS.XViD-GROUP', 'Test.Show.S01E02.BDRIP.WS.DiVX-GROUP', - 'Test.Show.S01E02.BDRIP.WS.x264-GROUP']) - - def test_HDTV(self): - self.check_quality_names(common.Quality.HDTV, [ + 'Test.Show.S01E02.BDRIP.WS.x264-GROUP'], + common.Quality.HDTV: [ 'Test.Show.S01E02.720p.HDTV.x264-GROUP', 'Test.Show.S01E02.720p.HDTV.VP9-GROUP', 'Test.Show.S01E02.HR.WS.PDTV.x264-GROUP', - 'Test.Show.S01E02.720p.AHDTV.x264-GROUP']) - - def test_RAWHDTV(self): - self.check_quality_names(common.Quality.RAWHDTV, [ + 'Test.Show.S01E02.720p.AHDTV.x264-GROUP'], + common.Quality.RAWHDTV: [ 'Test.Show.S01E02.720p.HDTV.DD5.1.MPEG2-GROUP', 'Test.Show.S01E02.1080i.HDTV.DD2.0.MPEG2-GROUP', 'Test.Show.S01E02.1080i.HDTV.H.264.DD2.0-GROUP', 'Test Show - S01E02 - 1080i HDTV MPA1.0 H.264 - GROUP', - 'Test.Show.S01E02.1080i.HDTV.DD.5.1.h264-GROUP']) - - def test_FULLHDTV(self): - self.check_quality_names(common.Quality.FULLHDTV, [ + 'Test.Show.S01E02.1080i.HDTV.DD.5.1.h264-GROUP'], + common.Quality.FULLHDTV: [ 'Test.Show.S01E02.1080p.HDTV.x264-GROUP', 'Test.Show.S01E02.1080p.HDTV.vp9-GROUP', - 'Test.Show.S01E02.1080p.AHDTV.x264-GROUP']) - - def test_HDWEBDL(self): - self.check_quality_names(common.Quality.HDWEBDL, [ + 'Test.Show.S01E02.1080p.AHDTV.x264-GROUP'], + common.Quality.HDWEBDL: [ 'Test.Show.S01E02.720p.WEB-DL-GROUP', 'Test.Show.S01E02.720p.WEBRip-GROUP', 'Test.Show.S01E02.WEBRip.720p.H.264.AAC.2.0-GROUP', @@ -138,10 +90,8 @@ class QualityTests(unittest.TestCase): 'Test.Show.S01E02.720p.WEB.x265-GROUP', 'Test.Show.S01E02.720p.WEB.vp9-GROUP', 'Test.Show.S01E02.720p.WEBRip.h264-GROUP', - 'Test.Show.S01E02.720p.WEBRip.x264-GROUP']) - - def test_FULLHDWEBDL(self): - self.check_quality_names(common.Quality.FULLHDWEBDL, [ + 'Test.Show.S01E02.720p.WEBRip.x264-GROUP'], + common.Quality.FULLHDWEBDL: [ 'Test.Show.S01E02.1080p.WEB-DL-GROUP', 'Test.Show.S01E02.1080p.WEBRip-GROUP', 'Test.Show.S01E02.WEBRip.1080p.H.264.AAC.2.0-GROUP', @@ -159,32 +109,80 @@ class QualityTests(unittest.TestCase): 'Test.Show.S01E02.1080p.WEB.x265-GROUP', 'Test.Show.S01E02.1080p.WEB.VP9-GROUP', 'Test.Show.S01E02.1080p.WEBRip.h264-GROUP', - 'Test.Show.S01E02.1080p.WEBRip.x264-GROUP']) - - def test_HDBLURAY(self): - self.check_quality_names(common.Quality.HDBLURAY, [ + 'Test.Show.S01E02.1080p.WEBRip.x264-GROUP'], + common.Quality.HDBLURAY: [ 'Test.Show.S01E02.720p.BluRay.x264-GROUP', 'Test.Show.S01E02.720p.BluRay.vp9-GROUP', 'Test.Show.S01E02.720p.HDDVD.x264-GROUP', - 'Test.Show.S01E02.720p.Blu-ray.x264-GROUP']) - - def test_FULLHDBLURAY(self): - self.check_quality_names(common.Quality.FULLHDBLURAY, [ + 'Test.Show.S01E02.720p.Blu-ray.x264-GROUP'], + common.Quality.FULLHDBLURAY: [ 'Test.Show.S01E02.1080p.BluRay.x264-GROUP', 'Test.Show.S01E02.1080p.HDDVD.x264-GROUP', 'Test.Show.S01E02.1080p.Blu-ray.x264-GROUP', 'Test.Show.S01E02.1080p.Blu-ray.vp9-GROUP', - 'Test Show S02 1080p Remux AVC FLAC 5.1']) - - def test_UHD4KWEB(self): - self.check_quality_names(common.Quality.UHD4KWEB, [ + 'Test Show S02 1080p Remux AVC FLAC 5.1'], + common.Quality.UHD4KWEB: [ 'Test.Show.S01E02.2160p.WEBRip.h264-GROUP', 'Test.Show.S01E02.2160p.WEBRip.x264-GROUP', 'Test.Show.S01E02.2160p.WEBRip.x265-GROUP', - 'Test.Show.S01E02.2160p.WEBRip.vp9-GROUP']) + 'Test.Show.S01E02.2160p.WEBRip.vp9-GROUP'], + common.Quality.UNKNOWN: ['Test.Show.S01E02-SiCKGEAR'] +} + + +class QualityTests(unittest.TestCase): + + def check_quality_names(self, quality, cases): + for fn in cases: + second = common.Quality.nameQuality(fn) + self.assertEqual(quality, second, msg='fail [%s] != [%s] for case: %s' % + (Quality.qualityStrings[quality], Quality.qualityStrings[second], fn)) + + def check_proper_level(self, cases, is_anime=False): + np = NameParser(False, indexer_lookup=False, try_scene_exceptions=False, testing=True) + for case, level in cases: + p = np.parse(case) + second = common.Quality.get_proper_level(p.extra_info_no_name(), p.version, is_anime) + self.assertEqual(level, second, 'fail %s != %s for case: %s' % (level, second, case)) + + def check_wantedquality_list(self, cases): + for show_quality, result in cases: + sq = common.Quality.combineQualities(*show_quality) + wd = common.wantedQualities() + _ = wd.get_wantedlist(sq, False, common.Quality.NONE, common.UNAIRED, manual=True) + for w, v in wd.iteritems(): + if w == sq: + for u, o in sorted(v.iteritems()): + self.assertEqual(o, result.get(u)) - def test_UNKNOWN(self): - self.check_quality_names(common.Quality.UNKNOWN, ['Test.Show.S01E02-SiCKGEAR']) + def check_wantedquality_get_wantedlist(self, cases): + for show_quality, result in cases: + sq = common.Quality.combineQualities(*show_quality) + wd = common.wantedQualities() + for case, wlist in result: + ka = {'qualities': sq} + ka.update(case) + res = wd.get_wantedlist(**ka) + self.assertEqual(res, wlist) + + def check_sceneQuality(self, cases): + msg = 'Test case: "%s", actual: [%s] != expected: [%s]' + for show_name, result in cases: + sq = common.Quality.sceneQuality(show_name[0], show_name[1]) + self.assertEqual(result, sq, msg=msg % (show_name[0], Quality.qualityStrings[sq], + Quality.qualityStrings[result])) + + # TODO: repack / proper ? air-by-date ? season rip? multi-ep? + + def test_SDTV(self): + + self.assertEqual(common.Quality.compositeStatus(common.DOWNLOADED, common.Quality.SDTV), + common.Quality.statusFromName('Test.Show.S01E02-GROUP.mkv')) + + def test_qualites(self): + self.longMessage = True + for q, l in quality_tests.iteritems(): + self.check_quality_names(q, l) def test_reverse_parsing(self): self.check_quality_names(common.Quality.SDTV, ['Test Show - S01E02 - SD TV - GROUP']) @@ -3068,6 +3066,35 @@ class QualityTests(unittest.TestCase): ]), ]) + def test_sceneQuality(self): + self.longMessage = True + self.check_sceneQuality([ + (('The.Show.S01E01.720p.HDTV.x264-Group', False), common.Quality.HDTV), + (('The.Show.S01E01.HDTV.x264-Group', False), common.Quality.SDTV), + (('The.Show.S01E01.x265-Group', False), common.Quality.SDTV), + (('The.Show.S02E04.DVDRip.XviD-Group', False), common.Quality.SDDVD), + (('The.Show.S02E04.DVDRip.x265-Group', False), common.Quality.SDDVD), + (('The.Show.S02E04.1080i.HDTV.MPA2.0.H.264-Group', False), common.Quality.RAWHDTV), + (('The.Show.S01E06.720p.BluRay.X264-Group ', False), common.Quality.HDBLURAY), + (('The.Show.S06E06.BluRay.1080p.DD5.1.H.265-Group', False), common.Quality.FULLHDBLURAY), + (('The.Show.S47E79.1080p.WEB.x264-Group', False), common.Quality.FULLHDWEBDL), + (('The.Show.S03E08.720p.WEB-DL.AAC5.1.H.264', False), common.Quality.HDWEBDL), + (('The Show S01E01 720p hevc-Group', False), common.Quality.HDTV), + (('The Show S01E01 720p x265-Group', False), common.Quality.HDTV), + (('The Show S01E01 720p HEVC x265-Group', False), common.Quality.HDTV), + (('The.Show.S01E01.720p.HEVC.x265-Group', False), common.Quality.HDTV), + (('The.Show.S01E01.720p.x265.HEVC-Group', False), common.Quality.HDTV), + (('The.Show.S01E01.1080p.HEVC.x265-Group', False), common.Quality.FULLHDTV), + (('The.Show.s03e11.720p.web.hevc.x265.Group', False), common.Quality.HDWEBDL), + (('The Show (15 Jan 2019) [text] 720HD mp4', False), common.Quality.HDTV), + (('The.Show.s03e11.ep.name.1080p.web.dl.hevc.x265.Group', False), common.Quality.FULLHDWEBDL), + (('The.Show.S03E05.1080p.NF.WEB-DL.DD5.1.HDR.HEVC-Group', False), common.Quality.FULLHDWEBDL), + (('The.Show.S01E10.Name.2160p.UHD.BluRay.REMUX.HDR.HEVC.DTS-HD.MA.5.1', False), common.Quality.UNKNOWN), + (('Show.S01E07.2160p.4K.UHD.10bit.NF.WEBRip.5.1.x265.HEVC-Group', False), common.Quality.UHD4KWEB), + ]) + for q, l in quality_tests.iteritems(): + self.check_sceneQuality([((v, False), q) for v in l]) + # TODO: fix these anime test cases at travis def test_get_proper_level_anime(self): # release_name, expected level