Browse Source

Use guessit for sorting and sample detection (#1916)

* Use guessit for sorting and sample detection

* Fix bad logic in is_sample

* address comments, pt. 1

* address comments pt. 2

* address comments, pt. 3

* don't reference title before assignment

* whoops... overlooked the lowercasing

* add another title safeguard

* prevent uninitialized use of variable

* fix for jobs that should not be sorted

* don't list excluded guessit props in the interface

* insert linebreak between guessit props under pattern key

* use constant for excluded props

* dump COUNTRY_REP

* block rebulk log spam

* remove redundant season default; don't set for episodes

* make substitution regex a raw str
pull/1920/head
jcfp 4 years ago
committed by GitHub
parent
commit
b2cbb8c8d0
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 218
      interfaces/Config/templates/config_sorting.tmpl
  2. 1
      requirements.txt
  3. 4
      sabnzbd/cfg.py
  4. 19
      sabnzbd/constants.py
  5. 5
      sabnzbd/database.py
  6. 0
      sabnzbd/deobfuscate_filenames.py
  7. 7
      sabnzbd/interface.py
  8. 21
      sabnzbd/newsunpack.py
  9. 48
      sabnzbd/nzbstuff.py
  10. 23
      sabnzbd/postproc.py
  11. 5
      sabnzbd/skintext.py
  12. 1643
      sabnzbd/sorting.py
  13. 697
      tests/test_sorting.py

218
interfaces/Config/templates/config_sorting.tmpl

@ -96,6 +96,26 @@
<td>$T('show-us-name')</td> <td>$T('show-us-name')</td>
</tr> </tr>
<tr> <tr>
<td class="align-right"><b>$T('Resolution'):</b></td>
<td>%r</td>
<td>1080p</td>
</tr>
<tr>
<td class="align-right"><b>$T('year'):</b></td>
<td>%y</td>
<td>2021</td>
</tr>
<tr>
<td class="align-right"><b>$T('decade'):</b></td>
<td>%decade</td>
<td>20</td>
</tr>
<tr>
<td>&nbsp;</td>
<td>%0decade</td>
<td>2020</td>
</tr>
<tr>
<td class="align-right"><b>$T('show-seasonNum'):</b></td> <td class="align-right"><b>$T('show-seasonNum'):</b></td>
<td>%s</td> <td>%s</td>
<td>1</td> <td>1</td>
@ -131,11 +151,6 @@
<td>$T('ep-us-name')</td> <td>$T('ep-us-name')</td>
</tr> </tr>
<tr> <tr>
<td class="align-right"><b>$T('Resolution'):</b></td>
<td>%r</td>
<td>1080p</td>
</tr>
<tr>
<td class="align-right"><b>$T('fileExt'):</b></td> <td class="align-right"><b>$T('fileExt'):</b></td>
<td>%ext</td> <td>%ext</td>
<td>avi</td> <td>avi</td>
@ -156,6 +171,43 @@
<td>$T('text')</td> <td>$T('text')</td>
</tr> </tr>
</tbody> </tbody>
<tbody>
<tr>
<th class="align-right"><b>GuessIt</b></th>
<th>$T('sort-pattern')</th>
<th>$T('sort-result')</th>
</tr>
</tbody>
<tbody>
<tr>
<td class="align-right"><b>$T('sort-guessitMeaning'):</b></td>
<td>%GI&lt;$T('sort-guessitProperty')&gt;</td>
<td>$T('guessit-sp-property')</td>
</tr>
<tr>
<td class="align-right"><b></b></td>
<td>%G.I&lt;$T('sort-guessitProperty')&gt;</td>
<td>$T('guessit-dot-property')</td>
</tr>
<tr>
<td class="align-right"><b></b></td>
<td>%G_I&lt;$T('sort-guessitProperty')&gt;</td>
<td>$T('guessit-us-property')</td>
</tr>
<tr>
<td class="align-right"><b>$T('Example')</b></td>
<td>%GI&lt;audio_codec&gt;</td>
<td>DTS</td>
</tr>
<tr>
<td class="align-right" style="vertical-align:top;"><b>$T('Valid properties')</b></td>
<td colspan=2>
<!--#for $prop in $guessit_properties#-->
$prop<br>
<!--#end for#-->
</td>
</tr>
</tbody>
</table> </table>
</div> </div>
<div class="field-pair"> <div class="field-pair">
@ -232,6 +284,21 @@
<tbody> <tbody>
<tr> <tr>
<td class="align-right"><b>$T('sort-title'):</b></td> <td class="align-right"><b>$T('sort-title'):</b></td>
<td>%sn</td>
<td>$T('movie-sp-name') ($T('case-adjusted'))</td>
</tr>
<tr>
<td>&nbsp;</td>
<td>%s.n</td>
<td>$T('movie-dot-name') ($T('case-adjusted'))</td>
</tr>
<tr>
<td>&nbsp;</td>
<td>%s_n</td>
<td>$T('movie-us-name') ($T('case-adjusted'))</td>
</tr>
<tr>
<td class="align-right"><b>$T('sort-title'):</b></td>
<td>%title</td> <td>%title</td>
<td>$T('movie-sp-name')</td> <td>$T('movie-sp-name')</td>
</tr> </tr>
@ -246,29 +313,29 @@
<td>$T('movie-us-name')</td> <td>$T('movie-us-name')</td>
</tr> </tr>
<tr> <tr>
<td class="align-right"><b>$T('year'):</b></td>
<td>%y</td>
<td>2009</td>
</tr>
<tr>
<td class="align-right"><b>$T('Resolution'):</b></td> <td class="align-right"><b>$T('Resolution'):</b></td>
<td>%r</td> <td>%r</td>
<td>1080p</td> <td>1080p</td>
</tr> </tr>
<tr> <tr>
<td class="align-right"><b>$T('extension'):</b></td> <td class="align-right"><b>$T('year'):</b></td>
<td>%ext</td> <td>%y</td>
<td>avi</td> <td>2021</td>
</tr> </tr>
<tr> <tr>
<td class="align-right"><b>$T('decade'):</b></td> <td class="align-right"><b>$T('decade'):</b></td>
<td>%decade</td> <td>%decade</td>
<td>00</td> <td>20</td>
</tr> </tr>
<tr> <tr>
<td>&nbsp;</td> <td>&nbsp;</td>
<td>%0decade</td> <td>%0decade</td>
<td>2000</td> <td>2020</td>
</tr>
<tr>
<td class="align-right"><b>$T('extension'):</b></td>
<td>%ext</td>
<td>avi</td>
</tr> </tr>
<tr> <tr>
<td class="align-right"><b>$T('orgFilename'):</b></td> <td class="align-right"><b>$T('orgFilename'):</b></td>
@ -300,6 +367,43 @@
<td>1</td> <td>1</td>
</tr> </tr>
</tbody> </tbody>
<tbody>
<tr>
<th class="align-right"><b>GuessIt</b></th>
<th>$T('sort-pattern')</th>
<th>$T('sort-result')</th>
</tr>
</tbody>
<tbody>
<tr>
<td class="align-right"><b>$T('sort-guessitMeaning'):</b></td>
<td>%GI&lt;$T('sort-guessitProperty')&gt;</td>
<td>$T('guessit-sp-property')</td>
</tr>
<tr>
<td class="align-right"><b></b></td>
<td>%G.I&lt;$T('sort-guessitProperty')&gt;</td>
<td>$T('guessit-dot-property')</td>
</tr>
<tr>
<td class="align-right"><b></b></td>
<td>%G_I&lt;$T('sort-guessitProperty')&gt;</td>
<td>$T('guessit-us-property')</td>
</tr>
<tr>
<td class="align-right"><b>$T('Example')</b></td>
<td>%GI&lt;audio_codec&gt;</td>
<td>DTS</td>
</tr>
<tr>
<td class="align-right" style="vertical-align:top;"><b>$T('Valid properties')</b></td>
<td colspan=2>
<!--#for $prop in $guessit_properties#-->
$prop<br>
<!--#end for#-->
</td>
</tr>
</tbody>
</table> </table>
</div> </div>
<div class="field-pair"> <div class="field-pair">
@ -369,6 +473,21 @@
<tbody> <tbody>
<tr> <tr>
<td class="align-right"><b>$T('show-name'):</b></td> <td class="align-right"><b>$T('show-name'):</b></td>
<td>%sn</td>
<td>$T('show-sp-name') ($T('case-adjusted'))</td>
</tr>
<tr>
<td>&nbsp;</td>
<td>%s.n</td>
<td>$T('show-dot-name') ($T('case-adjusted'))</td>
</tr>
<tr>
<td>&nbsp;</td>
<td>%s_n</td>
<td>$T('show-us-name') ($T('case-adjusted'))</td>
</tr>
<tr>
<td class="align-right"><b>$T('show-name'):</b></td>
<td>%t</td> <td>%t</td>
<td>$T('show-sp-name')</td> <td>$T('show-sp-name')</td>
</tr> </tr>
@ -383,9 +502,24 @@
<td>$T('show-us-name')</td> <td>$T('show-us-name')</td>
</tr> </tr>
<tr> <tr>
<td class="align-right"><b>$T('Resolution'):</b></td>
<td>%r</td>
<td>1080p</td>
</tr>
<tr>
<td class="align-right"><b>$T('year'):</b></td> <td class="align-right"><b>$T('year'):</b></td>
<td>%y</td> <td>%y</td>
<td>2009</td> <td>2021</td>
</tr>
<tr>
<td class="align-right"><b>$T('decade'):</b></td>
<td>%decade</td>
<td>20</td>
</tr>
<tr>
<td>&nbsp;</td>
<td>%0decade</td>
<td>2020</td>
</tr> </tr>
<tr> <tr>
<td class="align-right"><b>$T('month'):</b></td> <td class="align-right"><b>$T('month'):</b></td>
@ -408,19 +542,14 @@
<td>02</td> <td>02</td>
</tr> </tr>
<tr> <tr>
<td class="align-right"><b>$T('decade'):</b></td> <td class="align-right"><b>$T('ep-name'):</b></td>
<td>%decade</td> <td>%en</td>
<td>00</td> <td>$T('ep-sp-name')</td>
</tr> </tr>
<tr> <tr>
<td>&nbsp;</td> <td>&nbsp;</td>
<td>%0decade</td> <td>%e.n</td>
<td>2000</td> <td>$T('ep-dot-name')</td>
</tr>
<tr>
<td class="align-right"><b>$T('Resolution'):</b></td>
<td>%r</td>
<td>1080p</td>
</tr> </tr>
<tr> <tr>
<td class="align-right"><b>$T('orgFilename'):</b></td> <td class="align-right"><b>$T('orgFilename'):</b></td>
@ -438,6 +567,43 @@
<td>$T('text')</td> <td>$T('text')</td>
</tr> </tr>
</tbody> </tbody>
<tbody>
<tr>
<th class="align-right"><b>GuessIt</b></th>
<th>$T('sort-pattern')</th>
<th>$T('sort-result')</th>
</tr>
</tbody>
<tbody>
<tr>
<td class="align-right"><b>$T('sort-guessitMeaning'):</b></td>
<td>%GI&lt;$T('sort-guessitProperty')&gt;</td>
<td>$T('guessit-sp-property')</td>
</tr>
<tr>
<td class="align-right"><b></b></td>
<td>%G.I&lt;$T('sort-guessitProperty')&gt;</td>
<td>$T('guessit-dot-property')</td>
</tr>
<tr>
<td class="align-right"><b></b></td>
<td>%G_I&lt;$T('sort-guessitProperty')&gt;</td>
<td>$T('guessit-us-property')</td>
</tr>
<tr>
<td class="align-right"><b>$T('Example')</b></td>
<td>%GI&lt;audio_codec&gt;</td>
<td>DTS</td>
</tr>
<tr>
<td class="align-right" style="vertical-align:top;"><b>$T('Valid properties')</b></td>
<td colspan=2>
<!--#for $prop in $guessit_properties#-->
$prop<br>
<!--#end for#-->
</td>
</tr>
</tbody>
</table> </table>
</div> </div>
<div class="field-pair"> <div class="field-pair">

1
requirements.txt

@ -9,6 +9,7 @@ portend
chardet chardet
notify2 notify2
puremagic puremagic
guessit>=3.1.0
# Windows system integration # Windows system integration
pywin32>=227; sys_platform == 'win32' pywin32>=227; sys_platform == 'win32'

4
sabnzbd/cfg.py

@ -231,13 +231,12 @@ rating_filter_pause_keywords = OptionStr("misc", "rating_filter_pause_keywords")
############################################################################## ##############################################################################
enable_tv_sorting = OptionBool("misc", "enable_tv_sorting", False) enable_tv_sorting = OptionBool("misc", "enable_tv_sorting", False)
tv_sort_string = OptionStr("misc", "tv_sort_string") tv_sort_string = OptionStr("misc", "tv_sort_string")
tv_sort_countries = OptionNumber("misc", "tv_sort_countries", 1)
tv_categories = OptionList("misc", "tv_categories", "") tv_categories = OptionList("misc", "tv_categories", "")
enable_movie_sorting = OptionBool("misc", "enable_movie_sorting", False) enable_movie_sorting = OptionBool("misc", "enable_movie_sorting", False)
movie_sort_string = OptionStr("misc", "movie_sort_string") movie_sort_string = OptionStr("misc", "movie_sort_string")
movie_sort_extra = OptionStr("misc", "movie_sort_extra", "-cd%1", strip=False) movie_sort_extra = OptionStr("misc", "movie_sort_extra", "-cd%1", strip=False)
movie_extra_folders = OptionBool("misc", "movie_extra_folder", False) movie_extra_folder = OptionBool("misc", "movie_extra_folder", False)
movie_categories = OptionList("misc", "movie_categories", ["movies"]) movie_categories = OptionList("misc", "movie_categories", ["movies"])
enable_date_sorting = OptionBool("misc", "enable_date_sorting", False) enable_date_sorting = OptionBool("misc", "enable_date_sorting", False)
@ -296,6 +295,7 @@ rss_odd_titles = OptionList("misc", "rss_odd_titles", ["nzbindex.nl/", "nzbindex
req_completion_rate = OptionNumber("misc", "req_completion_rate", 100.2, 100, 200) req_completion_rate = OptionNumber("misc", "req_completion_rate", 100.2, 100, 200)
selftest_host = OptionStr("misc", "selftest_host", "self-test.sabnzbd.org") selftest_host = OptionStr("misc", "selftest_host", "self-test.sabnzbd.org")
movie_rename_limit = OptionStr("misc", "movie_rename_limit", "100M") movie_rename_limit = OptionStr("misc", "movie_rename_limit", "100M")
episode_rename_limit = OptionStr("misc", "episode_rename_limit", "20M")
size_limit = OptionStr("misc", "size_limit", "0") size_limit = OptionStr("misc", "size_limit", "0")
show_sysload = OptionNumber("misc", "show_sysload", 2, 0, 2) show_sysload = OptionNumber("misc", "show_sysload", 2, 0, 2)
history_limit = OptionNumber("misc", "history_limit", 10, 0) history_limit = OptionNumber("misc", "history_limit", 10, 0)

19
sabnzbd/constants.py

@ -123,25 +123,10 @@ CHEETAH_DIRECTIVES = {"directiveStartToken": "<!--#", "directiveEndToken": "#-->
IGNORED_FOLDERS = ("@eaDir", ".appleDouble") IGNORED_FOLDERS = ("@eaDir", ".appleDouble")
# (MATCHER, [EXTRA, MATCHERS]) EXCLUDED_GUESSIT_PROPERTIES = [
series_match = [ "part",
(compile(r"( [sS]|[\d]+)x(\d+)"), [compile(r"^[-\.]+([sS]|[\d])+x(\d+)"), compile(r"^[-\.](\d+)")]), # 1x01
(
compile(r"[Ss](\d+)[\.\-]?[Ee](\d+)"), # S01E01
[compile(r"^[-\.]+[Ss](\d+)[\.\-]?[Ee](\d+)"), compile(r"^[-\.](\d+)")],
),
(compile(r"[ \-_\.](\d)(\d{2,2})[ \-_\.]"), []), # .101. / _101_ / etc.
(compile(r"[ \-_\.](\d)(\d{2,2})$"), []), # .101 at end of title
] ]
date_match = [r"(\d{4})\W(\d{1,2})\W(\d{1,2})", r"(\d{1,2})\W(\d{1,2})\W(\d{4})"] # 2008-10-16 # 10.16.2008
year_match = r"[\W]([1|2]\d{3})([^\w]|$)" # Something '(YYYY)' or '.YYYY.' or ' YYYY '
sample_match = r"((^|[\W_])(sample|proof))" # something-sample or something-proof
resolution_match = r"(^|[\W_])((240|360|480|540|576|720|900|1080|1440|2160|4320)[piP])([\W_]|$)" # 576i, 720p, 1080P
class Status: class Status:
IDLE = "Idle" # Q: Nothing in the queue IDLE = "Idle" # Q: Nothing in the queue

5
sabnzbd/database.py

@ -346,9 +346,8 @@ class HistoryDB:
def have_episode(self, series, season, episode): def have_episode(self, series, season, episode):
"""Check whether History contains this series episode""" """Check whether History contains this series episode"""
total = 0 total = 0
series = series.lower().replace(".", " ").replace("_", " ").replace(" ", " ")
if series and season and episode: if series and season and episode:
pattern = "%s/%s/%s" % (series, season, episode) pattern = "%s/%s/%s" % (series.lower(), season, episode)
if self.execute( if self.execute(
"""SELECT COUNT(*) FROM History WHERE series = ? AND STATUS != ?""", (pattern, Status.FAILED) """SELECT COUNT(*) FROM History WHERE series = ? AND STATUS != ?""", (pattern, Status.FAILED)
): ):
@ -477,7 +476,7 @@ def build_history_info(nzo, workdir_complete="", postproc_time=0, script_output=
# Analyze series info only when job is finished # Analyze series info only when job is finished
series = "" series = ""
if series_info: if series_info:
seriesname, season, episode, _ = sabnzbd.newsunpack.analyse_show(nzo.final_name) seriesname, season, episode = sabnzbd.newsunpack.analyse_show(nzo.final_name)[:3]
if seriesname and season and episode: if seriesname and season and episode:
series = "%s/%s/%s" % (seriesname.lower(), season, episode) series = "%s/%s/%s" % (seriesname.lower(), season, episode)

0
sabnzbd/deobfuscate_filenames.py

7
sabnzbd/interface.py

@ -34,6 +34,7 @@ from random import randint
from xml.sax.saxutils import escape from xml.sax.saxutils import escape
from Cheetah.Template import Template from Cheetah.Template import Template
from typing import Optional, Callable, Union from typing import Optional, Callable, Union
from guessit.api import properties as guessit_properties
import sabnzbd import sabnzbd
import sabnzbd.rss import sabnzbd.rss
@ -71,7 +72,7 @@ from sabnzbd.utils.diskspeed import diskspeedmeasure
from sabnzbd.utils.getperformance import getpystone from sabnzbd.utils.getperformance import getpystone
from sabnzbd.utils.internetspeed import internetspeed from sabnzbd.utils.internetspeed import internetspeed
import sabnzbd.utils.ssdp import sabnzbd.utils.ssdp
from sabnzbd.constants import DEF_STDCONFIG, DEFAULT_PRIORITY, CHEETAH_DIRECTIVES from sabnzbd.constants import DEF_STDCONFIG, DEFAULT_PRIORITY, CHEETAH_DIRECTIVES, EXCLUDED_GUESSIT_PROPERTIES
from sabnzbd.lang import list_languages from sabnzbd.lang import list_languages
from sabnzbd.api import ( from sabnzbd.api import (
list_scripts, list_scripts,
@ -924,6 +925,7 @@ SPECIAL_VALUE_LIST = (
"downloader_sleep_time", "downloader_sleep_time",
"size_limit", "size_limit",
"movie_rename_limit", "movie_rename_limit",
"episode_rename_limit",
"nomedia_marker", "nomedia_marker",
"max_url_retries", "max_url_retries",
"req_completion_rate", "req_completion_rate",
@ -1897,6 +1899,9 @@ class ConfigSorting:
for kw in SORT_LIST: for kw in SORT_LIST:
conf[kw] = config.get_config("misc", kw)() conf[kw] = config.get_config("misc", kw)()
conf["categories"] = list_cats(False) conf["categories"] = list_cats(False)
conf["guessit_properties"] = tuple(
prop for prop in guessit_properties().keys() if prop not in EXCLUDED_GUESSIT_PROPERTIES
)
template = Template( template = Template(
file=os.path.join(sabnzbd.WEB_DIR_CONFIG, "config_sorting.tmpl"), file=os.path.join(sabnzbd.WEB_DIR_CONFIG, "config_sorting.tmpl"),

21
sabnzbd/newsunpack.py

@ -28,6 +28,7 @@ import time
import zlib import zlib
import shutil import shutil
import functools import functools
from typing import Tuple
import sabnzbd import sabnzbd
from sabnzbd.encoding import platform_btou, correct_unknown_encoding, ubtou from sabnzbd.encoding import platform_btou, correct_unknown_encoding, ubtou
@ -2297,16 +2298,18 @@ def crc_calculate(path):
return b"%08x" % (crc & 0xFFFFFFFF) return b"%08x" % (crc & 0xFFFFFFFF)
def analyse_show(name): def analyse_show(name: str) -> Tuple[str, str, str, str, bool]:
"""Do a quick SeasonSort check and return basic facts""" """Do a quick SeasonSort check and return basic facts"""
job = SeriesSorter(None, name, None, None) job = SeriesSorter(None, name, None, None, force=True)
job.match(force=True) if job.matched:
if job.is_match():
job.get_values() job.get_values()
info = job.show_info return (
show_name = info.get("show_name", "").replace(".", " ").replace("_", " ") job.info.get("title", ""),
show_name = show_name.replace(" ", " ") job.info.get("season_num", ""),
return show_name, info.get("season_num", ""), info.get("episode_num", ""), info.get("ep_name", "") job.info.get("episode_num", ""),
job.info.get("ep_name", ""),
job.is_proper(),
)
def pre_queue(nzo: NzbObject, pp, cat): def pre_queue(nzo: NzbObject, pp, cat):
@ -2334,7 +2337,7 @@ def pre_queue(nzo: NzbObject, pp, cat):
str(nzo.bytes), str(nzo.bytes),
" ".join(nzo.groups), " ".join(nzo.groups),
] ]
command.extend(analyse_show(nzo.final_name_with_password)) command.extend(analyse_show(nzo.final_name_with_password)[:4])
command = [fix(arg) for arg in command] command = [fix(arg) for arg in command]
# Fields not in the NZO directly # Fields not in the NZO directly

48
sabnzbd/nzbstuff.py

@ -94,7 +94,6 @@ RE_SUBJECT_FILENAME_QUOTES = re.compile(r'"([^"]*)"')
# Otherwise something that looks like a filename # Otherwise something that looks like a filename
RE_SUBJECT_BASIC_FILENAME = re.compile(r"([\w\-+()'\s.,]+\.[A-Za-z0-9]{2,4})[^A-Za-z0-9]") RE_SUBJECT_BASIC_FILENAME = re.compile(r"([\w\-+()'\s.,]+\.[A-Za-z0-9]{2,4})[^A-Za-z0-9]")
RE_RAR = re.compile(r"(\.rar|\.r\d\d|\.s\d\d|\.t\d\d|\.u\d\d|\.v\d\d)$", re.I) RE_RAR = re.compile(r"(\.rar|\.r\d\d|\.s\d\d|\.t\d\d|\.u\d\d|\.v\d\d)$", re.I)
RE_PROPER = re.compile(r"(^|[\. _-])(PROPER|REAL|REPACK)([\. _-]|$)")
############################################################################## ##############################################################################
@ -1973,39 +1972,38 @@ class NzbObject(TryList):
no_series_dupes = cfg.no_series_dupes() no_series_dupes = cfg.no_series_dupes()
series_propercheck = cfg.series_propercheck() series_propercheck = cfg.series_propercheck()
# abort logic if dupe check is off for both nzb+series # Abort if dupe check is off for both nzb and series
if not no_dupes and not no_series_dupes: if not no_dupes and not no_series_dupes:
return False, False return False, False
series = False series = False
res = False res = False
history_db = HistoryDB()
# dupe check off nzb contents with HistoryDB() as history_db:
if no_dupes: # Dupe check off nzb contents
res = history_db.have_name_or_md5sum(self.final_name, self.md5sum) if no_dupes:
logging.debug( res = history_db.have_name_or_md5sum(self.final_name, self.md5sum)
"Dupe checking NZB in history: filename=%s, md5sum=%s, result=%s", self.filename, self.md5sum, res
)
if not res and cfg.backup_for_duplicates():
res = sabnzbd.backup_exists(self.filename)
logging.debug("Dupe checking NZB against backup: filename=%s, result=%s", self.filename, res)
# dupe check off nzb filename
if not res and no_series_dupes:
series, season, episode, misc = sabnzbd.newsunpack.analyse_show(self.final_name)
if RE_PROPER.match(misc) and series_propercheck:
logging.debug("Dupe checking series+season+ep in history aborted due to PROPER/REAL/REPACK found")
else:
res = history_db.have_episode(series, season, episode)
logging.debug( logging.debug(
"Dupe checking series+season+ep in history: series=%s, season=%s, episode=%s, result=%s", "Dupe checking NZB in history: filename=%s, md5sum=%s, result=%s", self.filename, self.md5sum, res
series,
season,
episode,
res,
) )
if not res and cfg.backup_for_duplicates():
res = sabnzbd.backup_exists(self.filename)
logging.debug("Dupe checking NZB against backup: filename=%s, result=%s", self.filename, res)
# Dupe check off nzb filename
if not res and no_series_dupes:
series, season, episode, _, is_proper = sabnzbd.newsunpack.analyse_show(self.final_name)
if is_proper and series_propercheck:
logging.debug("Dupe checking series+season+ep in history aborted due to PROPER/REAL/REPACK found")
else:
res = history_db.have_episode(series, season, episode)
logging.debug(
"Dupe checking series+season+ep in history: series=%s, season=%s, episode=%s, result=%s",
series,
season,
episode,
res,
)
history_db.close()
return res, series return res, series
def is_gone(self): def is_gone(self):

23
sabnzbd/postproc.py

@ -65,13 +65,12 @@ from sabnzbd.filesystem import (
get_filename, get_filename,
) )
from sabnzbd.nzbstuff import NzbObject from sabnzbd.nzbstuff import NzbObject
from sabnzbd.sorting import Sorter from sabnzbd.sorting import Sorter, is_sample, move_to_parent_directory
from sabnzbd.constants import ( from sabnzbd.constants import (
REPAIR_PRIORITY, REPAIR_PRIORITY,
FORCE_PRIORITY, FORCE_PRIORITY,
POSTPROC_QUEUE_FILE_NAME, POSTPROC_QUEUE_FILE_NAME,
POSTPROC_QUEUE_VERSION, POSTPROC_QUEUE_VERSION,
sample_match,
JOB_ADMIN, JOB_ADMIN,
Status, Status,
VERIFIED_FILE, VERIFIED_FILE,
@ -93,9 +92,6 @@ import sabnzbd.deobfuscate_filenames as deobfuscate
MAX_FAST_JOB_COUNT = 3 MAX_FAST_JOB_COUNT = 3
# Match samples
RE_SAMPLE = re.compile(sample_match, re.I)
class PostProcessor(Thread): class PostProcessor(Thread):
"""PostProcessor thread, designed as Singleton""" """PostProcessor thread, designed as Singleton"""
@ -515,13 +511,12 @@ def process_job(nzo: NzbObject):
# TV/Movie/Date Renaming code part 2 - rename and move files to parent folder # TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok and file_sorter.sort_file: if all_ok and file_sorter.sort_file:
if newfiles: if newfiles:
file_sorter.rename(newfiles, workdir_complete) workdir_complete, ok = file_sorter.rename(newfiles, workdir_complete)
workdir_complete, ok = file_sorter.move(workdir_complete) if not ok:
else: workdir_complete, ok = move_to_parent_directory(workdir_complete)
workdir_complete, ok = file_sorter.rename_with_ext(workdir_complete) if not ok:
if not ok: nzo.set_unpack_info("Unpack", T("Failed to move files"))
nzo.set_unpack_info("Unpack", T("Failed to move files")) all_ok = False
all_ok = False
if cfg.deobfuscate_final_filenames() and all_ok and not nzb_list: if cfg.deobfuscate_final_filenames() and all_ok and not nzb_list:
# Deobfuscate the filenames # Deobfuscate the filenames
@ -749,7 +744,7 @@ def parring(nzo: NzbObject, workdir: str):
# Need to make a copy because it can change during iteration # Need to make a copy because it can change during iteration
single = len(nzo.extrapars) == 1 single = len(nzo.extrapars) == 1
for setname in list(nzo.extrapars): for setname in list(nzo.extrapars):
if cfg.ignore_samples() and RE_SAMPLE.search(setname.lower()): if cfg.ignore_samples() and is_sample(setname.lower()):
continue continue
# Skip sets that were already tried # Skip sets that were already tried
if not verified.get(setname, False): if not verified.get(setname, False):
@ -1156,7 +1151,7 @@ def remove_samples(path):
for root, _dirs, files in os.walk(path): for root, _dirs, files in os.walk(path):
for file_to_match in files: for file_to_match in files:
nr_files += 1 nr_files += 1
if RE_SAMPLE.search(file_to_match): if is_sample(file_to_match):
files_to_delete.append(os.path.join(root, file_to_match)) files_to_delete.append(os.path.join(root, file_to_match))
# Make sure we skip false-positives # Make sure we skip false-positives

5
sabnzbd/skintext.py

@ -827,6 +827,11 @@ SKIN_TEXT = {
"button-DailyF": TT("Daily Folders"), "button-DailyF": TT("Daily Folders"),
"case-adjusted": TT("case-adjusted"), #: Note for title expression in Sorting that does case adjustment "case-adjusted": TT("case-adjusted"), #: Note for title expression in Sorting that does case adjustment
"sortResult": TT("Processed Result"), "sortResult": TT("Processed Result"),
"sort-guessitMeaning": TT("Any property"),
"sort-guessitProperty": TT("property"),
"guessit-sp-property": TT("GuessIt Property"),
"guessit-dot-property": TT("GuessIt.Property"),
"guessit-us-property": TT("GuessIt_Property"),
# Config->Special # Config->Special
"explain-special": TT( "explain-special": TT(
"Rarely used options. For their meaning and explanation, click on the Help button to go to the Wiki page.<br>" "Rarely used options. For their meaning and explanation, click on the Help button to go to the Wiki page.<br>"

1643
sabnzbd/sorting.py

File diff suppressed because it is too large

697
tests/test_sorting.py

@ -18,29 +18,694 @@
""" """
tests.test_sorting - Testing functions in sorting.py tests.test_sorting - Testing functions in sorting.py
""" """
import os
import pyfakefs
import shutil
import sys
from random import choice
from sabnzbd import sorting from sabnzbd import sorting
from tests.testhelper import * from tests.testhelper import *
class TestSorting: class TestSortingFunctions:
@pytest.mark.parametrize( @pytest.mark.parametrize(
"job_name, result", "name, result",
[ [
("Ubuntu.optimized.for.1080p.Screens-Canonical", "1080p"), (
("Debian_for_240i_Scientific_Calculators-FTPMasters", "240i"), "2147.Confinement.2015.1080p.WEB-DL.DD5.1.H264-EMRG",
("OpenBSD Streaming Edition 4320P", "4320p"), # Lower-case result {"type": "movie", "title": "2147 Confinement"},
("Surely.1080p.is.better.than.720p", "720p"), # Last hit wins ), # Digit at the start
("2160p.Campaign.Video", "2160p"), # Resolution at the start (
("Some.Linux.Iso.1234p", ""), # Non-standard resolution "2146.Confinement.1080p.WEB-DL.DD5.1.H264-EMRG",
("No.Resolution.Anywhere", ""), {"type": "movie", "title": "2146 Confinement"},
("not.keeping.its1080p.distance", ""), # No separation ), # No year, guessit sets type to episode
("not_keeping_1440idistance_either", ""), ("Setup.exe", {"type": "unknown", "title": "Setup exe"}), # Guessit uses 'movie' as its default type
("240 is a semiperfect and highly composite number", ""), # Number only (
(480, ""), "25.817.hdtv-rofl",
(None, ""), {"type": "episode", "title": "25", "season": 8, "episode": 17},
), # Guessit comes up with bad episode info: [25, 17]
(
"The.Wonders.of.Usenet.E08.2160p-SABnzbd",
{"type": "episode", "season": 1, "episode": 8},
), # Episode without season
(
"Glade Runner 2094 2022.avi",
{"type": "movie", "title": "Glade Runner 2094", "year": 2022},
), # Double year
("Micro.Maffia.s01.web.aac.x265-Tfoe{{Wollah}}", {"release_group": "Tfoe"}), # Password in jobname
("No.Choking.Part.2.2008.360i-NotLOL", {"part": None, "title": "No Choking Part 2"}), # Part property
(
"John.Hamburger.III.US.S01E01.OMG.WTF.BBQ.4320p.WEB.H265-HeliUM.mkv",
{
"type": "episode",
"episode_title": "OMG WTF BBQ",
"screen_size": "4320p",
"title": "John Hamburger III",
"country": "US",
},
),
("Test Movie 720p HDTV AAC x265 sample-MYgroup", {"release_group": "MYgroup", "other": "Sample"}),
(None, None), # Jobname missing
("", None),
],
)
def test_guess_what(self, name, result):
"""Test guessing quirks"""
if not result:
# Bad input
with pytest.raises(ValueError):
guess = sorting.guess_what(name)
else:
guess = sorting.guess_what(name)
for key, value in result.items():
if value is None:
# Property should not exist in the guess
assert key not in guess
else:
assert guess[key] == value
@pytest.mark.parametrize(
"name, result",
[
("Free.Open.Source.Movie.2001.1080p.WEB-DL.DD5.1.H264-FOSS", False), # Not samples
("Setup.exe", False),
("23.123.hdtv-rofl", False),
("Something.1080p.WEB-DL.DD5.1.H264-EMRG-sample", True), # Samples
("Something.1080p.WEB-DL.DD5.1.H264-EMRG-sample.ogg", True),
("Sumtin_Else_1080p_WEB-DL_DD5.1_H264_proof-EMRG", True),
("Wot.Eva.540i.WEB-DL.aac.H264-Groupie sample.mp4", True),
("file-sample.mkv", True),
("PROOF.JPG", True),
("Bla.s01e02.title.1080p.aac-sample proof.mkv", True),
("Bla.s01e02.title.1080p.aac-proof.mkv", True),
("Bla.s01e02.title.1080p.aac sample proof.mkv", True),
("Bla.s01e02.title.1080p.aac proof.mkv", True),
("Not Death Proof (2022) 1080p x264 (DD5.1) BE Subs", False), # Try to trigger some false positives
("Proof.of.Everything.(2042).4320p.x266-4U", False),
("Crime_Scene_S01E13_Free_Sample_For_Sale_480p-OhDear", False),
("Sample That 2011 480p WEB-DL.H265-aMiGo", False),
("Look at That 2011 540i WEB-DL.H265-NoSample", False),
("NOT A SAMPLE.JPG", False),
],
)
def test_is_sample(self, name, result):
assert sorting.is_sample(name) == result
@pytest.mark.parametrize("platform", ["linux", "darwin", "win32"])
@pytest.mark.parametrize(
"path, result_unix, result_win",
[
("/tmp/test.file", True, True),
("/boot", True, True),
("/y.e.p", True, True),
("/ok/", True, True),
("/this.is.a/full.path", True, True),
("f:\\e.txt", False, True),
("\\\\relative.path", False, True),
("Z:\\some\\thing", False, True),
("Bitte ein Bit", False, False),
("this/is/not/an/abs.path", False, False),
("this\\is\\not\\an\\abs.path", False, False),
("AAA", False, False),
("", False, False),
],
)
def test_is_full_path(self, platform, path, result_unix, result_win):
@set_platform(platform)
def _func():
result = result_win if sabnzbd.WIN32 else result_unix
assert sorting.is_full_path(path) == result
_func()
@pytest.mark.skipif(not sys.platform.startswith("win"), reason="Windows tests")
@pytest.mark.parametrize(
"path, result",
[
("P:\\foo\\bar", "P:\\foo\\bar"),
("FOO\\bar\\", "FOO\\bar"),
("foo\\_bar_", "foo\\bar"),
("foo\\__bar", "foo\\bar"),
("foo\\bar__", "foo\\bar"),
("foo\\ bar ", "foo\\bar"),
("foo\\ bar", "foo\\bar"),
("E:\\foo\\bar _", "E:\\foo\\bar"),
("E:\\foo_\\_bar", "E:\\foo\\bar"),
("E:\\foo._\\bar", "E:\\foo\\bar"),
(".foo\\bar", "foo\\bar"), # Dots
("E:\\\\foo\\bar\\...", "E:\\foo\\bar"),
("E:\\\\foo\\bar\\...", "E:\\foo\\bar"),
("E:\\foo_\\bar\\...", "E:\\foo\\bar"),
("\\\\some.path.\\foo\\_bar_", "\\\\some.path\\foo\\bar"), # UNC
("\\\\some.path.\\foo\\_bar_", "\\\\some.path\\foo\\bar"),
(r"\\?\UNC\SRVR\SHR\__File.txt__", r"\\SRVR\SHR\File.txt"),
("F:\\.path.\\ more\\foo bar ", "F:\\path\\more\\foo bar"), # Drive letter
("c:\\.path.\\ more\\foo bar \\ ", "c:\\path\\more\\foo bar"),
("c:\\foo_.\\bar", "c:\\foo\\bar"), # The remainder are all regression tests
("c:\\foo_ _\\bar", "c:\\foo\\bar"),
("c:\\foo. _\\bar", "c:\\foo\\bar"),
("c:\\foo. .\\bar", "c:\\foo\\bar"),
("c:\\foo. _\\bar", "c:\\foo\\bar"),
("c:\\foo. .\\bar", "c:\\foo\\bar"),
("c:\\__\\foo\\bar", "c:\\foo\\bar"), # No double \\\\ when an entire element is stripped
("c:\\...\\foobar", "c:\\foobar"),
],
)
def test_strip_path_elements_win(self, path, result):
def _func():
assert sorting.strip_path_elements(path) == result
_func()
@pytest.mark.skipif(sys.platform.startswith("win"), reason="Unix tests")
@pytest.mark.parametrize(
"path, result",
[
("/foo/bar", "/foo/bar"),
("FOO/bar/", "FOO/bar"),
("foo/_bar_", "foo/bar"),
("foo/__bar", "foo/bar"),
("foo/bar__", "foo/bar"),
("foo/ bar ", "foo/bar"),
("foo/ bar", "foo/bar"),
("/foo/bar _", "/foo/bar"),
("/foo_/_bar", "/foo/bar"),
("/foo._/bar", "/foo./bar"),
(".foo/bar", ".foo/bar"), # Dots
("/foo/bar/...", "/foo/bar/..."),
("foo_\\bar\\...", "foo_\\bar\\..."),
("foo_./bar", "foo_./bar"), # The remainder are all regression tests
("foo_ _/bar", "foo/bar"),
("foo. _/bar", "foo./bar"),
("foo. ./bar", "foo. ./bar"),
("foo. _/bar", "foo./bar"),
("/foo. ./bar", "/foo. ./bar"),
("/__/foo/bar", "/foo/bar"), # No double // when an entire element is stripped
],
)
def test_strip_path_elements_unix(self, path, result):
def _func():
assert sorting.strip_path_elements(path) == result
_func()
@pytest.mark.parametrize(
"path, result",
[
("/Foo/Bar", "/Foo/Bar"), # Nothing to do
("/{Foo}/Bar", "/foo/Bar"),
("{/Foo/B}ar", "/foo/bar"),
("/{F}oo/B{AR}", "/foo/Bar"), # Multiple
("/F{{O}O/{B}A}R", "/Foo/baR"), # Multiple, overlapping
("/F}oo/B{ar", "/Foo/Bar"), # Wrong order, no lowercasing should be done but { and } removed still
("", ""), ("", ""),
], ],
) )
def test_get_resolution(self, job_name, result): def test_to_lowercase(self, path, result):
assert sorting.get_resolution(job_name) == result assert sorting.to_lowercase(path) == result
def test_has_subdirectory(self):
with pyfakefs.fake_filesystem_unittest.Patcher() as ffs:
pyfakefs.fake_filesystem_unittest.set_uid(0)
# Prep the fake filesystem
for test_dir in ["/another/test/dir", "/some/TEST/DIR"]:
ffs.fs.create_dir(test_dir, perm_bits=755)
# Sanity check
assert os.path.exists(test_dir) is True
assert sorting.has_subdirectory("/") is True
assert sorting.has_subdirectory("/some") is True
assert sorting.has_subdirectory("/another/test/") is True
# No subdirs
assert sorting.has_subdirectory("/another/test/dir") is False
assert sorting.has_subdirectory("/some/TEST/DIR/") is False
# Nonexistent dir
assert sorting.has_subdirectory("/some/TEST/NoSuchDir") is False
assert sorting.has_subdirectory("/some/TEST/NoSuchDir/") is False
# Relative path
assert sorting.has_subdirectory("some/TEST/NoSuchDir") is False
assert sorting.has_subdirectory("some/TEST/NoSuchDir/") is False
assert sorting.has_subdirectory("TEST") is False
assert sorting.has_subdirectory("TEST/") is False
# Empty input
assert sorting.has_subdirectory("") is False
@pytest.mark.parametrize(
"path, result",
[
("/Foo/Bar", False),
("", False),
("%fn", True),
(".%ext", True),
("%fn.%ext", True),
("{%fn}", True), # A single closing lowercase marker is allowed
("{.%ext}", True),
("%fn{}", False), # But not the opening lowercase marker
(".%ext{}", False),
("%fn}}", False), # Nor multiple closing lowercase markers
(".%ext}}", False),
("%ext.%fn", True),
("%ext", False), # Missing dot
("%fn.ext", False),
(".ext", False),
(".fn", False),
("", False),
],
)
def test_ends_in_file(self, path, result):
assert sorting.ends_in_file(path) is result
assert sorting.ends_in_file(os.path.join("/tmp", path)) is result # Prepending makes no difference
assert sorting.ends_in_file("foo.bar-" + path) is result
assert sorting.ends_in_file(path + "-foo.bar") is False # Appending does, obviously
assert sorting.ends_in_file(os.path.join("/tmp", path + "-foo.bar")) is False
@pytest.mark.skipif(sys.platform.startswith("win"), reason="Unix tests")
def test_move_to_parent_directory_unix(self):
# Standard files/dirs
with pyfakefs.fake_filesystem_unittest.Patcher() as ffs:
pyfakefs.fake_filesystem_unittest.set_uid(0)
# Create a fake filesystem with some file content in a random base directory
base_dir = "/" + os.urandom(4).hex() + "/" + os.urandom(2).hex()
for test_dir in ["dir/2", "TEST/DIR2"]:
ffs.fs.create_dir(base_dir + "/" + test_dir, perm_bits=755)
assert os.path.exists(base_dir + "/" + test_dir) is True
for test_file in ["dir/some.file", "TEST/DIR/FILE"]:
ffs.fs.create_file(base_dir + "/" + test_file, int("0644", 8))
assert os.path.exists(base_dir + "/" + test_file) is True
return_path, return_status = sorting.move_to_parent_directory(base_dir + "/TEST")
# Affected by move
assert not os.path.exists(base_dir + "/TEST/DIR/FILE") # Moved to subdir
assert not os.path.exists(base_dir + "/TEST/DIR2") # Deleted empty directory
assert not os.path.exists(base_dir + "/DIR2") # Dirs don't get moved, only their file content
assert os.path.exists(base_dir + "/DIR/FILE") # Moved file
# Not moved
assert not os.path.exists(base_dir + "/some.file")
assert not os.path.exists(base_dir + "/2")
assert os.path.exists(base_dir + "/dir/some.file")
assert os.path.exists(base_dir + "/dir/2")
# Function return values
assert (return_path) == base_dir
assert (return_status) is True
# Exception for DVD directories
with pyfakefs.fake_filesystem_unittest.Patcher() as ffs:
pyfakefs.fake_filesystem_unittest.set_uid(0)
# Create a fake filesystem in a random base directory, and included a typical DVD directory
base_dir = "/" + os.urandom(4).hex() + "/" + os.urandom(2).hex()
dvd = choice(("video_ts", "audio_ts", "bdmv"))
for test_dir in ["dir/2", "TEST/DIR2"]:
ffs.fs.create_dir(base_dir + "/" + test_dir, perm_bits=755)
assert os.path.exists(base_dir + "/" + test_dir) is True
for test_file in ["dir/some.file", "TEST/" + dvd + "/FILE"]:
ffs.fs.create_file(base_dir + "/" + test_file, int("0644", 8))
assert os.path.exists(base_dir + "/" + test_file) is True
return_path, return_status = sorting.move_to_parent_directory(base_dir + "/TEST")
# Nothing should move in the presence of a DVD directory structure
assert os.path.exists(base_dir + "/TEST/" + dvd + "/FILE")
assert os.path.exists(base_dir + "/TEST/DIR2")
assert not os.path.exists(base_dir + "/DIR2")
assert not os.path.exists(base_dir + "/DIR/FILE")
assert not os.path.exists(base_dir + "/some.file")
assert not os.path.exists(base_dir + "/2")
assert os.path.exists(base_dir + "/dir/some.file")
assert os.path.exists(base_dir + "/dir/2")
# Function return values
assert (return_path) == base_dir + "/TEST"
assert (return_status) is True
@pytest.mark.skipif(not sys.platform.startswith("win"), reason="Windows tests")
def test_move_to_parent_directory_win(self):
# Standard files/dirs
with pyfakefs.fake_filesystem_unittest.Patcher() as ffs:
pyfakefs.fake_filesystem_unittest.set_uid(0)
# Create a fake filesystem with some file content in a random base directory
base_dir = "Z:\\" + os.urandom(4).hex() + "\\" + os.urandom(2).hex()
for test_dir in ["dir\\2", "TEST\\DIR2"]:
ffs.fs.create_dir(base_dir + "\\" + test_dir, perm_bits=755)
assert os.path.exists(base_dir + "\\" + test_dir) is True
for test_file in ["dir\\some.file", "TEST\\DIR\\FILE"]:
ffs.fs.create_file(base_dir + "\\" + test_file, int("0644", 8))
assert os.path.exists(base_dir + "\\" + test_file) is True
return_path, return_status = sorting.move_to_parent_directory(base_dir + "\\TEST")
# Affected by move
assert not os.path.exists(base_dir + "\\TEST\\DIR\\FILE") # Moved to subdir
assert not os.path.exists(base_dir + "\\TEST\\DIR2") # Deleted empty directory
assert not os.path.exists(base_dir + "\\DIR2") # Dirs don't get moved, only their file content
assert os.path.exists(base_dir + "\\DIR\\FILE") # Moved file
# Not moved
assert not os.path.exists(base_dir + "\\some.file")
assert not os.path.exists(base_dir + "\\2")
assert os.path.exists(base_dir + "\\dir\\some.file")
assert os.path.exists(base_dir + "\\dir\\2")
# Function return values
assert (return_path) == base_dir
assert (return_status) is True
# Exception for DVD directories
with pyfakefs.fake_filesystem_unittest.Patcher() as ffs:
pyfakefs.fake_filesystem_unittest.set_uid(0)
# Create a fake filesystem in a random base directory, and included a typical DVD directory
base_dir = "D:\\" + os.urandom(4).hex() + "\\" + os.urandom(2).hex()
dvd = choice(("video_ts", "audio_ts", "bdmv"))
for test_dir in ["dir\\2", "TEST\\DIR2"]:
ffs.fs.create_dir(base_dir + "\\" + test_dir, perm_bits=755)
assert os.path.exists(base_dir + "\\" + test_dir) is True
for test_file in ["dir\\some.file", "TEST\\" + dvd + "\\FILE"]:
ffs.fs.create_file(base_dir + "\\" + test_file, int("0644", 8))
assert os.path.exists(base_dir + "\\" + test_file) is True
return_path, return_status = sorting.move_to_parent_directory(base_dir + "\\TEST")
# Nothing should move in the presence of a DVD directory structure
assert os.path.exists(base_dir + "\\TEST\\" + dvd + "\\FILE")
assert os.path.exists(base_dir + "\\TEST\\DIR2")
assert not os.path.exists(base_dir + "\\DIR2")
assert not os.path.exists(base_dir + "\\DIR\\FILE")
assert not os.path.exists(base_dir + "\\some.file")
assert not os.path.exists(base_dir + "\\2")
assert os.path.exists(base_dir + "\\dir\\some.file")
assert os.path.exists(base_dir + "\\dir\\2")
# Function return values
assert (return_path) == base_dir + "\\TEST"
assert (return_status) is True
@pytest.mark.usefixtures("clean_cache_dir")
class TestSortingSorters:
@pytest.mark.parametrize(
"s_class, jobname, sort_string, result_path, result_setname",
[
(
sorting.DateSorter,
"My.EveryDay.Show.20210203.Great.Success.1080p.aac.hdtv-mygrp.mkv",
"%y-%0m/%t - %y-%0m-%0d - %desc.%ext",
"2021-02",
"My EveryDay Show - 2021-02-03 - Great Success",
),
(
sorting.DateSorter,
"My.EveryDay.Show.20210606.Greater.Successes.2160p.dts.bluray-mygrp.mkv",
"%y-%m/%t - %y-%m-%d - %desc.%ext",
"2021-6",
"My EveryDay Show - 2021-6-6 - Greater Successes",
),
(
sorting.DateSorter,
"ME!.1999.12.31.720p.hd-tv",
"{%t}/%0decade_%r/%t - %y-%0m-%0d.%ext",
"me!/1990_720p",
"ME! - 1999-12-31",
),
(
sorting.DateSorter,
"2000 A.D. 28-01-2000 360i dvd-r.avi",
"%y/%0m/%0d/%r.%dn.%ext",
"2000/01/28",
"360i.2000 A.D. 28-01-2000 360i dvd-r.avi",
),
(sorting.DateSorter, "Allo_Allo_07-SEP-1984", "%y/%0m/%0d/%.t.%ext", "1984/09/07", "Allo.Allo"),
(
sorting.DateSorter,
"www.example.org Allo_Allo_07-SEP-1984",
"%GI<website>/%GI<nonexistent>/%y/%0m/%0d/%.t%GI<audio_codec>.%ext",
"www.example.org/1984/09/07",
"Allo.Allo",
),
(
sorting.SeriesSorter,
"onslow.goes.to.university.s06e66-grp.mkv",
"%sn/Season %s/%sn - %sx%0e - %en.%ext",
"Onslow Goes To University/Season 6",
"Onslow Goes To University - 6x66 - grp",
),
(
sorting.SeriesSorter,
"rose's_BEAUTY_parlour",
"%sn/S%0sE%0e - %en/%sn - S%0sE%0e - %en.%ext",
"Rose's Beauty Parlour/S01E -", # Season defaults to '1' if missing, episode doesn't
"Rose's Beauty Parlour - S01E -",
),
(
sorting.SeriesSorter,
"Cooking with Hyacinth S01E13 Biscuits 2160p DD5.1 Cookies",
"{%s.N}/%sx%0e - %en/%s_N - %0sx%0e - %en (%r).%ext",
"cooking.with.hyacinth/1x13 - Biscuits",
"Cooking_with_Hyacinth - 01x13 - Biscuits (2160p)",
),
(
sorting.SeriesSorter,
"Daisy _ (1987) _ S01E02 _ 480i.avi",
"%dn.%ext",
"",
"Daisy _ (1987) _ S01E02 _ 480i.avi",
),
(
sorting.SeriesSorter,
"Bruce.and.Violet.S126E202.Oh.Dear.Benz.4320p.mkv",
"%sn/Season W%s/W%0e_%desc.%ext",
"Bruce and Violet/Season W126",
"W202",
), # %desc should be stripped, season and episode numbers >=100 handled correctly, and "and" remain lowercase
(
sorting.SeriesSorter,
"[www.sabnzbd.org]Candle.Light.Dinners.S02E13.Elite.Soups.dts.hvec-NZBLuv.mkv",
"%s.N.S%0sE%0e.(%e.n).%G.I<audio_codec>.%GI<website>-%GI<release_group>.%ext",
"",
"Candle.Light.Dinners.S02E13.(Elite.Soups).DTS.www.sabnzbd.org-hvec-NZBLuv",
), # GI<property>
(
sorting.SeriesSorter,
"Candle.Light.Dinners.S02E13.DD+5.1.x265.Hi10-NZBLuv.mkv",
"%s_N_S%0sE%0e_%G_I<video_codec>_%G.I<color_depth>_%G_I<audio_codec>.%ext",
"",
"Candle_Light_Dinners_S02E13_H.265_10-bit_Dolby_Digital_Plus",
), # GI<property> with spacer
(
sorting.MovieSorter,
"Pantomimes.Lumineuses.1982.2160p.WEB-DL.DDP5.1.H.264-TheOpt.mkv",
"%r/%year/%title-%G.I<release_group>.%ext",
"2160p/1982",
"Pantomimes Lumineuses-TheOpt",
),
(
sorting.MovieSorter,
"The Lucky Dog 1921 540i Tape ac3 mono-LnH proof sample.avi",
"%year/%_t-%G.I<other>.%ext",
"1921",
"The_Lucky_Dog-Proof-Sample",
),
(
sorting.MovieSorter,
"Kid_Auto_Races_at_Venice_[2014]",
"%0decades/%y_%_t",
"2010s/2014_Kid_Auto_Races_at_Venice",
"",
),
],
)
@pytest.mark.parametrize("enable_sorting", [0, 1])
@pytest.mark.parametrize("category", ["sortme", "nosort", "*"])
def test_sorter_get_final_path(
self, s_class, enable_sorting, jobname, sort_string, category, result_path, result_setname
):
sort_cats = "*, sortme"
@set_config(
{
"date_sort_string": sort_string,
"date_categories": sort_cats,
"enable_date_sorting": enable_sorting,
"tv_sort_string": sort_string,
"tv_categories": sort_cats,
"enable_tv_sorting": enable_sorting,
"movie_sort_string": sort_string,
"movie_categories": sort_cats,
"enable_movie_sorting": enable_sorting,
"movie_sort_extra": " CD%1",
"movie_extra_folder": 0,
"movie_rename_limit": "100M",
}
)
def _func():
path = ("/tmp/" if not sys.platform.startswith("win") else "c:\\tmp\\") + os.urandom(4).hex()
sorter = s_class(None, jobname, path, category)
if bool(enable_sorting) and category in sort_cats:
if sys.platform.startswith("win"):
assert sorter.get_final_path() == (path + "/" + result_path).replace("/", "\\")
assert sorter.filename_set == result_setname.replace("/", "\\")
else:
assert sorter.get_final_path() == path + "/" + result_path
assert sorter.filename_set == result_setname
else:
if sys.platform.startswith("win"):
assert sorter.get_final_path() == (path + "/" + jobname).replace("/", "\\")
else:
assert sorter.get_final_path() == path + "/" + jobname
assert sorter.filename_set == ""
_func()
@pytest.mark.parametrize(
"s_class, job_tag, sort_string, sort_result", # sort_result without extension
[
(sorting.SeriesSorter, "S01E02", "%r/%sn s%0se%0e.%ext", "Simulated Job s01e02"),
(sorting.MovieSorter, "2021", "%y_%.title.%r.%ext", "2021_Simulated.Job.2160p"),
(sorting.DateSorter, "2020-02-29", "%y/%0m/%0d/%.t-%GI<release_group>", "Simulated.Job-SAB"),
],
)
@pytest.mark.parametrize("size_limit, file_size", [(512, 1024), (1024, 512)])
@pytest.mark.parametrize("extension", [".mkv", ".data", ".mkv", ".vob"])
@pytest.mark.parametrize("number_of_files", [1, 2])
@pytest.mark.parametrize("generate_sequential_filenames", [True, False])
def test_sorter_rename(
self,
s_class,
job_tag,
sort_string,
sort_result,
size_limit,
file_size,
extension,
number_of_files,
generate_sequential_filenames,
):
"""Test the file renaming of the Sorter classes"""
@set_config(
{
"tv_sort_string": sort_string, # TV
"tv_categories": "*",
"enable_tv_sorting": 1,
"movie_sort_string": sort_string, # Movie
"movie_categories": "*",
"enable_movie_sorting": 1,
"movie_sort_extra": " CD%1",
"movie_extra_folder": 0,
"movie_rename_limit": size_limit,
"date_sort_string": sort_string, # Date
"date_categories": "*",
"enable_date_sorting": 1,
"episode_rename_limit": size_limit, # TV & Date
}
)
def _func():
# Make up a job name
job_name = "Simulated.Job." + job_tag + ".2160p.Web.x264-SAB"
# Prep the filesystem
storage_dir = os.path.join(SAB_CACHE_DIR, "complete" + os.urandom(4).hex())
try:
shutil.rmtree(storage_dir)
except FileNotFoundError:
pass
job_dir = os.path.join(storage_dir, job_name)
os.makedirs(job_dir, exist_ok=True)
assert os.path.exists(job_dir) is True
# Create "downloaded" file(s)
all_files = []
fixed_random = os.urandom(8).hex()
for number in range(1, 1 + number_of_files):
if not generate_sequential_filenames:
job_file = os.urandom(8).hex() + extension
else:
job_file = fixed_random + ".CD" + str(number) + extension
job_filepath = os.path.join(job_dir, job_file)
with open(job_filepath, "wb") as f:
f.write(os.urandom(file_size))
assert os.path.exists(job_filepath) is True
all_files.append(job_file)
# Initialise the sorter and rename
sorter = s_class(None, job_name, job_dir, "*", force=True)
sorter.get_values()
sorter.construct_path()
sort_dest, is_ok = sorter.rename(all_files, job_dir, size_limit)
# Check the result
try:
if (
is_ok
and file_size > size_limit
and extension not in sorting.EXCLUDED_FILE_EXTS
and not (sorter.type == "movie" and number_of_files > 1 and not generate_sequential_filenames)
and not (sorter.type != "movie" and number_of_files > 1)
):
# File(s) should be renamed
if number_of_files > 1 and generate_sequential_filenames and sorter.type == "movie":
# Movie sequential file handling
for n in range(1, number_of_files + 1):
expected = os.path.join(sort_dest, sort_result + " CD" + str(n) + extension)
assert os.path.exists(expected)
else:
expected = os.path.join(sort_dest, sort_result + extension)
assert os.path.exists(expected)
else:
# No renaming should happen
expected = os.path.join(sort_dest, job_file)
assert os.path.exists(expected)
except AssertionError:
# Get some insight into what *did* happen and re-raise the error
for root, dirs, files in os.walk(sort_dest):
print(sort_dest, dirs, files)
raise AssertionError()
# Cleanup
try:
shutil.rmtree(storage_dir)
except FileNotFoundError:
pass
_func()
@pytest.mark.parametrize(
"job_name, result_sort_file, result_class",
[
("OGEL.NinjaGo.Masters.of.Jinspitzu.S13.1080p.CN.WEB-DL.AAC2.0.H.264", True, sorting.SeriesSorter),
(
"The.Hunt.for.Blue.November.1990.NORDiC.REMUX.2160p.DV.HDR.UHD-BluRay.HEVC.TrueHD.5.1-SLoWGoaTS",
True,
sorting.MovieSorter,
),
("가요무대.1985-11-18.480p.Sat.KorSub", True, sorting.DateSorter),
("Virus.cmd", False, None),
("SABnzbd 0.3.9 DeadyNas Mono (incl. Python2.3).pkg", False, None),
],
)
def test_sorter_generic(self, job_name, result_sort_file, result_class):
"""Check if the generic sorter makes the right choices"""
generic = sorting.Sorter(None, None)
generic.detect(job_name, SAB_CACHE_DIR)
assert generic.sort_file is result_sort_file
if result_sort_file:
assert generic.sorter
assert generic.sorter.__class__ is result_class
else:
assert not generic.sorter
@pytest.mark.parametrize(
"name, result",
[
("Undrinkable.2010.PROPER", True),
("Undrinkable.2010.EXTENDED.DVDRip.XviD-MoveIt", False),
("The.Choir.S01E02.The.Details.AC3.DVDRip.XviD-AD1100", False),
("The.Choir.S01E02.The.Real.Details.AC3.DVDRip.XviD-AD1100", False),
("The.Choir.S01E02.The.Details.REAL.AC3.DVDRip.XviD-AD1100", True),
("real.steal.2011.dvdrip.xvid.ac3-4lt1n", False),
("The.Stalking.Mad.S88E01.repack.ReaL.PROPER.CONVERT.1080p.WEB.h265-BTS", True),
("The.Stalking.Mad.S88E01.CONVERT.1080p.WEB.h265-BTS", False),
],
)
def test_sorter_is_proper(self, name, result):
"""Test the is_proper method of the BaseSorter class"""
sorter = sorting.BaseSorter.__new__(sorting.BaseSorter) # Skip __init__
sorter.guess = sorting.guess_what(name)
assert sorter.is_proper() is result

Loading…
Cancel
Save