Set version to 3.4.2

Closes #1964

.github/workflows/integration_testing.yml

@@ -8,15 +8,16 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
         os: [ubuntu-20.04]
         include:
+          # TODO: Update to 3.10 when all packages are available, currently lxml is missing
           - name: macOS
             os: macos-latest
-            python-version: 3.9
+            python-version: "3.9"
           - name: Windows
             os: windows-latest
-            python-version: 3.9
+            python-version: "3.9"
 
     steps:
     - uses: actions/checkout@v2
@@ -30,7 +31,7 @@ jobs:
     - name: Install Python dependencies
       run: |
         python --version
-        pip install --upgrade pip
+        pip install --upgrade pip wheel
         pip install --upgrade -r requirements.txt
         pip install --upgrade -r tests/requirements.txt
     - name: Test SABnzbd

PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 1.0
 Name: SABnzbd
-Version: 3.4.1
-Summary: SABnzbd-3.4.1
+Version: 3.4.2
+Summary: SABnzbd-3.4.2
 Home-page: https://sabnzbd.org
 Author: The SABnzbd Team
 Author-email: team@sabnzbd.org

README.mkd

@@ -1,6 +1,19 @@
-Release Notes - SABnzbd 3.4.1
+Release Notes - SABnzbd 3.4.2
 =========================================================
 
+## Bugfixes since 3.4.1
+- Sorting requires at least 1 category to be selected since 3.4.0.
+  Warning will be shown if no category is selected.
+- Sorting would fail if `%ext` or `%fn` was not used.
+- Job failure due to Sorting-problems was not shown in the History.
+- `Ignore Samples` did not remove all sample files.
+- Crash when `.par2` files were missing during download.
+- Prevent scanning the whole file to identify the correct extension.
+- `.rXX`, `.cbz` and `.cbr` extensions were wrongly renamed.
+- Processing unpacked `.par2` files would also process source
+  `.par2` files and could result in duplicate (`.1`) filenames.
+- Always show the number of MB missing during download.
+
 ## Bugfixes since 3.4.0
 - macOS: Failed to run on M1 systems or older macOS versions.
 
@@ -26,7 +39,7 @@ Release Notes - SABnzbd 3.4.1
 
 ## Upgrade notices
 - The download statistics file `totals10.sab` is updated in 3.2.x 
-  version. If you downgrade to 3.1.x or lower, detailed download 
+  version. If you downgrade to 3.1.x or lower, all detailed download 
   statistics will be lost.
 
 ## Known problems and solutions

SABnzbd.py

@@ -46,6 +46,8 @@ try:
     import portend
     import cryptography
     import chardet
+    import guessit
+    import puremagic
 except ImportError as e:
     print("Not all required Python modules are available, please check requirements.txt")
     print("Missing module:", e.name)
@@ -1439,12 +1441,11 @@ def main():
     try:
         cherrypy.engine.start()
     except:
+        # Since the webserver is started by cherrypy in a separate thread, we can't really catch any
+        # start-up errors. This try/except only catches very few errors, the rest is only shown in the console.
         logging.error(T("Failed to start web-interface: "), exc_info=True)
         abort_and_show_error(browserhost, cherryport)
 
-    # Wait for server to become ready
-    cherrypy.engine.wait(cherrypy.process.wspbus.states.STARTED)
-
     if sabnzbd.WIN32:
         if enable_https:
             mode = "s"

interfaces/Config/templates/config_notify.tmpl

@@ -22,6 +22,7 @@
                 <option value="$ct" <!--#if $ct in $getVar($section_label + '_cats') then 'selected="selected"' else ""#-->>$Tspec($ct)</option>
             <!--#end for#-->
         </select>
+        <p>$T('defaultNotifiesAll')</p>
     </div>
 <!--#end def#-->
 
@@ -40,6 +41,7 @@
                         <option value="$ct" <!--#if $ct in $email_cats then 'selected="selected"' else ""#-->>$Tspec($ct)</option>
                     <!--#end for#-->
                 </select>
+                <p>$T('defaultNotifiesAll')</p>
             </div>
         </div>
         <div class="col1">

interfaces/Config/templates/config_sorting.tmpl

@@ -11,12 +11,13 @@
                 <h3>$T('seriesSorting') <a href="$helpuri$help_uri#toc0" target="_blank"><span class="glyphicon glyphicon-question-sign"></span></a></h3>
                 <p>
                     <b>$T('affectedCat')</b><br/>
-                    <select name="tv_categories" multiple="multiple" class="multiple_cats">
+                    <select name="tv_categories" multiple="multiple" class="multiple_cats" required="required">
                         <!--#for $ct in $categories#-->
                         <option value="$ct" <!--#if $ct in $tv_categories then 'selected="selected"' else ""#--> >$Tspec($ct)</option>
                         <!--#end for#-->
                     </select>
                 </p>
+                <p>$T('selectOneCat')</p>
             </div>
             <!-- /col2 -->
             <div class="col1">
@@ -223,12 +224,13 @@
                 <h3>$T('movieSort') <a href="$helpuri$help_uri#toc6" target="_blank"><span class="glyphicon glyphicon-question-sign"></span></a></h3>
                 <p>
                     <b>$T('affectedCat')</b><br/>
-                    <select name="movie_categories" multiple="multiple" class="multiple_cats">
+                    <select name="movie_categories" multiple="multiple" class="multiple_cats" required="required">
                         <!--#for $ct in $categories#-->
                         <option value="$ct" <!--#if $ct in $movie_categories then 'selected="selected"' else ""#--> >$Tspec($ct)</option>
                         <!--#end for#-->
                     </select>
                 </p>
+                <p>$T('selectOneCat')</p>
             </div>
             <!-- /col2 -->
             <div class="col1">
@@ -419,12 +421,13 @@
                 <h3>$T('dateSorting') <a href="$helpuri$help_uri#toc9" target="_blank"><span class="glyphicon glyphicon-question-sign"></span></a></h3>
                 <p>
                     <b>$T('affectedCat')</b><br/>
-                    <select name="date_categories" multiple="multiple" class="multiple_cats">
+                    <select name="date_categories" multiple="multiple" class="multiple_cats" required="required">
                         <!--#for $ct in $categories#-->
                         <option value="$ct" <!--#if $ct in $date_categories then 'selected="selected"' else ""#--> >$Tspec($ct)</option>
                         <!--#end for#-->
                     </select>
                 </p>
+                <p>$T('selectOneCat')</p>
             </div>
             <!-- /col2 -->
             <div class="col1">

interfaces/Glitter/templates/static/javascripts/glitter.queue.js

@@ -531,11 +531,10 @@ function QueueModel(parent, data) {
         return self.name()
     })
     self.missingText = ko.pureComputed(function() {
-        // Check for missing data, the value is arbitrary! (1%)
-        if(self.missingMB()/self.totalMB() > 0.01) {
+        // Check for missing data, can show 0 if article-size is smaller than 500K, but we accept that
+        if(self.missingMB()) {
             return self.missingMB().toFixed(0) + ' MB ' + glitterTranslate.misingArt
         }
-        return;
     })
     self.statusText = ko.computed(function() {
         // Checking

sabnzbd/cfg.py

@@ -233,7 +233,7 @@ rating_filter_pause_keywords = OptionStr("misc", "rating_filter_pause_keywords")
 ##############################################################################
 enable_tv_sorting = OptionBool("misc", "enable_tv_sorting", False)
 tv_sort_string = OptionStr("misc", "tv_sort_string")
-tv_categories = OptionList("misc", "tv_categories", "")
+tv_categories = OptionList("misc", "tv_categories", ["tv"])
 
 enable_movie_sorting = OptionBool("misc", "enable_movie_sorting", False)
 movie_sort_string = OptionStr("misc", "movie_sort_string")

sabnzbd/constants.py

@@ -122,6 +122,7 @@ VALID_NZB_FILES = (".nzb", ".gz", ".bz2")
 CHEETAH_DIRECTIVES = {"directiveStartToken": "<!--#", "directiveEndToken": "#-->", "prioritizeSearchListOverSelf": True}
 
 IGNORED_FOLDERS = ("@eaDir", ".appleDouble")
+IGNORED_MOVIE_FOLDERS = ("video_ts", "audio_ts", "bdmv")
 
 EXCLUDED_GUESSIT_PROPERTIES = [
     "part",

sabnzbd/deobfuscate_filenames.py

@@ -64,9 +64,9 @@ def decode_par2(parfile: str) -> List[str]:
             with open(filepath, "rb") as fileToMatch:
                 first16k_data = fileToMatch.read(16384)
 
-            # Check if we have this hash
+            # Check if we have this hash and the filename is different
             file_md5of16k = hashlib.md5(first16k_data).digest()
-            if file_md5of16k in md5of16k:
+            if file_md5of16k in md5of16k and fn != md5of16k[file_md5of16k]:
                 new_path = os.path.join(dirname, md5of16k[file_md5of16k])
                 # Make sure it's a unique name
                 unique_filename = get_unique_filename(new_path)
@@ -166,7 +166,7 @@ def deobfuscate_list(filelist: List[str], usefulname: str):
     # 2. if no meaningful extension, add it
     # 3. based on detecting obfuscated filenames
 
-    # to be sure, only keep really exsiting files:
+    # to be sure, only keep really existing files:
     filelist = [f for f in filelist if os.path.isfile(f)]
 
     # let's see if there are files with uncommon/unpopular (so: obfuscated) extensions
@@ -176,7 +176,7 @@ def deobfuscate_list(filelist: List[str], usefulname: str):
     for file in filelist:
         if file_extension.has_popular_extension(file):
             # common extension, like .doc or .iso, so assume OK and change nothing
-            logging.debug("extension of %s looks common", file)
+            logging.debug("Extension of %s looks common", file)
             newlist.append(file)
         else:
             # uncommon (so: obfuscated) extension
@@ -220,6 +220,7 @@ def deobfuscate_list(filelist: List[str], usefulname: str):
         # check that file is still there (and not renamed by the secondary renaming process below)
         if not os.path.isfile(filename):
             continue
+
         logging.debug("Deobfuscate inspecting %s", filename)
         # Do we need to rename this file?
         # Criteria: big, not-excluded extension, obfuscated (in that order)

sabnzbd/filesystem.py

@@ -480,6 +480,61 @@ def check_mount(path: str) -> bool:
     return not m
 
 
+RAR_RE = re.compile(r"\.(?P<ext>part\d*\.rar|rar|r\d\d|s\d\d|t\d\d|u\d\d|v\d\d|\d\d\d?\d)$", re.I)
+SPLITFILE_RE = re.compile(r"\.(\d\d\d?\d$)", re.I)
+ZIP_RE = re.compile(r"\.(zip$)", re.I)
+SEVENZIP_RE = re.compile(r"\.7z$", re.I)
+SEVENMULTI_RE = re.compile(r"\.7z\.\d+$", re.I)
+TS_RE = re.compile(r"\.(\d+)\.(ts$)", re.I)
+
+
+def build_filelists(
+    workdir: Optional[str], workdir_complete: Optional[str] = None, check_both: bool = False, check_rar: bool = True
+) -> Tuple[List[str], List[str], List[str], List[str], List[str]]:
+    """Build filelists, if workdir_complete has files, ignore workdir.
+    Optionally scan both directories.
+    Optionally test content to establish RAR-ness
+    """
+    sevens, joinables, zips, rars, ts, filelist = ([], [], [], [], [], [])
+
+    if workdir_complete:
+        filelist.extend(listdir_full(workdir_complete))
+
+    if workdir and (not filelist or check_both):
+        filelist.extend(listdir_full(workdir, recursive=False))
+
+    for file in filelist:
+        # Extra check for rar (takes CPU/disk)
+        file_is_rar = False
+        if check_rar:
+            file_is_rar = rarfile.is_rarfile(file)
+
+        # Run through all the checks
+        if SEVENZIP_RE.search(file) or SEVENMULTI_RE.search(file):
+            # 7zip
+            sevens.append(file)
+        elif SPLITFILE_RE.search(file) and not file_is_rar:
+            # Joinables, optional with RAR check
+            joinables.append(file)
+        elif ZIP_RE.search(file):
+            # ZIP files
+            zips.append(file)
+        elif RAR_RE.search(file):
+            # RAR files
+            rars.append(file)
+        elif TS_RE.search(file):
+            # TS split files
+            ts.append(file)
+
+    logging.debug("build_filelists(): joinables: %s", joinables)
+    logging.debug("build_filelists(): zips: %s", zips)
+    logging.debug("build_filelists(): rars: %s", rars)
+    logging.debug("build_filelists(): 7zips: %s", sevens)
+    logging.debug("build_filelists(): ts: %s", ts)
+
+    return joinables, zips, rars, sevens, ts
+
+
 def safe_fnmatch(f: str, pattern: str) -> bool:
     """fnmatch will fail if the pattern contains any of it's
     key characters, like [, ] or !.

sabnzbd/misc.py

@@ -43,6 +43,7 @@ from sabnzbd.filesystem import userxbit
 TAB_UNITS = ("", "K", "M", "G", "T", "P")
 RE_UNITS = re.compile(r"(\d+\.*\d*)\s*([KMGTP]?)", re.I)
 RE_VERSION = re.compile(r"(\d+)\.(\d+)\.(\d+)([a-zA-Z]*)(\d*)")
+RE_SAMPLE = re.compile(r"((^|[\W_])(sample|proof))", re.I)  # something-sample or something-proof
 RE_IP4 = re.compile(r"inet\s+(addr:\s*)?(\d+\.\d+\.\d+\.\d+)")
 RE_IP6 = re.compile(r"inet6\s+(addr:\s*)?([0-9a-f:]+)", re.I)
 
@@ -808,6 +809,11 @@ def get_all_passwords(nzo) -> List[str]:
     return unique_passwords
 
 
+def is_sample(filename: str) -> bool:
+    """Try to determine if filename is (most likely) a sample"""
+    return bool(re.search(RE_SAMPLE, filename))
+
+
 def find_on_path(targets):
     """Search the PATH for a program and return full path"""
     if sabnzbd.WIN32:

sabnzbd/newsunpack.py

@@ -56,6 +56,8 @@ from sabnzbd.filesystem import (
     setname_from_path,
     get_ext,
     get_filename,
+    TS_RE,
+    build_filelists,
 )
 from sabnzbd.nzbstuff import NzbObject, NzbFile
 from sabnzbd.sorting import SeriesSorter
@@ -63,18 +65,12 @@ import sabnzbd.cfg as cfg
 from sabnzbd.constants import Status
 
 # Regex globals
-RAR_RE = re.compile(r"\.(?P<ext>part\d*\.rar|rar|r\d\d|s\d\d|t\d\d|u\d\d|v\d\d|\d\d\d?\d)$", re.I)
 RAR_RE_V3 = re.compile(r"\.(?P<ext>part\d*)$", re.I)
-
 TARGET_RE = re.compile(r'^(?:File|Target): "(.+)" -')
 EXTRACTFROM_RE = re.compile(r"^Extracting\sfrom\s(.+)")
 EXTRACTED_RE = re.compile(r"^(Extracting|Creating|...)\s+(.*?)\s+OK\s*$")
-SPLITFILE_RE = re.compile(r"\.(\d\d\d?\d$)", re.I)
-ZIP_RE = re.compile(r"\.(zip$)", re.I)
-SEVENZIP_RE = re.compile(r"\.7z$", re.I)
-SEVENMULTI_RE = re.compile(r"\.7z\.\d+$", re.I)
-TS_RE = re.compile(r"\.(\d+)\.(ts$)", re.I)
 
+# Constants
 PAR2_COMMAND = None
 MULTIPAR_COMMAND = None
 RAR_COMMAND = None
@@ -1118,8 +1114,7 @@ def par2_repair(parfile_nzf: NzbFile, nzo: NzbObject, workdir, setname, single):
         readd = False
         for extrapar in nzo.extrapars[setname][:]:
             # Make sure we only get new par2 files
-            if extrapar not in nzo.finished_files and extrapar not in nzo.files:
-                nzo.add_parfile(extrapar)
+            if nzo.add_parfile(extrapar):
                 readd = True
         if readd:
             return readd, result
@@ -1995,51 +1990,6 @@ def rar_sort(a, b):
         return cmp(a, b)
 
 
-def build_filelists(workdir, workdir_complete=None, check_both=False, check_rar=True):
-    """Build filelists, if workdir_complete has files, ignore workdir.
-    Optionally scan both directories.
-    Optionally test content to establish RAR-ness
-    """
-    sevens, joinables, zips, rars, ts, filelist = ([], [], [], [], [], [])
-
-    if workdir_complete:
-        filelist.extend(listdir_full(workdir_complete))
-
-    if workdir and (not filelist or check_both):
-        filelist.extend(listdir_full(workdir, recursive=False))
-
-    for file in filelist:
-        # Extra check for rar (takes CPU/disk)
-        file_is_rar = False
-        if check_rar:
-            file_is_rar = rarfile.is_rarfile(file)
-
-        # Run through all the checks
-        if SEVENZIP_RE.search(file) or SEVENMULTI_RE.search(file):
-            # 7zip
-            sevens.append(file)
-        elif SPLITFILE_RE.search(file) and not file_is_rar:
-            # Joinables, optional with RAR check
-            joinables.append(file)
-        elif ZIP_RE.search(file):
-            # ZIP files
-            zips.append(file)
-        elif RAR_RE.search(file):
-            # RAR files
-            rars.append(file)
-        elif TS_RE.search(file):
-            # TS split files
-            ts.append(file)
-
-    logging.debug("build_filelists(): joinables: %s", joinables)
-    logging.debug("build_filelists(): zips: %s", zips)
-    logging.debug("build_filelists(): rars: %s", rars)
-    logging.debug("build_filelists(): 7zips: %s", sevens)
-    logging.debug("build_filelists(): ts: %s", ts)
-
-    return joinables, zips, rars, sevens, ts
-
-
 def quick_check_set(set, nzo):
     """Check all on-the-fly md5sums of a set"""
     md5pack = nzo.md5packs.get(set)

sabnzbd/nzbstuff.py

@@ -1105,8 +1105,7 @@ class NzbObject(TryList):
                 self.postpone_pars(nzf, setname)
             # Get the next one
             for new_nzf in self.extrapars[setname]:
-                if not new_nzf.completed:
-                    self.add_parfile(new_nzf)
+                if self.add_parfile(new_nzf):
                     # Add it to the top
                     self.files.remove(new_nzf)
                     self.files.insert(0, new_nzf)
@@ -1143,8 +1142,8 @@ class NzbObject(TryList):
             added_blocks = 0
             while added_blocks < needed_blocks:
                 new_nzf = block_list.pop()
-                self.add_parfile(new_nzf)
-                added_blocks += new_nzf.blocks
+                if self.add_parfile(new_nzf):
+                    added_blocks += new_nzf.blocks
 
             logging.info("Added %s blocks to %s", added_blocks, self.final_name)
             return added_blocks
@@ -1433,15 +1432,18 @@ class NzbObject(TryList):
             self.unwanted_ext = 2
 
     @synchronized(NZO_LOCK)
-    def add_parfile(self, parfile: NzbFile):
+    def add_parfile(self, parfile: NzbFile) -> bool:
         """Add parfile to the files to be downloaded
         Resets trylist just to be sure
         Adjust download-size accordingly
+        Returns False when the file couldn't be added
         """
         if not parfile.completed and parfile not in self.files and parfile not in self.finished_files:
             parfile.reset_try_list()
             self.files.append(parfile)
             self.bytes_tried -= parfile.bytes_left
+            return True
+        return False
 
     @synchronized(NZO_LOCK)
     def remove_parset(self, setname: str):
@@ -1468,12 +1470,12 @@ class NzbObject(TryList):
                 # from all the sets. This probably means we get too much par2, but it's worth it.
                 blocks_new = 0
                 for new_nzf in self.extrapars[parset]:
-                    self.add_parfile(new_nzf)
-                    blocks_new += new_nzf.blocks
-                    # Enough now?
-                    if blocks_new >= self.bad_articles:
-                        logging.info("Prospectively added %s repair blocks to %s", blocks_new, self.final_name)
-                        break
+                    if self.add_parfile(new_nzf):
+                        blocks_new += new_nzf.blocks
+                        # Enough now?
+                        if blocks_new >= self.bad_articles:
+                            logging.info("Prospectively added %s repair blocks to %s", blocks_new, self.final_name)
+                            break
             # Reset NZO TryList
             self.reset_try_list()
 

sabnzbd/postproc.py

@@ -39,7 +39,7 @@ from sabnzbd.newsunpack import (
     is_sfv_file,
 )
 from threading import Thread
-from sabnzbd.misc import on_cleanup_list
+from sabnzbd.misc import on_cleanup_list, is_sample
 from sabnzbd.filesystem import (
     real_path,
     get_unique_path,
@@ -65,7 +65,7 @@ from sabnzbd.filesystem import (
     get_filename,
 )
 from sabnzbd.nzbstuff import NzbObject
-from sabnzbd.sorting import Sorter, is_sample
+from sabnzbd.sorting import Sorter
 from sabnzbd.constants import (
     REPAIR_PRIORITY,
     FORCE_PRIORITY,
@@ -74,6 +74,7 @@ from sabnzbd.constants import (
     JOB_ADMIN,
     Status,
     VERIFIED_FILE,
+    IGNORED_MOVIE_FOLDERS,
 )
 from sabnzbd.nzbparser import process_single_nzb
 import sabnzbd.emailer as emailer
@@ -499,7 +500,7 @@ def process_job(nzo: NzbObject):
                     )
                     logging.info("Traceback: ", exc_info=True)
                     # Better disable sorting because filenames are all off now
-                    file_sorter.sort_file = None
+                    file_sorter.sorter_active = None
 
             if empty:
                 job_result = -1
@@ -510,17 +511,19 @@ def process_job(nzo: NzbObject):
                 remove_samples(workdir_complete)
 
             # TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
-            if all_ok and file_sorter.sort_file:
+            if all_ok and file_sorter.sorter_active:
                 if newfiles:
                     workdir_complete, ok = file_sorter.sorter.rename(newfiles, workdir_complete)
                     if not ok:
                         nzo.set_unpack_info("Unpack", T("Failed to move files"))
+                        nzo.fail_msg = T("Failed to move files")
                         all_ok = False
 
             # Run further post-processing
             if (all_ok or not cfg.safe_postproc()) and not nzb_list:
                 # Use par2 files to deobfuscate unpacked file names
-                if cfg.process_unpacked_par2():
+                # Only if we also run cleanup, so not to process the "regular" par2 files
+                if flag_delete and cfg.process_unpacked_par2():
                     newfiles = deobfuscate.recover_par2_names(newfiles)
 
                 if cfg.deobfuscate_final_filenames():
@@ -691,7 +694,7 @@ def prepare_extraction_path(nzo: NzbObject) -> Tuple[str, str, Sorter, bool, Opt
     else:
         file_sorter = Sorter(None, nzo.cat)
     complete_dir = file_sorter.detect(nzo.final_name, complete_dir)
-    if file_sorter.sort_file:
+    if file_sorter.sorter_active:
         one_folder = False
 
     complete_dir = sanitize_and_trim_path(complete_dir)
@@ -1175,7 +1178,7 @@ def rename_and_collapse_folder(oldpath, newpath, files):
     if len(items) == 1:
         folder = items[0]
         folder_path = os.path.join(oldpath, folder)
-        if os.path.isdir(folder_path) and folder not in ("VIDEO_TS", "AUDIO_TS"):
+        if os.path.isdir(folder_path) and folder.lower() not in IGNORED_MOVIE_FOLDERS:
             logging.info("Collapsing %s", os.path.join(newpath, folder))
             oldpath = folder_path
 

sabnzbd/skintext.py

@@ -701,6 +701,9 @@ SKIN_TEXT = {
     "link-download": TT("Download"),  #: Config->RSS button "download item"
     "button-rssNow": TT("Read All Feeds Now"),  #: Config->RSS button
     # Config->Notifications
+    "defaultNotifiesAll": TT(
+        "If only the <em>Default</em> category is selected, notifications are enabled for jobs in all categories."
+    ),
     "opt-email_endjob": TT("Email Notification On Job Completion"),
     "email-never": TT("Never"),  #: When to send email
     "email-always": TT("Always"),  #: When to send email
@@ -773,6 +776,7 @@ SKIN_TEXT = {
     "catTags": TT("Indexer Categories / Groups"),
     "button-delCat": TT("X"),  #: Small delete button
     # Config->Sorting
+    "selectOneCat": TT("Select at least 1 category."),
     "seriesSorting": TT("Series Sorting"),
     "opt-tvsort": TT("Enable TV Sorting"),
     "sort-legenda": TT("Pattern Key"),

sabnzbd/sorting.py

@@ -38,7 +38,8 @@ from sabnzbd.filesystem import (
     clip_path,
 )
 import sabnzbd.cfg as cfg
-from sabnzbd.constants import EXCLUDED_GUESSIT_PROPERTIES
+from sabnzbd.constants import EXCLUDED_GUESSIT_PROPERTIES, IGNORED_MOVIE_FOLDERS
+from sabnzbd.misc import is_sample
 from sabnzbd.nzbstuff import NzbObject, scan_password
 
 # Do not rename .vob files as they are usually DVD's
@@ -76,7 +77,7 @@ class BaseSorter:
         self.cat = cat
         self.filename_set = ""
         self.fname = ""  # Value for %fn substitution in folders
-        self.do_rename = False
+        self.rename_files = False
         self.info = {}
         self.type = None
         self.guess = guess
@@ -259,7 +260,7 @@ class BaseSorter:
         # Split the last part of the path up for the renamer
         if extension:
             path, self.filename_set = os.path.split(path)
-            self.do_rename = True
+            self.rename_files = True
 
         # The normpath function translates "" to "." which results in an incorrect path
         return os.path.normpath(path) if path else path
@@ -305,7 +306,7 @@ class BaseSorter:
                 except:
                     logging.error(T("Failed to rename: %s to %s"), clip_path(current_path), clip_path(newpath))
                     logging.info("Traceback: ", exc_info=True)
-                rename_similar(current_path, ext, self.filename_set, ())
+                rename_similar(current_path, ext, self.filename_set)
         else:
             logging.debug("Nothing to rename, %s", files)
 
@@ -317,7 +318,7 @@ class Sorter:
 
     def __init__(self, nzo: Optional[NzbObject], cat: str):
         self.sorter: Optional[BaseSorter] = None
-        self.sort_file = False
+        self.sorter_active = False
         self.nzo = nzo
         self.cat = cat
 
@@ -334,9 +335,9 @@ class Sorter:
             self.sorter = MovieSorter(self.nzo, job_name, complete_dir, self.cat, guess)
 
         if self.sorter and self.sorter.matched:
-            self.sort_file = True
+            self.sorter_active = True
 
-        return self.sorter.get_final_path() if self.sort_file else complete_dir
+        return self.sorter.get_final_path() if self.sorter_active else complete_dir
 
 
 class SeriesSorter(BaseSorter):
@@ -357,12 +358,17 @@ class SeriesSorter(BaseSorter):
     def match(self):
         """Try to guess series info if config and category sort out or force is set"""
         if self.force or (cfg.enable_tv_sorting() and cfg.tv_sort_string() and self.cat.lower() in self.cats):
-            self.guess = guess_what(self.original_job_name, sort_type="episode")
+            if not self.guess:
+                self.guess = guess_what(self.original_job_name, sort_type="episode")
             if self.guess.get("type") == "episode" and "date" not in self.guess:
                 logging.debug("Using tv sorter for %s", self.original_job_name)
                 self.matched = True
                 self.type = "tv"
 
+        # Require at least 1 category, this was not enforced before 3.4.0
+        if cfg.enable_tv_sorting() and not self.cats:
+            logging.warning("%s: %s", T("Series Sorting"), T("Select at least 1 category."))
+
     def get_values(self):
         """Collect all values needed for path replacement"""
         self.get_year()
@@ -394,8 +400,8 @@ class SeriesSorter(BaseSorter):
         """Rename for Series"""
         if min_size < 0:
             min_size = cfg.episode_rename_limit.get_int()
-        if not self.do_rename:
-            return current_path, False
+        if not self.rename_files:
+            return move_to_parent_directory(current_path)
         else:
             logging.debug("Renaming series file(s)")
             return super().rename(files, current_path, min_size)
@@ -420,12 +426,17 @@ class MovieSorter(BaseSorter):
     def match(self):
         """Try to guess movie info if config and category sort out or force is set"""
         if self.force or (cfg.enable_movie_sorting() and self.sort_string and self.cat.lower() in self.cats):
-            self.guess = guess_what(self.original_job_name, sort_type="movie")
+            if not self.guess:
+                self.guess = guess_what(self.original_job_name, sort_type="movie")
             if self.guess.get("type") == "movie":
                 logging.debug("Using movie sorter for %s", self.original_job_name)
                 self.matched = True
                 self.type = "movie"
 
+        # Require at least 1 category, this was not enforced before 3.4.0
+        if cfg.enable_movie_sorting() and not self.cats:
+            logging.warning("%s: %s", T("Movie Sorting"), T("Select at least 1 category."))
+
     def get_values(self):
         """Collect all values needed for path replacement"""
         self.get_year()
@@ -437,8 +448,9 @@ class MovieSorter(BaseSorter):
         if min_size < 0:
             min_size = cfg.movie_rename_limit.get_int()
 
-        if not self.do_rename:
-            return current_path, False
+        if not self.rename_files:
+            return move_to_parent_directory(current_path)
+
         logging.debug("Renaming movie file(s)")
 
         def filter_files(f, current_path):
@@ -500,12 +512,17 @@ class DateSorter(BaseSorter):
     def match(self):
         """Checks the category for a match, if so set self.matched to true"""
         if self.force or (cfg.enable_date_sorting() and self.sort_string and self.cat.lower() in self.cats):
-            self.guess = guess_what(self.original_job_name, sort_type="episode")
+            if not self.guess:
+                self.guess = guess_what(self.original_job_name, sort_type="episode")
             if self.guess.get("type") == "episode" and "date" in self.guess:
                 logging.debug("Using date sorter for %s", self.original_job_name)
                 self.matched = True
                 self.type = "date"
 
+        # Require at least 1 category, this was not enforced before 3.4.0
+        if cfg.enable_date_sorting() and not self.cats:
+            logging.warning("%s: %s", T("Date Sorting"), T("Select at least 1 category."))
+
     def get_date(self):
         """Get month and day"""
         self.info["month"] = str(self.guess.get("date").month)
@@ -526,8 +543,8 @@ class DateSorter(BaseSorter):
         """Renaming Date file"""
         if min_size < 0:
             min_size = cfg.episode_rename_limit.get_int()
-        if not self.do_rename:
-            return current_path, False
+        if not self.rename_files:
+            return move_to_parent_directory(current_path)
         else:
             logging.debug("Renaming date file(s)")
             return super().rename(files, current_path, min_size)
@@ -546,9 +563,11 @@ def move_to_parent_directory(workdir: str) -> Tuple[str, bool]:
     workdir = os.path.abspath(os.path.normpath(workdir))
     dest = os.path.abspath(os.path.normpath(os.path.join(workdir, "..")))
 
+    logging.debug("Moving all files from %s to %s", workdir, dest)
+
     # Check for DVD folders and bail out if found
     for item in os.listdir(workdir):
-        if item.lower() in ("video_ts", "audio_ts", "bdmv"):
+        if item.lower() in IGNORED_MOVIE_FOLDERS:
             return workdir, True
 
     for root, dirs, files in os.walk(workdir):
@@ -612,40 +631,9 @@ def guess_what(name: str, sort_type: Optional[str] = None) -> MatchesDict:
         ):
             guess["type"] = "unknown"
 
-    # Remove sample indicators from groupnames, e.g. 'sample-groupname' or 'groupname-proof'
-    group = guess.get("release_group", "")
-    if group.lower().startswith(("sample-", "proof-")) or group.lower().endswith(("-sample", "-proof")):
-        # Set clean groupname
-        guess["release_group"] = re.sub("^(sample|proof)-|-(sample|proof)$", "", group, re.I)
-        # Add 'Sample' property to the guess
-        other = guess.get("other")
-        if not other:
-            guess.setdefault("other", "Sample")
-        else:
-            if "Sample" not in guess["other"]:
-                # Pre-existing 'other' may be a string or a list
-                try:
-                    guess["other"].append("Sample")
-                except AttributeError:
-                    guess["other"] = [other, "Sample"]
-
     return guess
 
 
-def is_sample(filename: str) -> bool:
-    """Try to determine if filename belongs to a sample"""
-    if os.path.splitext(filename)[0].lower().strip() in ("sample", "proof"):
-        # The entire filename is just 'sample.ext' or similar
-        return True
-
-    # If that didn't work, start guessing
-    guess = guess_what(filename).get("other", "")
-    if isinstance(guess, list):
-        return any(item in ("Sample", "Proof") for item in guess)
-    else:
-        return guess in ("Sample", "Proof")
-
-
 def path_subst(path: str, mapping: List[Tuple[str, str]]) -> str:
     """Replace the sort string elements in the path with the real values provided by the mapping;
     non-elements are copied verbatim."""
@@ -794,7 +782,7 @@ def strip_path_elements(path: str) -> str:
     return "\\\\" + path if is_unc else path
 
 
-def rename_similar(folder: str, skip_ext: str, name: str, skipped_files: List[str]):
+def rename_similar(folder: str, skip_ext: str, name: str, skipped_files: Optional[List[str]] = None):
     """Rename all other files in the 'folder' hierarchy after 'name'
     and move them to the root of 'folder'.
     Files having extension 'skip_ext' will be moved, but not renamed.
@@ -807,7 +795,7 @@ def rename_similar(folder: str, skip_ext: str, name: str, skipped_files: List[st
     for root, dirs, files in os.walk(folder):
         for f in files:
             path = os.path.join(root, f)
-            if path in skipped_files:
+            if skipped_files and path in skipped_files:
                 continue
             org, ext = os.path.splitext(f)
             if ext.lower() == skip_ext:
@@ -861,7 +849,7 @@ def eval_sort(sort_type: str, expression: str, name: str = None, multipart: str
     if "%fn" in path:
         path = path.replace("%fn", fname + ".ext")
     else:
-        if sorter.do_rename:
+        if sorter.rename_files:
             path = fpath + ".ext"
         else:
             path += "\\" if sabnzbd.WIN32 else "/"

sabnzbd/utils/file_extension.py

@@ -9,8 +9,7 @@ import puremagic
 import os
 import sys
 from typing import List
-from pathlib import Path
-from sabnzbd.filesystem import get_ext
+from sabnzbd.filesystem import get_ext, RAR_RE
 
 # common extension from https://www.computerhope.com/issues/ch001789.htm
 POPULAR_EXT = (
@@ -168,6 +167,8 @@ DOWNLOAD_EXT = (
     "bdmv",
     "bin",
     "bup",
+    "cbr",
+    "cbz",
     "clpi",
     "crx",
     "db",
@@ -234,16 +235,16 @@ DOWNLOAD_EXT = (
     "xpi",
 )
 
-# combine to one tuple, with unique entries:
+# Combine to one tuple, with unique entries:
 ALL_EXT = tuple(set(POPULAR_EXT + DOWNLOAD_EXT))
-# prepend a dot to each extension, because we work with a leading dot in extensions
+# Prepend a dot to each extension, because we work with a leading dot in extensions
 ALL_EXT = tuple(["." + i for i in ALL_EXT])
 
 
 def has_popular_extension(file_path: str) -> bool:
     """returns boolean if the extension of file_path is a popular, well-known extension"""
     file_extension = get_ext(file_path)
-    return file_extension in ALL_EXT
+    return file_extension in ALL_EXT or RAR_RE.match(file_extension)
 
 
 def all_possible_extensions(file_path: str) -> List[str]:
@@ -264,9 +265,12 @@ def what_is_most_likely_extension(file_path: str) -> str:
 
     # Check if text or NZB, as puremagic is not good at that.
     try:
-        txt = Path(file_path).read_text()
+        # Only read the start, don't need the whole file
+        with open(file_path, "r") as inp_file:
+            txt = inp_file.read(200).lower()
+
         # Yes, a text file ... so let's check if it's even an NZB:
-        if txt.lower().find("<nzb xmlns=") >= 0 or txt.lower().find("!doctype nzb public") >= 0:
+        if "!doctype nzb public" in txt or "<nzb xmlns=" in txt:
             # yes, contains NZB signals:
             return ".nzb"
         else:

sabnzbd/version.py

@@ -5,5 +5,5 @@
 
 # You MUST use double quotes (so " and not ')
 
-__version__ = "develop"
-__baseline__ = "unknown"
+__version__ = "3.4.2"
+__baseline__ = "d8ab19087d541cb3c6bfe123d8f8e2619946995a"

tests/requirements.txt

@@ -7,6 +7,7 @@ pytest-httpbin
 pytest-httpserver
 flaky
 xmltodict
-tavern
+tavern<1.16.2; python_version == '3.6'
+tavern; python_version > '3.6'
 tavalidate
 lxml>=4.5.0 # needed by tavalidate

tests/test_file_extension.py

@@ -29,6 +29,10 @@ class Test_File_Extension:
         assert file_extension.has_popular_extension("blabla/blabla.mkv")
         assert file_extension.has_popular_extension("blabla/blabla.srt")
         assert file_extension.has_popular_extension("djjddj/aaaaa.epub")
+        assert file_extension.has_popular_extension("test/testing.r01")
+        assert file_extension.has_popular_extension("test/testing.s91")
+        assert not file_extension.has_popular_extension("test/testing")
+        assert not file_extension.has_popular_extension("test/testing.rar01")
         assert not file_extension.has_popular_extension("98ads098f098fa.a0ds98f098asdf")
 
     def test_what_is_most_likely_extension(self):

tests/test_misc.py

@@ -215,6 +215,44 @@ class TestMisc:
         os.unlink("test.key")
 
     @pytest.mark.parametrize(
+        "name, result",
+        [
+            ("Free.Open.Source.Movie.2001.1080p.WEB-DL.DD5.1.H264-FOSS", False),  # Not samples
+            ("Setup.exe", False),
+            ("23.123.hdtv-rofl", False),
+            ("Something.1080p.WEB-DL.DD5.1.H264-EMRG-sample", True),  # Samples
+            ("Something.1080p.WEB-DL.DD5.1.H264-EMRG-sample.ogg", True),
+            ("Sumtin_Else_1080p_WEB-DL_DD5.1_H264_proof-EMRG", True),
+            ("Wot.Eva.540i.WEB-DL.aac.H264-Groupie sample.mp4", True),
+            ("file-sample.mkv", True),
+            ("PROOF.JPG", True),
+            ("Bla.s01e02.title.1080p.aac-sample proof.mkv", True),
+            ("Bla.s01e02.title.1080p.aac-proof.mkv", True),
+            ("Bla.s01e02.title.1080p.aac sample proof.mkv", True),
+            ("Bla.s01e02.title.1080p.aac proof.mkv", True),
+            ("Lwtn.s08e26.1080p.web.h264-glhf-sample.par2", True),
+            ("Lwtn.s08e26.1080p.web.h264-glhf-sample.vol001-002.par2", True),
+            ("Look at That 2011 540i WEB-DL.H265-NoSample", False),
+        ],
+    )
+    def test_is_sample(self, name, result):
+        assert misc.is_sample(name) == result
+
+    @pytest.mark.parametrize(
+        "name, result",
+        [
+            ("Not Death Proof (2022) 1080p x264 (DD5.1) BE Subs", False),  # Try to trigger some false positives
+            ("Proof.of.Everything.(2042).4320p.x266-4U", False),
+            ("Crime_Scene_S01E13_Free_Sample_For_Sale_480p-OhDear", False),
+            ("Sample That 2011 480p WEB-DL.H265-aMiGo", False),
+            ("NOT A SAMPLE.JPG", False),
+        ],
+    )
+    def test_is_sample_known_false_positives(self, name, result):
+        """We know these fail, but don't have a better solution for them at the moment."""
+        assert misc.is_sample(name) != result
+
+    @pytest.mark.parametrize(
         "test_input, expected_output",
         [
             (["cmd1", 9, "cmd3"], '"cmd1" "9" "cmd3"'),  # sending all commands as valid string

tests/test_sorting.py

@@ -25,6 +25,7 @@ import sys
 from random import choice
 
 from sabnzbd import sorting
+from sabnzbd.constants import IGNORED_MOVIE_FOLDERS
 from tests.testhelper import *
 
 
@@ -65,7 +66,7 @@ class TestSortingFunctions:
                     "country": "US",
                 },
             ),
-            ("Test Movie 720p HDTV AAC x265 sample-MYgroup", {"release_group": "MYgroup", "other": "Sample"}),
+            ("Test Movie 720p HDTV AAC x265 MYgroup-Sample", {"release_group": "MYgroup", "other": "Sample"}),
             (None, None),  # Jobname missing
             ("", None),
         ],
@@ -85,33 +86,6 @@ class TestSortingFunctions:
                 else:
                     assert guess[key] == value
 
-    @pytest.mark.parametrize(
-        "name, result",
-        [
-            ("Free.Open.Source.Movie.2001.1080p.WEB-DL.DD5.1.H264-FOSS", False),  # Not samples
-            ("Setup.exe", False),
-            ("23.123.hdtv-rofl", False),
-            ("Something.1080p.WEB-DL.DD5.1.H264-EMRG-sample", True),  # Samples
-            ("Something.1080p.WEB-DL.DD5.1.H264-EMRG-sample.ogg", True),
-            ("Sumtin_Else_1080p_WEB-DL_DD5.1_H264_proof-EMRG", True),
-            ("Wot.Eva.540i.WEB-DL.aac.H264-Groupie sample.mp4", True),
-            ("file-sample.mkv", True),
-            ("PROOF.JPG", True),
-            ("Bla.s01e02.title.1080p.aac-sample proof.mkv", True),
-            ("Bla.s01e02.title.1080p.aac-proof.mkv", True),
-            ("Bla.s01e02.title.1080p.aac sample proof.mkv", True),
-            ("Bla.s01e02.title.1080p.aac proof.mkv", True),
-            ("Not Death Proof (2022) 1080p x264 (DD5.1) BE Subs", False),  # Try to trigger some false positives
-            ("Proof.of.Everything.(2042).4320p.x266-4U", False),
-            ("Crime_Scene_S01E13_Free_Sample_For_Sale_480p-OhDear", False),
-            ("Sample That 2011 480p WEB-DL.H265-aMiGo", False),
-            ("Look at That 2011 540i WEB-DL.H265-NoSample", False),
-            ("NOT A SAMPLE.JPG", False),
-        ],
-    )
-    def test_is_sample(self, name, result):
-        assert sorting.is_sample(name) == result
-
     @pytest.mark.parametrize("platform", ["linux", "darwin", "win32"])
     @pytest.mark.parametrize(
         "path, result_unix, result_win",
@@ -315,7 +289,7 @@ class TestSortingFunctions:
             pyfakefs.fake_filesystem_unittest.set_uid(0)
             # Create a fake filesystem in a random base directory, and included a typical DVD directory
             base_dir = "/" + os.urandom(4).hex() + "/" + os.urandom(2).hex()
-            dvd = choice(("video_ts", "audio_ts", "bdmv"))
+            dvd = choice(IGNORED_MOVIE_FOLDERS)
             for test_dir in ["dir/2", "TEST/DIR2"]:
                 ffs.fs.create_dir(base_dir + "/" + test_dir, perm_bits=755)
                 assert os.path.exists(base_dir + "/" + test_dir) is True
@@ -373,7 +347,7 @@ class TestSortingFunctions:
             pyfakefs.fake_filesystem_unittest.set_uid(0)
             # Create a fake filesystem in a random base directory, and included a typical DVD directory
             base_dir = "D:\\" + os.urandom(4).hex() + "\\" + os.urandom(2).hex()
-            dvd = choice(("video_ts", "audio_ts", "bdmv"))
+            dvd = choice(IGNORED_MOVIE_FOLDERS)
             for test_dir in ["dir\\2", "TEST\\DIR2"]:
                 ffs.fs.create_dir(base_dir + "\\" + test_dir, perm_bits=755)
                 assert os.path.exists(base_dir + "\\" + test_dir) is True
@@ -553,11 +527,14 @@ class TestSortingSorters:
         _func()
 
     @pytest.mark.parametrize(
-        "s_class, job_tag, sort_string, sort_result",  # sort_result without extension
+        "s_class, job_tag, sort_string, sort_filename_result",  # Supply sort_filename_result without extension
         [
             (sorting.SeriesSorter, "S01E02", "%r/%sn s%0se%0e.%ext", "Simulated Job s01e02"),
+            (sorting.SeriesSorter, "S01E02", "%r/%sn s%0se%0e", ""),
             (sorting.MovieSorter, "2021", "%y_%.title.%r.%ext", "2021_Simulated.Job.2160p"),
-            (sorting.DateSorter, "2020-02-29", "%y/%0m/%0d/%.t-%GI<release_group>", "Simulated.Job-SAB"),
+            (sorting.MovieSorter, "2021", "%y_%.title.%r", ""),
+            (sorting.DateSorter, "2020-02-29", "%y/%0m/%0d/%.t-%GI<release_group>.%ext", "Simulated.Job-SAB"),
+            (sorting.DateSorter, "2020-02-29", "%y/%0m/%0d/%.t-%GI<release_group>", ""),
         ],
     )
     @pytest.mark.parametrize("size_limit, file_size", [(512, 1024), (1024, 512)])
@@ -569,7 +546,7 @@ class TestSortingSorters:
         s_class,
         job_tag,
         sort_string,
-        sort_result,
+        sort_filename_result,
         size_limit,
         file_size,
         extension,
@@ -631,8 +608,10 @@ class TestSortingSorters:
 
             # Check the result
             try:
+                # If there's no "%ext" in the sort_string, no filenames should be changed
                 if (
                     is_ok
+                    and sort_filename_result
                     and file_size > size_limit
                     and extension not in sorting.EXCLUDED_FILE_EXTS
                     and not (sorter.type == "movie" and number_of_files > 1 and not generate_sequential_filenames)
@@ -642,10 +621,10 @@ class TestSortingSorters:
                     if number_of_files > 1 and generate_sequential_filenames and sorter.type == "movie":
                         # Movie sequential file handling
                         for n in range(1, number_of_files + 1):
-                            expected = os.path.join(sort_dest, sort_result + " CD" + str(n) + extension)
+                            expected = os.path.join(sort_dest, sort_filename_result + " CD" + str(n) + extension)
                             assert os.path.exists(expected)
                     else:
-                        expected = os.path.join(sort_dest, sort_result + extension)
+                        expected = os.path.join(sort_dest, sort_filename_result + extension)
                         assert os.path.exists(expected)
                 else:
                     # No renaming should happen
@@ -699,7 +678,7 @@ class TestSortingSorters:
             generic = sorting.Sorter(None, "test_cat")
             generic.detect(job_name, SAB_CACHE_DIR)
 
-            assert generic.sort_file is result_sort_file
+            assert generic.sorter_active is result_sort_file
             if result_sort_file:
                 assert generic.sorter
                 assert generic.sorter.__class__ is result_class