Always deobfuscate names from par2 (#1935)

* Always deobfuscate names from par2

* Different par2 test

* Different par2 test take 2

* Make par2 filename decoding optional and add some typing

* Rename variable

sabnzbd/cfg.py

@@ -190,6 +190,7 @@ replace_spaces = OptionBool("misc", "replace_spaces", False)
 replace_dots = OptionBool("misc", "replace_dots", False)
 safe_postproc = OptionBool("misc", "safe_postproc", True)
 pause_on_post_processing = OptionBool("misc", "pause_on_post_processing", False)
+enable_all_par = OptionBool("misc", "enable_all_par", False)
 sanitize_safe = OptionBool("misc", "sanitize_safe", False)
 cleanup_list = OptionList("misc", "cleanup_list")
 unwanted_extensions = OptionList("misc", "unwanted_extensions")
@@ -199,6 +200,7 @@ new_nzb_on_failure = OptionBool("misc", "new_nzb_on_failure", False)
 history_retention = OptionStr("misc", "history_retention", "0")
 enable_meta = OptionBool("misc", "enable_meta", True)
 
+
 quota_size = OptionStr("misc", "quota_size")
 quota_day = OptionStr("misc", "quota_day")
 quota_resume = OptionBool("misc", "quota_resume", False)
@@ -258,8 +260,8 @@ rss_rate = OptionNumber("misc", "rss_rate", 60, 15, 24 * 60)
 ampm = OptionBool("misc", "ampm", False)
 replace_illegal = OptionBool("misc", "replace_illegal", True)
 start_paused = OptionBool("misc", "start_paused", False)
-enable_all_par = OptionBool("misc", "enable_all_par", False)
 enable_par_cleanup = OptionBool("misc", "enable_par_cleanup", True)
+process_unpacked_par2 = OptionBool("misc", "process_unpacked_par2", True)
 enable_unrar = OptionBool("misc", "enable_unrar", True)
 enable_unzip = OptionBool("misc", "enable_unzip", True)
 enable_7zip = OptionBool("misc", "enable_7zip", True)

sabnzbd/deobfuscate_filenames.py

@@ -36,13 +36,14 @@ import re
 from sabnzbd.filesystem import get_unique_filename, renamer, get_ext
 from sabnzbd.par2file import is_parfile, parse_par2_file
 import sabnzbd.utils.file_extension as file_extension
+from typing import List
 
 # Files to exclude and minimal file size for renaming
 EXCLUDED_FILE_EXTS = (".vob", ".rar", ".par2", ".mts", ".m2ts", ".cpi", ".clpi", ".mpl", ".mpls", ".bdm", ".bdmv")
 MIN_FILE_SIZE = 10 * 1024 * 1024
 
 
-def decode_par2(parfile):
+def decode_par2(parfile: str) -> List[str]:
     """Parse a par2 file and rename files listed in the par2 to their real name. Resturn list of generated files"""
     # Check if really a par2 file
     if not is_parfile(parfile):
@@ -74,7 +75,31 @@ def decode_par2(parfile):
     return new_files
 
 
-def is_probably_obfuscated(myinputfilename):
+def recover_par2_names(filelist: List[str]) -> List[str]:
+    """Find par2 files and use them for renaming"""
+    # Check that files exists
+    filelist = [f for f in filelist if os.path.isfile(f)]
+    # Search for par2 files in the filelist
+    par2_files = [f for f in filelist if f.endswith(".par2")]
+    # Found any par2 files we can use?
+    if not par2_files:
+        logging.debug("No par2 files found to process, running renamer")
+    else:
+        # Run par2 from SABnzbd on them
+        for par2_file in par2_files:
+            # Analyse data and analyse result
+            logging.debug("Deobfuscate par2: handling %s", par2_file)
+            new_files = decode_par2(par2_file)
+            if new_files:
+                logging.debug("Deobfuscate par2 repair/verify finished")
+                filelist += new_files
+                filelist = [f for f in filelist if os.path.isfile(f)]
+            else:
+                logging.debug("Deobfuscate par2 repair/verify did not find anything to rename")
+    return filelist
+
+
+def is_probably_obfuscated(myinputfilename: str) -> bool:
     """Returns boolean if filename is likely obfuscated. Default: True
     myinputfilename (string) can be a plain file name, or a full path"""
 
@@ -133,7 +158,7 @@ def is_probably_obfuscated(myinputfilename):
     return True  # default not obfuscated
 
 
-def deobfuscate_list(filelist, usefulname):
+def deobfuscate_list(filelist: List[str], usefulname: str):
     """Check all files in filelist, and if wanted, deobfuscate: rename to filename based on usefulname"""
 
     # Methods
@@ -144,25 +169,6 @@ def deobfuscate_list(filelist, usefulname):
     # to be sure, only keep really exsiting files:
     filelist = [f for f in filelist if os.path.isfile(f)]
 
-    # Search for par2 files in the filelist
-    par2_files = [f for f in filelist if f.endswith(".par2")]
-    # Found any par2 files we can use?
-    par2_renaming_done = False
-    if not par2_files:
-        logging.debug("No par2 files found to process, running renamer")
-    else:
-        # Run par2 from SABnzbd on them
-        for par2_file in par2_files:
-            # Analyse data and analyse result
-            logging.debug("Deobfuscate par2: handling %s", par2_file)
-            new_files = decode_par2(par2_file)
-            if new_files:
-                logging.debug("Deobfuscate par2 repair/verify finished")
-                filelist += new_files
-                filelist = [f for f in filelist if os.path.isfile(f)]
-            else:
-                logging.debug("Deobfuscate par2 repair/verify did not find anything to rename")
-
     # let's see if there are files with uncommon/unpopular (so: obfuscated) extensions
     # if so, let's give them a better extension based on their internal content/info
     # Example: if 'kjladsflkjadf.adsflkjads' is probably a PNG, rename to 'kjladsflkjadf.adsflkjads.png'

sabnzbd/interface.py

@@ -880,6 +880,7 @@ SPECIAL_BOOL_LIST = (
     "fast_fail",
     "overwrite_files",
     "enable_par_cleanup",
+    "process_unpacked_par2",
     "queue_complete_pers",
     "api_warnings",
     "helpfull_warnings",

sabnzbd/postproc.py

@@ -517,6 +517,10 @@ def process_job(nzo: NzbObject):
                         nzo.set_unpack_info("Unpack", T("Failed to move files"))
                         all_ok = False
 
+            # Use par2 files to deobfuscate unpacked file names
+            if cfg.process_unpacked_par2():
+                newfiles = deobfuscate.recover_par2_names(newfiles)
+
             if cfg.deobfuscate_final_filenames() and all_ok and not nzb_list:
                 # Deobfuscate the filenames
                 logging.info("Running deobfuscate")

tests/test_deobfuscate_filenames.py

@@ -316,7 +316,7 @@ class TestDeobfuscateFinalResult:
         for (dirpath, dirnames, filenames) in os.walk(test_dir):
             list_of_files += [os.path.join(dirpath, file) for file in filenames]
         # Run deobfuscate
-        deobfuscate_list(list_of_files, "doesnt_matter")
+        recover_par2_names(list_of_files)
 
         # Should now be renamed to the filename in the par2 file
         assert not os.path.exists(test_input)
@@ -344,9 +344,11 @@ class TestDeobfuscateFinalResult:
         # deobfuscate will do:
         # first par2 based renaming aaaaaaaaaaa to twentymb.bin,
         # then deobfuscate twentymb.bin to the job name (with same extension)
-        deobfuscate_list(list_of_files, "My Great Download")
+        list_of_files = recover_par2_names(list_of_files)
+        assert os.path.isfile(os.path.join(work_dir, "twentymb.bin"))  # should exist
 
+        deobfuscate_list(list_of_files, "My Great Download")
         assert os.path.isfile(os.path.join(work_dir, "My Great Download.bin"))  # the twentymb.bin should be renamed
-        assert not os.path.isfile(os.path.join(work_dir, "twentymb.bin"))  # should be gone
+        assert not os.path.isfile(os.path.join(work_dir, "twentymb.bin"))  # should now be gone
 
         shutil.rmtree(work_dir)