find and use obfuscated SFV files (#1459)

* isSFVfile(): Checks if given file is a SFV file, and returns result as boolean

* Obfuscated SFV files: find and use them

* Obfuscated SFV files: rename to is_sfv_file()

* is_sfv_file(): regexp based, minor stuff, pytest

* is_sfv_file(): pytest according to black

sabnzbd/newsunpack.py

@@ -28,7 +28,9 @@ import time
 import zlib
 import shutil
 import functools
+import re
 from subprocess import Popen
+from sabnzbd.encoding import ubtou
 
 import sabnzbd
 from sabnzbd.encoding import platform_btou, correct_unknown_encoding
@@ -2210,6 +2212,49 @@ def par2_mt_check(par2_path):
     return False
 
 
+def is_sfv_file(myfile):
+    """ Checks if given file is a SFV file, and returns result as boolean """
+
+    # based on https://stackoverflow.com/a/7392391/5235502
+    textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7F})
+    is_ascii_string = lambda bytes: not bool(bytes.translate(None, textchars))
+
+    # first check if it's plain text (ASCII or Unicode)
+    try:
+        with open(myfile, "rb") as f:
+            # get first 10000 bytes to check
+            myblock = f.read(10000)
+            if is_ascii_string(myblock):
+                # ASCII, so store lines for further inspection
+                try:
+                    lines = ubtou(myblock).split("\n")
+                except UnicodeDecodeError:
+                    return False
+            else:
+                # non-ASCII, so not SFV
+                return False
+    except:
+        # the with-open() went wrong, so not an existing file, so certainly not a SFV file
+        return False
+
+    sfv_info_line_counter = 0
+    for line in lines:
+        if re.search("^[^;].*\ +[A-Fa-f0-9]{8}$", line):
+            # valid, useful SFV line: some text, then one or more space, and a 8-digit hex number
+            sfv_info_line_counter += 1
+            if sfv_info_line_counter >= 10:
+                # with 10 valid, useful lines we're confident enough
+                # (note: if we find less lines (even just 1 line), with no negatives, it is OK. See below)
+                break
+        elif re.search("^;", line) or re.search("^\ *$", line):
+            # comment line or just spaces, so continue to next line
+            continue
+        else:
+            # not a valid SFV line, so not a SFV file:
+            return False
+    # if we get here, no negatives were found, and at least 1 valid line is OK
+    return sfv_info_line_counter >= 1
+
 def sfv_check(sfvs, nzo, workdir):
     """ Verify files using SFV files """
     # Update status

sabnzbd/postproc.py

@@ -34,6 +34,7 @@ from sabnzbd.newsunpack import (
     sfv_check,
     build_filelists,
     rar_sort,
+    is_sfv_file,
 )
 from threading import Thread
 from sabnzbd.misc import on_cleanup_list
@@ -799,9 +800,16 @@ def try_sfv_check(nzo, workdir):
     # Get list of SFV names
     sfvs = globber_full(workdir, "*.sfv")
 
-    # Skip if there's no SFV's
+    # If no files named *.sfv, let's search for obfuscated SFV files
     if not sfvs:
-        return None
+        files = globber_full(workdir, "*")
+        for file in files:
+            if sabnzbd.newsunpack.is_sfv_file(file):
+                logging.debug("Found and will use obfuscated SFV file: %s", file)
+                sfvs.append(file)
+        if not sfvs:
+            # still no SFV, so:
+            return None
 
     result = sfv_check(sfvs, nzo, workdir)
     if not result:

tests/data/good_sfv_unicode.sfv

@@ -0,0 +1,11 @@
+; Generated by cksfv v1.3.14 on 2020-05-25 at 17:49.33
+; Project web site: http://www.iki.fi/shd/foss/cksfv/
+;
+;     10485760  17:47.45 2020-05-25 normalascanbe.bin
+;     10485760  17:44.36 2020-05-25 schöne_Türen_öffnen
+;     10485760  17:44.05 2020-05-25 this is a file with spaces
+;     10485760  17:47.30 2020-05-25 你好世界
+normalascanbe.bin 681718CC
+schöne_Türen_öffnen 788E9541
+this is a file with spaces 24041228
+你好世界 78938E11

tests/data/one_line.sfv

@@ -0,0 +1 @@
+My Only File ABCD1234

tests/data/only_comments.sfv

@@ -0,0 +1,4 @@
+; this is a line
+; and this too
+; and more
+; and more ...

tests/test_is_sfv_file.py

@@ -0,0 +1,28 @@
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+"""
+tests.test_is_sfv_file- Testing SABnzbd is_sfv_file()
+"""
+
+from sabnzbd.newsunpack import is_sfv_file
+
+
+class Test_is_sfv_file:
+    """ Tests of is_sfv_file() against various input files
+    """
+
+    def test_valid_unicode_sfv(self):
+        assert is_sfv_file("tests/data/good_sfv_unicode.sfv")
+
+    def test_valid_one_line_sfv(self):
+        assert is_sfv_file("tests/data/one_line.sfv")
+
+    def test_only_comments(self):
+        assert not is_sfv_file("tests/data/only_comments.sfv")
+
+    def test_random_bin(self):
+        assert not is_sfv_file("tests/data/random.bin")