Browse Source
* correct_extension: basics, including unittest * correct_extension: basics, including unittest * correct_extension: puremagic into requirements.txt * correct_extension: introduce a main for testing from CLI * correct_extension: parse all parameters on CLI as files * correct_extension: parse all parameters on CLI as files * correct_extension: CLI parameter "-p" for privacy output * correct_extension: has_common_extension() and most_likely_extension() * correct_extension: has_common_extension() and most_likely_extension() * correct_extension: add extension if file has no commonly used extension * correct_extension: Black happy ... hopefully * correct_extension: Black happy ... hopefully * correct_extension: process feedback, mainly the extenions lists ^H^H^H^ tuples * correct_extension: process feedback, mainly the extenions lists ^H^H^H^ tuples * correct_extension: process feedback, mainly the extenions lists ^H^H^H^ tuples * correct_extension: process feedback, mainly the extenions lists ^H^H^H^ tuples * correct_extension: cleaned up * correct_extension: cleaned up ... github-black now happy? * correct_extension: cleaned up ... github-black now happy? * correct_extension: cleaned up ... github-black now happy? * correct_extension: cleaned up ... github-black now happy? * correct_extension: cleaned up ... github-black now happy? * correct_extension: easier if-then-logic, check if new_extension_to_add is filled. * correct_extension: if puremagic does recoging txt or nzb, check ourselves * correct_extension: if puremagic does recoging txt or nzb, check ourselves * correct_extension: only files! * correct_extension: only files! * correct_extension: rNN files not common extension, plus easier testing * correct_extension: clean-up ... no more boolean extension_too * correct_extension: requirements.txt, solved a TODO, and use get_ext() * correct_extension: a comment added * correct_extension: correct typing, correct txt and nzb extension * correct_extension: extensions always with dots, bug fix in what_is_most_likely_extension() * correct_extension: back on track? * correct_extension: back on track? * correct_extension: better commentspull/1920/head
Sander
4 years ago
committed by
GitHub
GitHub
12 changed files with 567 additions and 82 deletions
@ -0,0 +1,314 @@ |
|||
#!/usr/bin/python3 |
|||
|
|||
""" function to check and find correct extension of a (deobfuscated) file |
|||
Note: extension always contains a leading dot |
|||
""" |
|||
|
|||
|
|||
import puremagic |
|||
import os |
|||
import sys |
|||
from typing import List |
|||
from pathlib import Path |
|||
from sabnzbd.filesystem import get_ext |
|||
|
|||
# common extension from https://www.computerhope.com/issues/ch001789.htm |
|||
POPULAR_EXT = ( |
|||
"3g2", |
|||
"3gp", |
|||
"7z", |
|||
"ai", |
|||
"aif", |
|||
"apk", |
|||
"arj", |
|||
"asp", |
|||
"aspx", |
|||
"avi", |
|||
"bak", |
|||
"bat", |
|||
"bin", |
|||
"bmp", |
|||
"c", |
|||
"cab", |
|||
"cda", |
|||
"cer", |
|||
"cfg", |
|||
"cfm", |
|||
"cgi", |
|||
"cgi", |
|||
"cgi", |
|||
"class", |
|||
"com", |
|||
"cpl", |
|||
"cpp", |
|||
"cs", |
|||
"css", |
|||
"csv", |
|||
"cur", |
|||
"dat", |
|||
"db", |
|||
"dbf", |
|||
"deb", |
|||
"dll", |
|||
"dmg", |
|||
"dmp", |
|||
"doc", |
|||
"docx", |
|||
"drv", |
|||
"email", |
|||
"eml", |
|||
"emlx", |
|||
"exe", |
|||
"flv", |
|||
"fnt", |
|||
"fon", |
|||
"gadget", |
|||
"gif", |
|||
"h", |
|||
"h264", |
|||
"htm", |
|||
"html", |
|||
"icns", |
|||
"ico", |
|||
"ico", |
|||
"ini", |
|||
"iso", |
|||
"jar", |
|||
"java", |
|||
"jpeg", |
|||
"jpg", |
|||
"js", |
|||
"jsp", |
|||
"key", |
|||
"lnk", |
|||
"log", |
|||
"m4v", |
|||
"mdb", |
|||
"mid", |
|||
"midi", |
|||
"mkv", |
|||
"mov", |
|||
"mp3", |
|||
"mp4", |
|||
"mpa", |
|||
"mpeg", |
|||
"mpg", |
|||
"msg", |
|||
"msi", |
|||
"msi", |
|||
"odp", |
|||
"ods", |
|||
"odt", |
|||
"oft", |
|||
"ogg", |
|||
"ost", |
|||
"otf", |
|||
"part", |
|||
"pdf", |
|||
"php", |
|||
"php", |
|||
"pkg", |
|||
"pl", |
|||
"pl", |
|||
"pl", |
|||
"png", |
|||
"pps", |
|||
"ppt", |
|||
"pptx", |
|||
"ps", |
|||
"psd", |
|||
"pst", |
|||
"py", |
|||
"py", |
|||
"py", |
|||
"rar", |
|||
"rm", |
|||
"rpm", |
|||
"rss", |
|||
"rtf", |
|||
"sav", |
|||
"sh", |
|||
"sql", |
|||
"svg", |
|||
"swf", |
|||
"swift", |
|||
"sys", |
|||
"tar", |
|||
"tar", |
|||
"gz", |
|||
"tex", |
|||
"tif", |
|||
"tiff", |
|||
"tmp", |
|||
"toast", |
|||
"ttf", |
|||
"txt", |
|||
"vb", |
|||
"vcd", |
|||
"vcf", |
|||
"vob", |
|||
"wav", |
|||
"wma", |
|||
"wmv", |
|||
"wpd", |
|||
"wpl", |
|||
"wsf", |
|||
"xhtml", |
|||
"xls", |
|||
"xlsm", |
|||
"xlsx", |
|||
"z", |
|||
"zip", |
|||
) |
|||
|
|||
DOWNLOAD_EXT = ( |
|||
"ass", |
|||
"avi", |
|||
"bat", |
|||
"bdmv", |
|||
"bin", |
|||
"bup", |
|||
"clpi", |
|||
"crx", |
|||
"db", |
|||
"diz", |
|||
"djvu", |
|||
"docx", |
|||
"epub", |
|||
"exe", |
|||
"flac", |
|||
"gif", |
|||
"gz", |
|||
"htm", |
|||
"html", |
|||
"icns", |
|||
"ico", |
|||
"idx", |
|||
"ifo", |
|||
"img", |
|||
"inf", |
|||
"info", |
|||
"ini", |
|||
"iso", |
|||
"jpg", |
|||
"log", |
|||
"m2ts", |
|||
"m3u", |
|||
"m4a", |
|||
"mkv", |
|||
"mp3", |
|||
"mp4", |
|||
"mpls", |
|||
"mx", |
|||
"nfo", |
|||
"nib", |
|||
"nzb", |
|||
"otf", |
|||
"par2", |
|||
"part", |
|||
"pdf", |
|||
"pem", |
|||
"php", |
|||
"plist", |
|||
"png", |
|||
"py", |
|||
"rar", |
|||
"releaseinfo", |
|||
"rev", |
|||
"sfv", |
|||
"sh", |
|||
"srr", |
|||
"srs", |
|||
"srt", |
|||
"strings", |
|||
"sub", |
|||
"sup", |
|||
"sys", |
|||
"tif", |
|||
"ttf", |
|||
"txt", |
|||
"url", |
|||
"vob", |
|||
"website", |
|||
"wmv", |
|||
"xpi", |
|||
) |
|||
|
|||
# combine to one tuple, with unique entries: |
|||
ALL_EXT = tuple(set(POPULAR_EXT + DOWNLOAD_EXT)) |
|||
# prepend a dot to each extension, because we work with a leading dot in extensions |
|||
ALL_EXT = tuple(["." + i for i in ALL_EXT]) |
|||
|
|||
|
|||
def has_popular_extension(file_path: str) -> bool: |
|||
"""returns boolean if the extension of file_path is a popular, well-known extension""" |
|||
file_extension = get_ext(file_path) |
|||
return file_extension in ALL_EXT |
|||
|
|||
|
|||
def all_possible_extensions(file_path: str) -> List[str]: |
|||
"""returns a list with all possible extensions (with leading dot) for given file_path as reported by puremagic""" |
|||
extension_list = [] |
|||
for i in puremagic.magic_file(file_path): |
|||
extension_list.append(i.extension) |
|||
return extension_list |
|||
|
|||
|
|||
def what_is_most_likely_extension(file_path: str) -> str: |
|||
"""Returns most_likely extension, with a leading dot""" |
|||
for possible_extension in all_possible_extensions(file_path): |
|||
# let's see if technically-suggested extension by puremagic is also likely IRL |
|||
if possible_extension in ALL_EXT: |
|||
# Yes, looks likely |
|||
return possible_extension |
|||
|
|||
# Check if text or NZB, as puremagic is not good at that. |
|||
try: |
|||
txt = Path(file_path).read_text() |
|||
# Yes, a text file ... so let's check if it's even an NZB: |
|||
if txt.lower().find("<nzb xmlns=") >= 0 or txt.lower().find("!doctype nzb public") >= 0: |
|||
# yes, contains NZB signals: |
|||
return ".nzb" |
|||
else: |
|||
return ".txt" |
|||
except UnicodeDecodeError: |
|||
# not txt (and not nzb) |
|||
pass |
|||
|
|||
# no popular extension found, so just trust puremagic and return the first extension (if any) |
|||
try: |
|||
return all_possible_extensions(file_path)[0] |
|||
except IndexError: |
|||
return "" |
|||
|
|||
|
|||
if __name__ == "__main__": |
|||
privacy = False |
|||
|
|||
# parse all parameters on CLI as files to be ext-checked |
|||
for i in range(1, len(sys.argv)): |
|||
if sys.argv[i] == "-p": |
|||
# privacy, please ... so only print last 10 chars of a file |
|||
privacy = True |
|||
continue |
|||
|
|||
file_path = sys.argv[i] |
|||
|
|||
if privacy: |
|||
to_be_printed = file_path[-10:] |
|||
else: |
|||
to_be_printed = file_path |
|||
|
|||
if has_popular_extension(file_path): |
|||
# a common extension, so let's see what puremagic says, so that we can learn |
|||
filename, file_extension = os.path.splitext(file_path) |
|||
file_extension = file_extension[1:].lower() |
|||
|
|||
print( |
|||
"IRL-ext", |
|||
file_extension, |
|||
"most_likely", |
|||
what_is_most_likely_extension(file_path), |
|||
"puremagic", |
|||
all_possible_extensions(file_path), |
|||
) |
|||
|
After Width: | Height: | Size: 8.9 KiB |
Binary file not shown.
@ -0,0 +1,84 @@ |
|||
<?xml version="1.0" encoding="utf-8"?> |
|||
<!DOCTYPE nzb PUBLIC "-//newzBin//DTD NZB 1.1//EN" "http://www.newzbin.com/DTD/nzb/nzb-1.1.dtd"> |
|||
<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb"> |
|||
<file poster="blablamannetje <blabla@example.com>" date="1623601671" subject="mix no ext 38f38a34acc2 [02/10] - "inthemix.par2" yEnc (1/1) 39860"> |
|||
<groups> |
|||
<group>alt.binaries.test</group> |
|||
</groups> |
|||
<segments> |
|||
<segment bytes="41232" number="1">QoEbWuJpTnYmReOxUbFmBvLx-1623601671928@nyuu</segment> |
|||
</segments> |
|||
</file> |
|||
<file poster="blablamannetje <blabla@example.com>" date="1623601671" subject="mix no ext 38f38a34acc2 [03/10] - "inthemix.vol000+001.par2" yEnc (1/1) 40196"> |
|||
<groups> |
|||
<group>alt.binaries.test</group> |
|||
</groups> |
|||
<segments> |
|||
<segment bytes="41590" number="1">OfUzNpRoQlEkAkJwUoHxJlJj-1623601671929@nyuu</segment> |
|||
</segments> |
|||
</file> |
|||
<file poster="blablamannetje <blabla@example.com>" date="1623601671" subject="mix no ext 38f38a34acc2 [04/10] - "inthemix.vol001+002.par2" yEnc (1/1) 40532"> |
|||
<groups> |
|||
<group>alt.binaries.test</group> |
|||
</groups> |
|||
<segments> |
|||
<segment bytes="41938" number="1">TsNlKcDyMiCiNeHrMhFrQwPu-1623601671929@nyuu</segment> |
|||
</segments> |
|||
</file> |
|||
<file poster="blablamannetje <blabla@example.com>" date="1623601672" subject="mix no ext 38f38a34acc2 [06/10] - "inthemix.vol007+008.par2" yEnc (1/1) 122036"> |
|||
<groups> |
|||
<group>alt.binaries.test</group> |
|||
</groups> |
|||
<segments> |
|||
<segment bytes="125956" number="1">RvFtBzLeVzYhCiSjNkYqPkYv-1623601672004@nyuu</segment> |
|||
</segments> |
|||
</file> |
|||
<file poster="blablamannetje <blabla@example.com>" date="1623601672" subject="mix no ext 38f38a34acc2 [05/10] - "inthemix.vol003+004.par2" yEnc (1/1) 80948"> |
|||
<groups> |
|||
<group>alt.binaries.test</group> |
|||
</groups> |
|||
<segments> |
|||
<segment bytes="83601" number="1">CyBcLhFsErVvWhKaJbKySsLh-1623601672003@nyuu</segment> |
|||
</segments> |
|||
</file> |
|||
<file poster="blablamannetje <blabla@example.com>" date="1623601671" subject="mix no ext 38f38a34acc2 [01/10] - "inthemix.rar" yEnc (1/1) 528471"> |
|||
<groups> |
|||
<group>alt.binaries.test</group> |
|||
</groups> |
|||
<segments> |
|||
<segment bytes="544152" number="1">ZtFjLqEiBmQgZyHyRjIvLmDq-1623601671925@nyuu</segment> |
|||
</segments> |
|||
</file> |
|||
<file poster="blablamannetje <blabla@example.com>" date="1623601672" subject="mix no ext 38f38a34acc2 [07/10] - "inthemix.vol015+016.par2" yEnc (1/1) 164468"> |
|||
<groups> |
|||
<group>alt.binaries.test</group> |
|||
</groups> |
|||
<segments> |
|||
<segment bytes="169681" number="1">ZbBmMqCmJyRgOjAiSgMmFhUs-1623601672012@nyuu</segment> |
|||
</segments> |
|||
</file> |
|||
<file poster="blablamannetje <blabla@example.com>" date="1623601672" subject="mix no ext 38f38a34acc2 [08/10] - "inthemix.vol031+032.par2" yEnc (1/1) 209588"> |
|||
<groups> |
|||
<group>alt.binaries.test</group> |
|||
</groups> |
|||
<segments> |
|||
<segment bytes="216211" number="1">OmEhDrElGwEkYrHsTcFlYeYp-1623601672019@nyuu</segment> |
|||
</segments> |
|||
</file> |
|||
<file poster="blablamannetje <blabla@example.com>" date="1623601672" subject="mix no ext 38f38a34acc2 [09/10] - "inthemix.vol063+064.par2" yEnc (1/1) 260084"> |
|||
<groups> |
|||
<group>alt.binaries.test</group> |
|||
</groups> |
|||
<segments> |
|||
<segment bytes="268274" number="1">SkUsGaAkBjNpHoCsLtLiBcYn-1623601672044@nyuu</segment> |
|||
</segments> |
|||
</file> |
|||
<file poster="blablamannetje <blabla@example.com>" date="1623601672" subject="mix no ext 38f38a34acc2 [10/10] - "inthemix.vol127+071.par2" yEnc (1/1) 262436"> |
|||
<groups> |
|||
<group>alt.binaries.test</group> |
|||
</groups> |
|||
<segments> |
|||
<segment bytes="270671" number="1">PfYdNqVpPpLvOqTvYrXoRbQi-1623601672045@nyuu</segment> |
|||
</segments> |
|||
</file> |
|||
</nzb> |
|||
Binary file not shown.
@ -0,0 +1,4 @@ |
|||
Yes, this is a text file. |
|||
|
|||
The END |
|||
|
|||
@ -0,0 +1,54 @@ |
|||
#!/usr/bin/python3 -OO |
|||
# Copyright 2007-2021 The SABnzbd-Team <team@sabnzbd.org> |
|||
# |
|||
# This program is free software; you can redistribute it and/or |
|||
# modify it under the terms of the GNU General Public License |
|||
# as published by the Free Software Foundation; either version 2 |
|||
# of the License, or (at your option) any later version. |
|||
# |
|||
# This program is distributed in the hope that it will be useful, |
|||
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|||
# GNU General Public License for more details. |
|||
# |
|||
# You should have received a copy of the GNU General Public License |
|||
# along with this program; if not, write to the Free Software |
|||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
|||
|
|||
""" |
|||
Testing SABnzbd correct extension functionality module |
|||
""" |
|||
|
|||
import os |
|||
from tests.testhelper import * |
|||
import sabnzbd.utils.file_extension as file_extension |
|||
|
|||
|
|||
class Test_File_Extension: |
|||
def test_has_popular_extension(self): |
|||
assert file_extension.has_popular_extension("blabla/blabla.mkv") |
|||
assert file_extension.has_popular_extension("blabla/blabla.srt") |
|||
assert file_extension.has_popular_extension("djjddj/aaaaa.epub") |
|||
assert not file_extension.has_popular_extension("98ads098f098fa.a0ds98f098asdf") |
|||
|
|||
def test_what_is_most_likely_extension(self): |
|||
# These are real-content files, where the contents determine the extension |
|||
filename = "tests/data/test_file_extension/apeeengeee" # A PNG |
|||
assert os.path.isfile(filename) |
|||
assert file_extension.what_is_most_likely_extension(filename) == ".png" |
|||
|
|||
filename = "tests/data/test_file_extension/somepeedeef" # Some PDF |
|||
assert os.path.isfile(filename) |
|||
assert file_extension.what_is_most_likely_extension(filename) == ".pdf" |
|||
|
|||
filename = "tests/data/test_file_extension/my_matroska" # my Matroska MKV |
|||
assert os.path.isfile(filename) |
|||
assert file_extension.what_is_most_likely_extension(filename) == ".mkv" |
|||
|
|||
filename = "tests/data/test_file_extension/sometxtfile" # a txt file |
|||
assert os.path.isfile(filename) |
|||
assert file_extension.what_is_most_likely_extension(filename) == ".txt" |
|||
|
|||
filename = "tests/data/test_file_extension/some_nzb_file" # a NZB file |
|||
assert os.path.isfile(filename) |
|||
assert file_extension.what_is_most_likely_extension(filename) == ".nzb" |
|||
Loading…
Reference in new issue