Browse Source

Merge branch '3.2.x'

tags/3.2.1
Safihre 4 years ago
parent
commit
2abe4c3cef
  1. 3
      .gitignore
  2. 4
      PKG-INFO
  3. 20
      README.mkd
  4. 28
      SABnzbd.py
  5. 8
      interfaces/Config/templates/config_rss.tmpl
  6. 3
      interfaces/Config/templates/config_server.tmpl
  7. 4
      interfaces/Config/templates/config_switches.tmpl
  8. 11
      interfaces/Config/templates/staticcfg/bootstrap/css/bootstrap.min.css
  9. 27
      interfaces/Config/templates/staticcfg/css/style.css
  10. 1
      interfaces/Glitter/templates/main.tmpl
  11. 2
      interfaces/Glitter/templates/static/javascripts/glitter.history.js
  12. 2
      interfaces/Glitter/templates/static/javascripts/glitter.queue.js
  13. 10
      interfaces/Glitter/templates/static/stylesheets/colorschemes/Night.css
  14. 21
      sabnzbd/assembler.py
  15. 7
      sabnzbd/bpsmeter.py
  16. 2
      sabnzbd/config.py
  17. 7
      sabnzbd/decoder.py
  18. 48
      sabnzbd/deobfuscate_filenames.py
  19. 18
      sabnzbd/filesystem.py
  20. 9
      sabnzbd/misc.py
  21. 10
      sabnzbd/newsunpack.py
  22. 2
      sabnzbd/sorting.py
  23. 5
      sabnzbd/utils/diskspeed.py
  24. 106
      tests/test_deobfuscate_filenames.py
  25. 73
      tests/test_filesystem.py

3
.gitignore

@ -31,6 +31,9 @@ SABnzbd-*/
*.wp[ru]
.idea
# VScode
.vscode/
# Testing folders
.cache
.xprocess

4
PKG-INFO

@ -1,7 +1,7 @@
Metadata-Version: 1.0
Name: SABnzbd
Version: 3.2.0
Summary: SABnzbd-3.2.0
Version: 3.2.1
Summary: SABnzbd-3.2.1
Home-page: https://sabnzbd.org
Author: The SABnzbd Team
Author-email: team@sabnzbd.org

20
README.mkd

@ -1,6 +1,24 @@
Release Notes - SABnzbd 3.2.0
Release Notes - SABnzbd 3.2.1
=========================================================
## Changes and bugfixes since 3.2.0
- Single `Indexer Categories` in Categories were broken.
- Program would fail to start if Quota was previously exceeded.
- Setting `Automatically sort queue` by `Age` was inverted.
- Show the name of the item to be deleted from the Queue/History
in the confirmation dialog.
- Handle directories in `.par2`-files during Quick-check.
- Do not discard data for articles with CRC-errors.
- Improvements to `Deobfuscate final filenames`:
Rename accompanying (smaller) files with the same basename.
Do not rename collections of the same extension.
- Sanitize names possibly derived from `X-DNZB-EpisodeName`.
- Widened the RSS feeds table.
- Show server expiration date in server summary.
- Improvements to the encrypted RAR-detection.
- Add traceback-logging when failing to read the password file.
- Windows: Use binary mode to make the write test more accurate.
## Changes since 3.1.1
- Python 3.6 is the minimum required version.
- The Windows installer can only be used on 64bit Windows 8.1 and

28
SABnzbd.py

@ -19,7 +19,7 @@ import sys
if sys.hexversion < 0x03060000:
print("Sorry, requires Python 3.6 or above")
print("You can read more at: https://sabnzbd.org/python3")
print("You can read more at: https://sabnzbd.org/wiki/installation/install-off-modules")
sys.exit(1)
import logging
@ -48,7 +48,7 @@ try:
except ImportError as e:
print("Not all required Python modules are available, please check requirements.txt")
print("Missing module:", e.name)
print("You can read more at: https://sabnzbd.org/python3")
print("You can read more at: https://sabnzbd.org/wiki/installation/install-off-modules")
print("If you still experience problems, remove all .pyc files in this folder and subfolders")
sys.exit(1)
@ -68,7 +68,6 @@ from sabnzbd.misc import (
get_serv_parms,
get_from_url,
upload_file_to_sabnzbd,
is_ipv4_addr,
is_localhost,
is_lan_addr,
)
@ -137,16 +136,29 @@ class GUIHandler(logging.Handler):
except TypeError:
parsed_msg = record.msg + str(record.args)
if record.levelno == logging.WARNING:
sabnzbd.notifier.send_notification(T("Warning"), parsed_msg, "warning")
else:
sabnzbd.notifier.send_notification(T("Error"), parsed_msg, "error")
warning = {
"type": record.levelname,
"text": parsed_msg,
"time": int(time.time()),
"origin": "%s%d" % (record.filename, record.lineno),
}
# Append traceback, if available
warning = {"type": record.levelname, "text": parsed_msg, "time": int(time.time())}
if record.exc_info:
warning["text"] = "%s\n%s" % (warning["text"], traceback.format_exc())
# Do not notify the same notification within 1 minute from the same source
# This prevents endless looping if the notification service itself throws an error/warning
# We don't check based on message content, because if it includes a timestamp it's not unique
if not any(
stored_warning["origin"] == warning["origin"] and stored_warning["time"] + DEF_TIMEOUT > time.time()
for stored_warning in self.store
):
if record.levelno == logging.WARNING:
sabnzbd.notifier.send_notification(T("Warning"), parsed_msg, "warning")
else:
sabnzbd.notifier.send_notification(T("Error"), parsed_msg, "error")
# Loose the oldest record
if len(self.store) >= self._size:
self.store.pop(0)

8
interfaces/Config/templates/config_rss.tmpl

@ -10,7 +10,7 @@
<p>$T('explain-RSS')</p>
<form action="add_rss_feed" method="post" autocomplete="off">
<input type="hidden" name="apikey" value="$apikey" />
<table class="catTable">
<table class="catTable addRssTable">
<tr>
<th>&nbsp;</th>
<th>$T('name')</th>
@ -21,10 +21,10 @@
<td>
<input type="checkbox" name="enable" value="1" checked />
</td>
<td>
<input type="text" name="feed" class="smaller_input" value="$feed" />
<td class="new-feed-title">
<input type="text" name="feed" value="$feed" />
</td>
<td>
<td class="new-feed-url">
<input type="text" name="uri" placeholder="$T('addMultipleFeeds')" />
</td>
<td class="nowrap">

3
interfaces/Config/templates/config_server.tmpl

@ -273,6 +273,9 @@
<b>$T('srv-article-availability'):</b><br/>
$T('selectedDates'): <span id="server-article-value-${cur}"></span>
</p>
<!--#if $server['expire_date']#-->
<p><b>$T('srv-expire_date'):</b> $(server['expire_date'])</p>
<!--#end if#-->
<!--#if $server['quota']#-->
<p><b>$T('quota-left'):</b> $(server['quota_left'])B</p>
<!--#end if#-->

4
interfaces/Config/templates/config_switches.tmpl

@ -136,8 +136,8 @@
<label class="config" for="auto_sort">$T('opt-auto_sort')</label>
<select name="auto_sort" id="auto_sort">
<option value="">$T('default')</option>
<option value="avg_age asc" <!--#if $auto_sort == "avg_age asc" then 'selected="selected"' else ""#--> >$T('Glitter-sortAgeAsc')</option>
<option value="avg_age desc" <!--#if $auto_sort == "avg_age desc" then 'selected="selected"' else ""#--> >$T('Glitter-sortAgeDesc')</option>
<option value="avg_age desc" <!--#if $auto_sort == "avg_age desc" then 'selected="selected"' else ""#--> >$T('Glitter-sortAgeAsc')</option>
<option value="avg_age asc" <!--#if $auto_sort == "avg_age asc" then 'selected="selected"' else ""#--> >$T('Glitter-sortAgeDesc')</option>
<option value="name asc" <!--#if $auto_sort == "name asc" then 'selected="selected"' else ""#--> >$T('Glitter-sortNameAsc')</option>
<option value="name desc" <!--#if $auto_sort == "name desc" then 'selected="selected"' else ""#--> >$T('Glitter-sortNameDesc')</option>
<option value="size asc" <!--#if $auto_sort == "size asc" then 'selected="selected"' else ""#--> >$T('Glitter-sortSizeAsc')</option>

11
interfaces/Config/templates/staticcfg/bootstrap/css/bootstrap.min.css

File diff suppressed because one or more lines are too long

27
interfaces/Config/templates/staticcfg/css/style.css

@ -551,6 +551,16 @@ tr.separator {
padding-right: 13px;
}
/* -- */
.RSS .addRssTable,
.RSS .addRssTable input[type="text"] {
width: 100%;
}
.RSS .addRssTable .new-feed-title {
max-width: 250px;
}
.RSS .addRssTable .new-feed-url {
width: 70%;
}
h2.activeRSS {
margin-bottom: 10px;
}
@ -560,12 +570,12 @@ h2.activeRSS {
text-decoration: underline !important;
}
.favicon {
background-position: center center!important;
background-size: 16px 16px;
background-position: center center !important;
background-size: 22px 22px;
opacity: 1;
top: -1px;
height: 16px;
width: 16px;
height: 22px;
width: 22px;
float: left;
margin: 0 6px 0 2px;
text-align: center;
@ -585,6 +595,7 @@ h2.activeRSS {
}
#subscriptions {
border: 1px solid #E5E5E5;
width: 100%;
}
.data-row {
border-top: 1px solid #E5E5E5;
@ -596,6 +607,7 @@ h2.activeRSS {
#subscriptions .chk {
padding: 8px 5px 5px;
vertical-align: middle;
width: 40px;
}
#subscriptions .title {
font-weight: bold;
@ -603,10 +615,11 @@ h2.activeRSS {
width: auto;
}
#subscriptions .favicon {
margin-left: 8px;
margin-left: 7px;
margin-top: -2px;
}
.ie6 .subscription-title {
width: 20em;
#subscriptions .glyphicon {
margin-top: 3px;
}
.subscription-title,
.subscription-title:hover {

1
interfaces/Glitter/templates/main.tmpl

@ -59,6 +59,7 @@
glitterTranslate.shutdown = "$T('shutdownOK?')";
glitterTranslate.restart = "$T('explain-Restart') $T('explain-needNewLogin')".replace(/\<br(\s*\/|)\>/g, '\n');
glitterTranslate.repair = "$T('explain-Repair')".replace(/<br \/>/g, "\n").replace(/&quot;/g,'"');
glitterTranslate.deleteMsg = "$T('nzo-delete')";
glitterTranslate.removeDown = "$T('Glitter-confirmClearDownloads')";
glitterTranslate.removeDow1 = "$T('Glitter-confirmClear1Download')";
glitterTranslate.retryAll = "$T('link-retryAll')?";

2
interfaces/Glitter/templates/static/javascripts/glitter.history.js

@ -421,7 +421,7 @@ function HistoryModel(parent, data) {
// Delete button
self.deleteSlot = function(item, event) {
// Confirm?
if(!self.parent.parent.confirmDeleteHistory() || confirm(glitterTranslate.removeDow1)) {
if(!self.parent.parent.confirmDeleteHistory() || confirm(glitterTranslate.deleteMsg + ":\n" + item.historyStatus.name() + "\n\n" + glitterTranslate.removeDow1)) {
// Are we still processing and it can be stopped?
if(item.processingDownload() == 2) {
callAPI({

2
interfaces/Glitter/templates/static/javascripts/glitter.queue.js

@ -724,7 +724,7 @@ function QueueModel(parent, data) {
// Remove 1 download from queue
self.removeDownload = function(item, event) {
// Confirm and remove
if(!self.parent.parent.confirmDeleteQueue() || confirm(glitterTranslate.removeDow1)) {
if(!self.parent.parent.confirmDeleteQueue() || confirm(glitterTranslate.deleteMsg + ":\n" + item.name() + "\n\n" + glitterTranslate.removeDow1)) {
var itemToDelete = this;
// Show notification

10
interfaces/Glitter/templates/static/stylesheets/colorschemes/Night.css

@ -55,6 +55,10 @@ legend,
opacity: 0.7;
}
.form-control[disabled] {
opacity: 0.65;
}
.progress {
background-color: #DADADA;
}
@ -126,6 +130,10 @@ select.form-control,
.main-content .btn-default,
.modal-body .btn-default,
.modal-footer .btn-default,
.btn-default.disabled:hover,
.btn-default.disabled:active,
.btn-default.disabled:focus,
.form-control[disabled],
#modal-options .options-function-box .input-group-addon {
background-color: #555555;
color: #EBEBEB;
@ -157,6 +165,8 @@ tbody>tr:last-child td,
input,
input.form-control,
.input-group-addon,
.search-box input:focus,
.search-box input:valid,
select.form-control,
#modal-options .table-server-connections th,
.main-content .btn-default,

21
sabnzbd/assembler.py

@ -334,29 +334,30 @@ def check_encrypted_and_unwanted_files(nzo: NzbObject, filepath: str) -> Tuple[b
zf.setpassword(password)
except rarfile.Error:
# On weird passwords the setpassword() will fail
# but the actual rartest() will work
# but the actual testrar() will work
pass
try:
zf.testrar()
password_hit = password
break
except rarfile.RarWrongPassword:
# This one really didn't work
pass
except rarfile.RarCRCError as e:
# CRC errors can be thrown for wrong password or
# missing the next volume (with correct password)
if "cannot find volume" in str(e).lower():
# We assume this one worked!
password_hit = password
break
# This one didn't work
pass
except Exception as e:
# Did we start from the right volume? Skip the checks for now.
if match_str(
str(e).lower(),
("need to start extraction from a previous volume", "non-fatal error"),
):
return encrypted, unwanted
# This one failed
pass
except:
# All the other errors we skip, they might be fixable in post-proc.
# For example starting from the wrong volume, or damaged files
# This will cause the check to be performed again for the next rar, might
# be disk-intensive! Could be removed later and just accept the password.
return encrypted, unwanted
# Did any work?
if password_hit:

7
sabnzbd/bpsmeter.py

@ -195,13 +195,6 @@ class BPSMeter:
res = self.reset_quota()
except:
self.defaults()
# Force update of counters and validate data
try:
for server in self.grand_total.keys():
self.update(server)
except TypeError:
self.defaults()
self.update()
return res
def update(self, server: Optional[str] = None, amount: int = 0):

2
sabnzbd/config.py

@ -1137,7 +1137,7 @@ def validate_single_tag(value):
"""
if len(value) == 3:
if value[1] == ">":
return None, " ".join(value)
return None, [" ".join(value)]
return None, value

7
sabnzbd/decoder.py

@ -46,7 +46,7 @@ except ImportError:
class CrcError(Exception):
def __init__(self, needcrc, gotcrc, data):
def __init__(self, needcrc: int, gotcrc: int, data: bytes):
super().__init__()
self.needcrc = needcrc
self.gotcrc = gotcrc
@ -154,13 +154,16 @@ class DecoderWorker(Thread):
sabnzbd.NzbQueue.reset_try_lists(article)
continue
except CrcError:
except CrcError as crc_error:
logging.info("CRC Error in %s" % art_id)
# Continue to the next one if we found new server
if search_new_server(article):
continue
# Store data, maybe par2 can still fix it
decoded_data = crc_error.data
except (BadYenc, ValueError):
# Handles precheck and badly formed articles
if nzo.precheck and raw_data and raw_data[0].startswith(b"223 "):

48
sabnzbd/deobfuscate_filenames.py

@ -127,14 +127,13 @@ def is_probably_obfuscated(myinputfilename):
def deobfuscate_list(filelist, usefulname):
""" Check all files in filelist, and if wanted, deobfuscate """
""" Check all files in filelist, and if wanted, deobfuscate: rename to filename based on usefulname"""
# to be sure, only keep really exsiting files:
filelist = [f for f in filelist if os.path.exists(f)]
# Search for par2 files in the filelist
par2_files = [f for f in filelist if f.endswith(".par2")]
# Found any par2 files we can use?
run_renamer = True
if not par2_files:
@ -152,22 +151,57 @@ def deobfuscate_list(filelist, usefulname):
# No par2 files? Then we try to rename qualifying (big, not-excluded, obfuscated) files to the job-name
if run_renamer:
excluded_file_exts = EXCLUDED_FILE_EXTS
# If there is a collection with bigger files with the same extension, we don't want to rename it
extcounter = {}
for file in filelist:
if os.path.getsize(file) < MIN_FILE_SIZE:
# too small to care
continue
_, ext = os.path.splitext(file)
if ext in extcounter:
extcounter[ext] += 1
else:
extcounter[ext] = 1
if extcounter[ext] >= 3 and ext not in excluded_file_exts:
# collection, and extension not yet in excluded_file_exts, so add it
excluded_file_exts = (*excluded_file_exts, ext)
logging.debug(
"Found a collection of at least %s files with extension %s, so not renaming those files",
extcounter[ext],
ext,
)
logging.debug("Trying to see if there are qualifying files to be deobfuscated")
# We start with he biggest file ... probably the most important file
filelist = sorted(filelist, key=os.path.getsize, reverse=True)
for filename in filelist:
# check that file is still there (and not renamed by the secondary renaming process below)
if not os.path.isfile(filename):
continue
logging.debug("Deobfuscate inspecting %s", filename)
file_size = os.path.getsize(filename)
# Do we need to rename this file?
# Criteria: big, not-excluded extension, obfuscated (in that order)
if (
file_size > MIN_FILE_SIZE
and get_ext(filename) not in EXCLUDED_FILE_EXTS
os.path.getsize(filename) > MIN_FILE_SIZE
and get_ext(filename) not in excluded_file_exts
and is_probably_obfuscated(filename) # this as last test to avoid unnecessary analysis
):
# OK, rename
# Rename and make sure the new filename is unique
path, file = os.path.split(filename)
# construct new_name: <path><usefulname><extension>
new_name = get_unique_filename("%s%s" % (os.path.join(path, usefulname), get_ext(filename)))
logging.info("Deobfuscate renaming %s to %s", filename, new_name)
# Rename and make sure the new filename is unique
renamer(filename, new_name)
# find other files with the same basename in filelist, and rename them in the same way:
basedirfile, _ = os.path.splitext(filename) # something like "/home/this/myiso"
for otherfile in filelist:
if otherfile.startswith(basedirfile + ".") and os.path.isfile(otherfile):
# yes, same basedirfile, only different extension
remainingextension = otherfile.replace(basedirfile, "") # might be long ext, like ".dut.srt"
new_name = get_unique_filename("%s%s" % (os.path.join(path, usefulname), remainingextension))
logging.info("Deobfuscate renaming %s to %s", otherfile, new_name)
# Rename and make sure the new filename is unique
renamer(otherfile, new_name)
else:
logging.info("No qualifying files found to deobfuscate")

18
sabnzbd/filesystem.py

@ -806,8 +806,9 @@ def get_filepath(path: str, nzo, filename: str):
@synchronized(DIR_LOCK)
def renamer(old: str, new: str):
""" Rename file/folder with retries for Win32 """
def renamer(old: str, new: str, create_local_directories: bool = False):
"""Rename file/folder with retries for Win32
Optionally alows the creation of local directories if they don't exist yet"""
# Sanitize last part of new name
path, name = os.path.split(new)
new = os.path.join(path, sanitize_filename(name))
@ -816,6 +817,19 @@ def renamer(old: str, new: str):
if old == new:
return
# In case we want nonexistent directories to be created, check for directory escape (forbidden)
if create_local_directories:
oldpath, _ = os.path.split(old)
# Check not outside directory
# In case of "same_file() == 1": same directory, so nothing to do
if same_file(oldpath, path) == 0:
# Outside current directory, this is most likely malicious
logging.error(T("Blocked attempt to create directory %s"), path)
raise OSError("Refusing to go outside directory")
elif same_file(oldpath, path) == 2:
# Sub-directory, so create if does not yet exist:
create_all_dirs(path)
logging.debug('Renaming "%s" to "%s"', old, new)
if sabnzbd.WIN32:
retries = 10

9
sabnzbd/misc.py

@ -781,11 +781,7 @@ def get_all_passwords(nzo):
meta_passwords.append(pw)
if meta_passwords:
if nzo.password == meta_passwords[0]:
# this nzo.password came from meta, so don't use it twice
passwords.extend(meta_passwords[1:])
else:
passwords.extend(meta_passwords)
passwords.extend(meta_passwords)
logging.info("Read %s passwords from meta data in NZB: %s", len(meta_passwords), meta_passwords)
pw_file = cfg.password_file.get_path()
@ -808,6 +804,7 @@ def get_all_passwords(nzo):
)
except:
logging.warning(T("Failed to read the password file %s"), pw_file)
logging.info("Traceback: ", exc_info=True)
if nzo.password:
# If an explicit password was set, add a retry without password, just in case.
@ -816,7 +813,7 @@ def get_all_passwords(nzo):
# If we're not sure about encryption, start with empty password
# and make sure we have at least the empty password
passwords.insert(0, "")
return passwords
return set(passwords)
def find_on_path(targets):

10
sabnzbd/newsunpack.py

@ -55,6 +55,7 @@ from sabnzbd.filesystem import (
setname_from_path,
get_ext,
get_filename,
same_file,
)
from sabnzbd.nzbstuff import NzbObject, NzbFile
from sabnzbd.sorting import SeriesSorter
@ -2077,7 +2078,14 @@ def quick_check_set(set, nzo):
if nzf.md5sum == md5pack[file]:
try:
logging.debug("Quick-check will rename %s to %s", nzf.filename, file)
renamer(os.path.join(nzo.download_path, nzf.filename), os.path.join(nzo.download_path, file))
# Note: file can and is allowed to be in a subdirectory.
# Subdirectories in par2 always contain "/", not "\"
renamer(
os.path.join(nzo.download_path, nzf.filename),
os.path.join(nzo.download_path, file),
create_local_directories=True,
)
renames[file] = nzf.filename
nzf.filename = file
result &= True

2
sabnzbd/sorting.py

@ -35,6 +35,7 @@ from sabnzbd.filesystem import (
get_unique_filename,
get_ext,
renamer,
sanitize_and_trim_path,
sanitize_foldername,
clip_path,
)
@ -492,6 +493,7 @@ class SeriesSorter:
newpath = os.path.join(current_path, newname)
# Replace %ext with extension
newpath = newpath.replace("%ext", self.ext)
newpath = sanitize_and_trim_path(newpath)
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)

5
sabnzbd/utils/diskspeed.py

@ -22,7 +22,10 @@ def diskspeedmeasure(my_dirname: str) -> float:
try:
# Use low-level I/O
fp_testfile = os.open(filename, os.O_CREAT | os.O_WRONLY, 0o777)
try:
fp_testfile = os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_BINARY, 0o777)
except AttributeError:
fp_testfile = os.open(filename, os.O_CREAT | os.O_WRONLY, 0o777)
# Start looping
total_time = 0.0

106
tests/test_deobfuscate_filenames.py

@ -32,6 +32,11 @@ def create_big_file(filename):
myfile.truncate(15 * 1024 * 1024)
def create_small_file(filename):
with open(filename, "wb") as myfile:
myfile.truncate(1024)
class TestDeobfuscateFinalResult:
def test_is_probably_obfuscated(self):
# Test the base function test_is_probably_obfuscated(), which gives a boolean as RC
@ -178,6 +183,107 @@ class TestDeobfuscateFinalResult:
# Done. Remove (non-empty) directory
shutil.rmtree(dirname)
def test_deobfuscate_big_file_small_accompanying_files(self):
# input: myiso.iso, with accompanying files (.srt files in this case)
# test that the small accompanying files (with same basename) are renamed accordingly to the big ISO
# Create directory (with a random directory name)
dirname = os.path.join(SAB_DATA_DIR, "testdir" + str(random.randint(10000, 99999)))
os.mkdir(dirname)
# Create a big enough file with a non-useful filename
isofile = os.path.join(dirname, "myiso.iso")
create_big_file(isofile)
assert os.path.isfile(isofile)
# and a srt file
srtfile = os.path.join(dirname, "myiso.srt")
create_small_file(srtfile)
assert os.path.isfile(srtfile)
# and a dut.srt file
dutsrtfile = os.path.join(dirname, "myiso.dut.srt")
create_small_file(dutsrtfile)
assert os.path.isfile(dutsrtfile)
# and a non-related file
txtfile = os.path.join(dirname, "something.txt")
create_small_file(txtfile)
assert os.path.isfile(txtfile)
# create the filelist, with just the above files
myfilelist = [isofile, srtfile, dutsrtfile, txtfile]
# and now unleash the magic on that filelist, with a more useful jobname:
jobname = "My Important Download 2020"
deobfuscate_list(myfilelist, jobname)
# Check original files:
assert not os.path.isfile(isofile) # original iso not be there anymore
assert not os.path.isfile(srtfile) # ... and accompanying file neither
assert not os.path.isfile(dutsrtfile) # ... and this one neither
assert os.path.isfile(txtfile) # should still be there: not accompanying, and too small to rename
# Check the renaming
assert os.path.isfile(os.path.join(dirname, jobname + ".iso")) # ... should be renamed to the jobname
assert os.path.isfile(os.path.join(dirname, jobname + ".srt")) # ... should be renamed to the jobname
assert os.path.isfile(os.path.join(dirname, jobname + ".dut.srt")) # ... should be renamed to the jobname
# Done. Remove (non-empty) directory
shutil.rmtree(dirname)
def test_deobfuscate_collection_with_same_extension(self):
# input: a collection of bigger files with the same extension
# test that there is no renaming on the collection ... as that's useless on a collection
# Create directory (with a random directory name)
dirname = os.path.join(SAB_DATA_DIR, "testdir" + str(random.randint(10000, 99999)))
os.mkdir(dirname)
# Create big enough files with a non-useful filenames, all with same extension
file1 = os.path.join(dirname, "file1.bla")
create_big_file(file1)
assert os.path.isfile(file1)
file2 = os.path.join(dirname, "file2.bla")
create_big_file(file2)
assert os.path.isfile(file2)
file3 = os.path.join(dirname, "file3.bla")
create_big_file(file3)
assert os.path.isfile(file3)
file4 = os.path.join(dirname, "file4.bla")
create_big_file(file4)
assert os.path.isfile(file4)
# other extension ... so this one should get renamed
otherfile = os.path.join(dirname, "other.bin")
create_big_file(otherfile)
assert os.path.isfile(otherfile)
# create the filelist, with the above files
myfilelist = [file1, file2, file3, file4, otherfile]
# and now unleash the magic on that filelist, with a more useful jobname:
jobname = "My Important Download 2020"
deobfuscate_list(myfilelist, jobname)
# Check original files:
# the collection with same extension should still be there:
assert os.path.isfile(file1) # still there
assert os.path.isfile(file2) # still there
assert os.path.isfile(file3) # still there
assert os.path.isfile(file4) # still there
# but the one separate file with obfuscated name should be renamed:
assert not os.path.isfile(otherfile) # should be renamed
# Check the renaming
assert os.path.isfile(os.path.join(dirname, jobname + ".bin")) # ... should be renamed to the jobname
# Done. Remove (non-empty) directory
shutil.rmtree(dirname)
def test_deobfuscate_filelist_nasty_tests(self):
# check no problems occur with nasty use cases

73
tests/test_filesystem.py

@ -21,6 +21,9 @@ tests.test_filesystem - Testing functions in filesystem.py
import stat
import sys
import os
import random
import shutil
from pathlib import Path
import pyfakefs.fake_filesystem_unittest as ffs
@ -983,3 +986,73 @@ class TestSetPermissions(ffs.TestCase, PermissionCheckerHelper):
def test_dir1755_umask4755_setting(self):
# Sticky bit on directory, umask with setuid
self._runner("1755", "4755")
class TestRenamer:
# test filesystem.renamer() for different scenario's
def test_renamer(self):
# First of all, create a working directory (with a random name)
dirname = os.path.join(SAB_DATA_DIR, "testdir" + str(random.randint(10000, 99999)))
os.mkdir(dirname)
# base case: rename file within directory
filename = os.path.join(dirname, "myfile.txt")
Path(filename).touch() # create file
newfilename = os.path.join(dirname, "newfile.txt")
filesystem.renamer(filename, newfilename) # rename() does not return a value ...
assert not os.path.isfile(filename)
assert os.path.isfile(newfilename)
# standard behaviour: renaming (moving) into an exiting other directory *is* allowed
filename = os.path.join(dirname, "myfile.txt")
Path(filename).touch() # create file
sameleveldirname = os.path.join(SAB_DATA_DIR, "othertestdir" + str(random.randint(10000, 99999)))
os.mkdir(sameleveldirname)
newfilename = os.path.join(sameleveldirname, "newfile.txt")
filesystem.renamer(filename, newfilename)
assert not os.path.isfile(filename)
assert os.path.isfile(newfilename)
shutil.rmtree(sameleveldirname)
# Default: renaming into a non-existing subdirectory not allowed
Path(filename).touch() # create file
newfilename = os.path.join(dirname, "nonexistingsubdir", "newfile.txt")
try:
filesystem.renamer(filename, newfilename) # rename() does not return a value ...
except:
pass
assert os.path.isfile(filename)
assert not os.path.isfile(newfilename)
# Creation of subdirectory is allowed if create_local_directories=True
Path(filename).touch()
newfilename = os.path.join(dirname, "newsubdir", "newfile.txt")
try:
filesystem.renamer(filename, newfilename, create_local_directories=True)
except:
pass
assert not os.path.isfile(filename)
assert os.path.isfile(newfilename)
# Creation of subdirectory plus deeper sudbdir is allowed if create_local_directories=True
Path(filename).touch()
newfilename = os.path.join(dirname, "newsubdir", "deepersubdir", "newfile.txt")
try:
filesystem.renamer(filename, newfilename, create_local_directories=True)
except:
pass
assert not os.path.isfile(filename)
assert os.path.isfile(newfilename)
# ... escaping the directory plus subdir creation is not allowed
Path(filename).touch()
newfilename = os.path.join(dirname, "..", "newsubdir", "newfile.txt")
try:
filesystem.renamer(filename, newfilename, create_local_directories=True)
except:
pass
assert os.path.isfile(filename)
assert not os.path.isfile(newfilename)
# Cleanup working directory
shutil.rmtree(dirname)

Loading…
Cancel
Save