diff --git a/INSTALL.txt b/INSTALL.txt index 68b5dcb..51c5f8e 100644 --- a/INSTALL.txt +++ b/INSTALL.txt @@ -60,6 +60,7 @@ All platforms Windows PyWin32 use "pip install pypiwin32" + subprocessww use "pip install subprocessww" Essential modules cheetah-2.0.1+ use "pip install cheetah" diff --git a/README.md b/README.md index 154df90..51c25d0 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,6 @@ SABnzbd has a good deal of dependencies you'll need before you can get running. - `python` (only 2.7.x and higher, but not 3.x.x) - `python-cheetah` -- `python-support` - `par2` (Multi-threaded par2 installation guide can be found [here](https://sabnzbd.org/wiki/installation/multicore-par2)) - `unrar` (Make sure you get the "official" non-free version of unrar) - `sabyenc` (installation guide can be found [here](https://sabnzbd.org/sabyenc)) diff --git a/appveyor.yml b/appveyor.yml index 5dd6606..8a353dd 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,6 +1,6 @@ install: - pip install --upgrade -r tests/requirements.txt - - pip install pypiwin32 + - pip install pypiwin32 subprocessww build_script: - pytest diff --git a/sabnzbd/directunpacker.py b/sabnzbd/directunpacker.py index 735fa09..4819500 100644 --- a/sabnzbd/directunpacker.py +++ b/sabnzbd/directunpacker.py @@ -36,11 +36,14 @@ from sabnzbd.utils.rarfile import RarFile from sabnzbd.utils.diskspeed import diskspeedmeasure if sabnzbd.WIN32: - # Load the POpen from the fixed unicode-subprocess - from sabnzbd.utils.subprocess_fix import Popen -else: - # Load the regular POpen - from subprocess import Popen + try: + # Use patched version of subprocess module for Unicode on Windows + import subprocessww + except ImportError: + pass + +# Load the regular POpen (which is now patched on Windows) +from subprocess import Popen MAX_ACTIVE_UNPACKERS = 10 ACTIVE_UNPACKERS = [] diff --git a/sabnzbd/newsunpack.py b/sabnzbd/newsunpack.py index 1e91ecd..588b848 100644 --- a/sabnzbd/newsunpack.py +++ b/sabnzbd/newsunpack.py @@ -43,10 +43,11 @@ if sabnzbd.WIN32: import win32api from win32con import SW_HIDE from win32process import STARTF_USESHOWWINDOW, IDLE_PRIORITY_CLASS + + # Use patched version of subprocess module for Unicode on Windows + import subprocessww except ImportError: pass - # Load the POpen from the fixed unicode-subprocess - from sabnzbd.utils.subprocess_fix import Popen else: # Define dummy WindowsError for non-Windows class WindowsError(Exception): @@ -55,8 +56,9 @@ else: def __str__(self): return repr(self.parameter) - # Load the regular POpen - from subprocess import Popen + +# Load the regular POpen (which is now patched on Windows) +from subprocess import Popen # Regex globals RAR_RE = re.compile(r'\.(?Ppart\d*\.rar|rar|r\d\d|s\d\d|t\d\d|u\d\d|v\d\d|\d\d\d)$', re.I) @@ -628,10 +630,6 @@ def rar_extract_core(rarfile_path, numrars, one_folder, nzo, setname, extraction command = ['%s' % RAR_COMMAND, action, '-idp', overwrite, rename, '-ai', password_command, '%s' % clip_path(rarfile_path), '%s\\' % extraction_path] - # The subprocess_fix requires time to clear the buffers to work, - # otherwise the inputs get send incorrectly and unrar breaks - time.sleep(0.5) - elif RAR_PROBLEM: # Use only oldest options (specifically no "-or") command = ['%s' % RAR_COMMAND, action, '-idp', overwrite, password_command, diff --git a/sabnzbd/utils/subprocess_fix.py b/sabnzbd/utils/subprocess_fix.py deleted file mode 100644 index 9184176..0000000 --- a/sabnzbd/utils/subprocess_fix.py +++ /dev/null @@ -1,181 +0,0 @@ -## Fixing python 2.7 windows unicode issue with ``subprocess.Popen``. - -## Copied from -## http://vaab.blog.kal.fr/2017/03/16/fixing-windows-python-2-7-unicode-issue-with-subprocesss-popen/ -## https://gist.github.com/vaab/2ad7051fc193167f15f85ef573e54eb9 - -## issue: https://bugs.python.org/issue19264 -import os -import time -import ctypes -import subprocess -import _subprocess -from ctypes import byref, windll, c_char_p, c_wchar_p, c_void_p, \ - Structure, sizeof, c_wchar, WinError -from ctypes.wintypes import BYTE, WORD, LPWSTR, BOOL, DWORD, LPVOID, \ - HANDLE - - -## -## Special counter because this function cannot -## be called within 2 seconds from each other! -## -_NEXT_PROCESS_START = 0.0 -_NEXT_PROCESS_DELAY = 2.0 - - -## -## Types -## - -CREATE_UNICODE_ENVIRONMENT = 0x00000400 -LPCTSTR = c_char_p -LPTSTR = c_wchar_p -LPSECURITY_ATTRIBUTES = c_void_p -LPBYTE = ctypes.POINTER(BYTE) - -class STARTUPINFOW(Structure): - _fields_ = [ - ("cb", DWORD), ("lpReserved", LPWSTR), - ("lpDesktop", LPWSTR), ("lpTitle", LPWSTR), - ("dwX", DWORD), ("dwY", DWORD), - ("dwXSize", DWORD), ("dwYSize", DWORD), - ("dwXCountChars", DWORD), ("dwYCountChars", DWORD), - ("dwFillAtrribute", DWORD), ("dwFlags", DWORD), - ("wShowWindow", WORD), ("cbReserved2", WORD), - ("lpReserved2", LPBYTE), ("hStdInput", HANDLE), - ("hStdOutput", HANDLE), ("hStdError", HANDLE), - ] - -LPSTARTUPINFOW = ctypes.POINTER(STARTUPINFOW) - - -class PROCESS_INFORMATION(Structure): - _fields_ = [ - ("hProcess", HANDLE), ("hThread", HANDLE), - ("dwProcessId", DWORD), ("dwThreadId", DWORD), - ] - -LPPROCESS_INFORMATION = ctypes.POINTER(PROCESS_INFORMATION) - - -class DUMMY_HANDLE(ctypes.c_void_p): - - def __init__(self, *a, **kw): - super(DUMMY_HANDLE, self).__init__(*a, **kw) - self.closed = False - - def Close(self): - if not self.closed: - windll.kernel32.CloseHandle(self) - self.closed = True - - def __int__(self): - return self.value - - -CreateProcessW = windll.kernel32.CreateProcessW -CreateProcessW.argtypes = [ - LPCTSTR, LPTSTR, LPSECURITY_ATTRIBUTES, - LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCTSTR, - LPSTARTUPINFOW, LPPROCESS_INFORMATION, -] -CreateProcessW.restype = BOOL - - -## -## Patched functions/classes -## - -def CreateProcess(executable, args, _p_attr, _t_attr, - inherit_handles, creation_flags, env, cwd, - startup_info): - """Create a process supporting unicode executable and args for win32 - - Python implementation of CreateProcess using CreateProcessW for Win32 - - """ - # Do we need to delay? - global _NEXT_PROCESS_START - diff_start = _NEXT_PROCESS_START - time.time() - if(diff_start > 0.0): - # Wait ourselves and make sure others also wait - _NEXT_PROCESS_START += _NEXT_PROCESS_DELAY - time.sleep(diff_start) - else: - _NEXT_PROCESS_START = time.time() + _NEXT_PROCESS_DELAY - - si = STARTUPINFOW( - dwFlags=startup_info.dwFlags, - wShowWindow=startup_info.wShowWindow, - cb=sizeof(STARTUPINFOW), - ) - - # Only cast to ints when it's given - if startup_info.hStdInput: - si.hStdInput = int(startup_info.hStdInput) - if startup_info.hStdOutput: - si.hStdOutput = int(startup_info.hStdOutput) - if startup_info.hStdError: - si.hStdError = int(startup_info.hStdError) - - wenv = None - if env is not None: - ## LPCWSTR seems to be c_wchar_p, so let's say CWSTR is c_wchar - env = (unicode("").join([ - unicode("%s=%s\0") % (k, v) - for k, v in env.items()])) + unicode("\0") - wenv = (c_wchar * len(env))() - wenv.value = env - - pi = PROCESS_INFORMATION() - creation_flags |= CREATE_UNICODE_ENVIRONMENT - - if CreateProcessW(executable, args, None, None, - inherit_handles, creation_flags, - wenv, cwd, byref(si), byref(pi)): - return (DUMMY_HANDLE(pi.hProcess), DUMMY_HANDLE(pi.hThread), - pi.dwProcessId, pi.dwThreadId) - raise WinError() - - -class Popen(subprocess.Popen): - """This superseeds Popen and corrects a bug in cPython 2.7 implem""" - - def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, to_close, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite): - """Code from part of _execute_child from Python 2.7 (9fbb65e) - - There are only 2 little changes concerning the construction of - the the final string in shell mode: we preempt the creation of - the command string when shell is True, because original function - will try to encode unicode args which we want to avoid to be able to - sending it as-is to ``CreateProcess``. - - """ - if not isinstance(args, subprocess.types.StringTypes): - args = subprocess.list2cmdline(args) - - if startupinfo is None: - startupinfo = subprocess.STARTUPINFO() - if shell: - startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW - startupinfo.wShowWindow = _subprocess.SW_HIDE - comspec = os.environ.get("COMSPEC", unicode("cmd.exe")) - args = unicode('{} /c "{}"').format(comspec, args) - if (_subprocess.GetVersion() >= 0x80000000 or - os.path.basename(comspec).lower() == "command.com"): - w9xpopen = self._find_w9xpopen() - args = unicode('"%s" %s') % (w9xpopen, args) - creationflags |= _subprocess.CREATE_NEW_CONSOLE - - super(Popen, self)._execute_child(args, executable, - preexec_fn, close_fds, cwd, env, universal_newlines, - startupinfo, creationflags, False, to_close, p2cread, - p2cwrite, c2pread, c2pwrite, errread, errwrite) - -_subprocess.CreateProcess = CreateProcess