Browse Source

Use special patched version of subprocess for Unicode POpen on Windows

tags/2.3.1Beta1
Safihre 8 years ago
parent
commit
52267a9565
  1. 1
      INSTALL.txt
  2. 1
      README.md
  3. 2
      appveyor.yml
  4. 13
      sabnzbd/directunpacker.py
  5. 14
      sabnzbd/newsunpack.py
  6. 181
      sabnzbd/utils/subprocess_fix.py

1
INSTALL.txt

@ -60,6 +60,7 @@ All platforms
Windows
PyWin32 use "pip install pypiwin32"
subprocessww use "pip install subprocessww"
Essential modules
cheetah-2.0.1+ use "pip install cheetah"

1
README.md

@ -13,7 +13,6 @@ SABnzbd has a good deal of dependencies you'll need before you can get running.
- `python` (only 2.7.x and higher, but not 3.x.x)
- `python-cheetah`
- `python-support`
- `par2` (Multi-threaded par2 installation guide can be found [here](https://sabnzbd.org/wiki/installation/multicore-par2))
- `unrar` (Make sure you get the "official" non-free version of unrar)
- `sabyenc` (installation guide can be found [here](https://sabnzbd.org/sabyenc))

2
appveyor.yml

@ -1,6 +1,6 @@
install:
- pip install --upgrade -r tests/requirements.txt
- pip install pypiwin32
- pip install pypiwin32 subprocessww
build_script:
- pytest

13
sabnzbd/directunpacker.py

@ -36,11 +36,14 @@ from sabnzbd.utils.rarfile import RarFile
from sabnzbd.utils.diskspeed import diskspeedmeasure
if sabnzbd.WIN32:
# Load the POpen from the fixed unicode-subprocess
from sabnzbd.utils.subprocess_fix import Popen
else:
# Load the regular POpen
from subprocess import Popen
try:
# Use patched version of subprocess module for Unicode on Windows
import subprocessww
except ImportError:
pass
# Load the regular POpen (which is now patched on Windows)
from subprocess import Popen
MAX_ACTIVE_UNPACKERS = 10
ACTIVE_UNPACKERS = []

14
sabnzbd/newsunpack.py

@ -43,10 +43,11 @@ if sabnzbd.WIN32:
import win32api
from win32con import SW_HIDE
from win32process import STARTF_USESHOWWINDOW, IDLE_PRIORITY_CLASS
# Use patched version of subprocess module for Unicode on Windows
import subprocessww
except ImportError:
pass
# Load the POpen from the fixed unicode-subprocess
from sabnzbd.utils.subprocess_fix import Popen
else:
# Define dummy WindowsError for non-Windows
class WindowsError(Exception):
@ -55,8 +56,9 @@ else:
def __str__(self):
return repr(self.parameter)
# Load the regular POpen
from subprocess import Popen
# Load the regular POpen (which is now patched on Windows)
from subprocess import Popen
# Regex globals
RAR_RE = re.compile(r'\.(?P<ext>part\d*\.rar|rar|r\d\d|s\d\d|t\d\d|u\d\d|v\d\d|\d\d\d)$', re.I)
@ -628,10 +630,6 @@ def rar_extract_core(rarfile_path, numrars, one_folder, nzo, setname, extraction
command = ['%s' % RAR_COMMAND, action, '-idp', overwrite, rename, '-ai', password_command,
'%s' % clip_path(rarfile_path), '%s\\' % extraction_path]
# The subprocess_fix requires time to clear the buffers to work,
# otherwise the inputs get send incorrectly and unrar breaks
time.sleep(0.5)
elif RAR_PROBLEM:
# Use only oldest options (specifically no "-or")
command = ['%s' % RAR_COMMAND, action, '-idp', overwrite, password_command,

181
sabnzbd/utils/subprocess_fix.py

@ -1,181 +0,0 @@
## Fixing python 2.7 windows unicode issue with ``subprocess.Popen``.
## Copied from
## http://vaab.blog.kal.fr/2017/03/16/fixing-windows-python-2-7-unicode-issue-with-subprocesss-popen/
## https://gist.github.com/vaab/2ad7051fc193167f15f85ef573e54eb9
## issue: https://bugs.python.org/issue19264
import os
import time
import ctypes
import subprocess
import _subprocess
from ctypes import byref, windll, c_char_p, c_wchar_p, c_void_p, \
Structure, sizeof, c_wchar, WinError
from ctypes.wintypes import BYTE, WORD, LPWSTR, BOOL, DWORD, LPVOID, \
HANDLE
##
## Special counter because this function cannot
## be called within 2 seconds from each other!
##
_NEXT_PROCESS_START = 0.0
_NEXT_PROCESS_DELAY = 2.0
##
## Types
##
CREATE_UNICODE_ENVIRONMENT = 0x00000400
LPCTSTR = c_char_p
LPTSTR = c_wchar_p
LPSECURITY_ATTRIBUTES = c_void_p
LPBYTE = ctypes.POINTER(BYTE)
class STARTUPINFOW(Structure):
_fields_ = [
("cb", DWORD), ("lpReserved", LPWSTR),
("lpDesktop", LPWSTR), ("lpTitle", LPWSTR),
("dwX", DWORD), ("dwY", DWORD),
("dwXSize", DWORD), ("dwYSize", DWORD),
("dwXCountChars", DWORD), ("dwYCountChars", DWORD),
("dwFillAtrribute", DWORD), ("dwFlags", DWORD),
("wShowWindow", WORD), ("cbReserved2", WORD),
("lpReserved2", LPBYTE), ("hStdInput", HANDLE),
("hStdOutput", HANDLE), ("hStdError", HANDLE),
]
LPSTARTUPINFOW = ctypes.POINTER(STARTUPINFOW)
class PROCESS_INFORMATION(Structure):
_fields_ = [
("hProcess", HANDLE), ("hThread", HANDLE),
("dwProcessId", DWORD), ("dwThreadId", DWORD),
]
LPPROCESS_INFORMATION = ctypes.POINTER(PROCESS_INFORMATION)
class DUMMY_HANDLE(ctypes.c_void_p):
def __init__(self, *a, **kw):
super(DUMMY_HANDLE, self).__init__(*a, **kw)
self.closed = False
def Close(self):
if not self.closed:
windll.kernel32.CloseHandle(self)
self.closed = True
def __int__(self):
return self.value
CreateProcessW = windll.kernel32.CreateProcessW
CreateProcessW.argtypes = [
LPCTSTR, LPTSTR, LPSECURITY_ATTRIBUTES,
LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCTSTR,
LPSTARTUPINFOW, LPPROCESS_INFORMATION,
]
CreateProcessW.restype = BOOL
##
## Patched functions/classes
##
def CreateProcess(executable, args, _p_attr, _t_attr,
inherit_handles, creation_flags, env, cwd,
startup_info):
"""Create a process supporting unicode executable and args for win32
Python implementation of CreateProcess using CreateProcessW for Win32
"""
# Do we need to delay?
global _NEXT_PROCESS_START
diff_start = _NEXT_PROCESS_START - time.time()
if(diff_start > 0.0):
# Wait ourselves and make sure others also wait
_NEXT_PROCESS_START += _NEXT_PROCESS_DELAY
time.sleep(diff_start)
else:
_NEXT_PROCESS_START = time.time() + _NEXT_PROCESS_DELAY
si = STARTUPINFOW(
dwFlags=startup_info.dwFlags,
wShowWindow=startup_info.wShowWindow,
cb=sizeof(STARTUPINFOW),
)
# Only cast to ints when it's given
if startup_info.hStdInput:
si.hStdInput = int(startup_info.hStdInput)
if startup_info.hStdOutput:
si.hStdOutput = int(startup_info.hStdOutput)
if startup_info.hStdError:
si.hStdError = int(startup_info.hStdError)
wenv = None
if env is not None:
## LPCWSTR seems to be c_wchar_p, so let's say CWSTR is c_wchar
env = (unicode("").join([
unicode("%s=%s\0") % (k, v)
for k, v in env.items()])) + unicode("\0")
wenv = (c_wchar * len(env))()
wenv.value = env
pi = PROCESS_INFORMATION()
creation_flags |= CREATE_UNICODE_ENVIRONMENT
if CreateProcessW(executable, args, None, None,
inherit_handles, creation_flags,
wenv, cwd, byref(si), byref(pi)):
return (DUMMY_HANDLE(pi.hProcess), DUMMY_HANDLE(pi.hThread),
pi.dwProcessId, pi.dwThreadId)
raise WinError()
class Popen(subprocess.Popen):
"""This superseeds Popen and corrects a bug in cPython 2.7 implem"""
def _execute_child(self, args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell, to_close,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite):
"""Code from part of _execute_child from Python 2.7 (9fbb65e)
There are only 2 little changes concerning the construction of
the the final string in shell mode: we preempt the creation of
the command string when shell is True, because original function
will try to encode unicode args which we want to avoid to be able to
sending it as-is to ``CreateProcess``.
"""
if not isinstance(args, subprocess.types.StringTypes):
args = subprocess.list2cmdline(args)
if startupinfo is None:
startupinfo = subprocess.STARTUPINFO()
if shell:
startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = _subprocess.SW_HIDE
comspec = os.environ.get("COMSPEC", unicode("cmd.exe"))
args = unicode('{} /c "{}"').format(comspec, args)
if (_subprocess.GetVersion() >= 0x80000000 or
os.path.basename(comspec).lower() == "command.com"):
w9xpopen = self._find_w9xpopen()
args = unicode('"%s" %s') % (w9xpopen, args)
creationflags |= _subprocess.CREATE_NEW_CONSOLE
super(Popen, self)._execute_child(args, executable,
preexec_fn, close_fds, cwd, env, universal_newlines,
startupinfo, creationflags, False, to_close, p2cread,
p2cwrite, c2pread, c2pwrite, errread, errwrite)
_subprocess.CreateProcess = CreateProcess
Loading…
Cancel
Save