diff --git a/Cargo.lock b/Cargo.lock index 0a4f1ada6..162f9b67e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1633,6 +1633,7 @@ dependencies = [ "bstr", "byteorder 1.3.4", "caseless", + "cfg-if", "chrono", "crc", "crc32fast", @@ -1694,7 +1695,6 @@ dependencies = [ "smallbox", "socket2", "statrs", - "subprocess", "thread_local", "uname", "unic-bidi", @@ -1962,16 +1962,6 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" -[[package]] -name = "subprocess" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5109e3c3215ce5339ca9766a6db7f39144ccc7d9bad53b96d65cfb14ae6c2985" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "subtle" version = "1.0.0" diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index de7b71d80..7c63fb904 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -9,8 +9,7 @@ if sys.platform != 'win32': # pragma: no cover import _winapi import itertools -# XXX RustPython TODO: msvcrt -# import msvcrt +import msvcrt import os import socket import subprocess diff --git a/Lib/platform.py b/Lib/platform.py index eadd73466..21d11ff17 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -281,10 +281,9 @@ def _syscmd_ver(system='', release='', version='', for cmd in ('ver', 'command /c ver', 'cmd /c ver'): try: info = subprocess.check_output(cmd, - stderr=subprocess.DEVNULL,) - # XXX RustPython TODO: more Popen args - # text=True, - # shell=True) + stderr=subprocess.DEVNULL, + text=True, + shell=True) except (OSError, subprocess.CalledProcessError) as why: #print('Command %s failed: %s' % (cmd, why)) continue diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 3788a100d..5ce787d61 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -41,89 +41,18 @@ getstatusoutput(...): Runs a command in the shell, waits for it to complete, then returns a (exitcode, output) tuple """ -import sys - +import builtins +import errno import io import os import time import signal -import builtins +import sys +import threading import warnings -import errno +import contextlib from time import monotonic as _time -from _subprocess import * - -# TODO: use these classes instead of the _subprocess ones - -# Exception classes used by this module. -# class SubprocessError(Exception): pass - - -# class CalledProcessError(SubprocessError): -# """Raised when run() is called with check=True and the process -# returns a non-zero exit status. - -# Attributes: -# cmd, returncode, stdout, stderr, output -# """ -# def __init__(self, returncode, cmd, output=None, stderr=None): -# self.returncode = returncode -# self.cmd = cmd -# self.output = output -# self.stderr = stderr - -# def __str__(self): -# if self.returncode and self.returncode < 0: -# try: -# return "Command '%s' died with %r." % ( -# self.cmd, signal.Signals(-self.returncode)) -# except ValueError: -# return "Command '%s' died with unknown signal %d." % ( -# self.cmd, -self.returncode) -# else: -# return "Command '%s' returned non-zero exit status %d." % ( -# self.cmd, self.returncode) - -# @property -# def stdout(self): -# """Alias for output attribute, to match stderr""" -# return self.output - -# @stdout.setter -# def stdout(self, value): -# # There's no obvious reason to set this, but allow it anyway so -# # .stdout is a transparent alias for .output -# self.output = value - - -# class TimeoutExpired(SubprocessError): -# """This exception is raised when the timeout expires while waiting for a -# child process. - -# Attributes: -# cmd, output, stdout, stderr, timeout -# """ -# def __init__(self, cmd, timeout, output=None, stderr=None): -# self.cmd = cmd -# self.timeout = timeout -# self.output = output -# self.stderr = stderr - -# def __str__(self): -# return ("Command '%s' timed out after %s seconds" % -# (self.cmd, self.timeout)) - -# @property -# def stdout(self): -# return self.output - -# @stdout.setter -# def stdout(self, value): -# # There's no obvious reason to set this, but allow it anyway so -# # .stdout is a transparent alias for .output -# self.output = value - __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL", @@ -131,22 +60,196 @@ __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", # NOTE: We intentionally exclude list2cmdline as it is # considered an internal implementation detail. issue10838. -# This lists holds Popen instances for which the underlying process had not -# exited at the time its __del__ method got called: those processes are wait()ed -# for synchronously from _cleanup() when a new Popen object is created, to avoid -# zombie processes. -_active = [] +try: + import msvcrt + import _winapi + _mswindows = True +except ModuleNotFoundError: + _mswindows = False + import _posixsubprocess + import select + import selectors +else: + from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, + STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, + STD_ERROR_HANDLE, SW_HIDE, + STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, + ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, + HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS, + NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS, + CREATE_NO_WINDOW, DETACHED_PROCESS, + CREATE_DEFAULT_ERROR_MODE, CREATE_BREAKAWAY_FROM_JOB) -def _cleanup(): - for inst in _active[:]: - res = inst._internal_poll(_deadstate=sys.maxsize) - if res is not None: + __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", + "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", + "STD_ERROR_HANDLE", "SW_HIDE", + "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW", + "STARTUPINFO", + "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", + "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", + "NORMAL_PRIORITY_CLASS", "REALTIME_PRIORITY_CLASS", + "CREATE_NO_WINDOW", "DETACHED_PROCESS", + "CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"]) + + +# Exception classes used by this module. +class SubprocessError(Exception): pass + + +class CalledProcessError(SubprocessError): + """Raised when run() is called with check=True and the process + returns a non-zero exit status. + + Attributes: + cmd, returncode, stdout, stderr, output + """ + def __init__(self, returncode, cmd, output=None, stderr=None): + self.returncode = returncode + self.cmd = cmd + self.output = output + self.stderr = stderr + + def __str__(self): + if self.returncode and self.returncode < 0: try: - _active.remove(inst) + return "Command '%s' died with %r." % ( + self.cmd, signal.Signals(-self.returncode)) except ValueError: - # This can happen if two threads create a new Popen instance. - # It's harmless that it was already removed, so ignore. - pass + return "Command '%s' died with unknown signal %d." % ( + self.cmd, -self.returncode) + else: + return "Command '%s' returned non-zero exit status %d." % ( + self.cmd, self.returncode) + + @property + def stdout(self): + """Alias for output attribute, to match stderr""" + return self.output + + @stdout.setter + def stdout(self, value): + # There's no obvious reason to set this, but allow it anyway so + # .stdout is a transparent alias for .output + self.output = value + + +class TimeoutExpired(SubprocessError): + """This exception is raised when the timeout expires while waiting for a + child process. + + Attributes: + cmd, output, stdout, stderr, timeout + """ + def __init__(self, cmd, timeout, output=None, stderr=None): + self.cmd = cmd + self.timeout = timeout + self.output = output + self.stderr = stderr + + def __str__(self): + return ("Command '%s' timed out after %s seconds" % + (self.cmd, self.timeout)) + + @property + def stdout(self): + return self.output + + @stdout.setter + def stdout(self, value): + # There's no obvious reason to set this, but allow it anyway so + # .stdout is a transparent alias for .output + self.output = value + + +if _mswindows: + class STARTUPINFO: + def __init__(self, *, dwFlags=0, hStdInput=None, hStdOutput=None, + hStdError=None, wShowWindow=0, lpAttributeList=None): + self.dwFlags = dwFlags + self.hStdInput = hStdInput + self.hStdOutput = hStdOutput + self.hStdError = hStdError + self.wShowWindow = wShowWindow + self.lpAttributeList = lpAttributeList or {"handle_list": []} + + def copy(self): + attr_list = self.lpAttributeList.copy() + if 'handle_list' in attr_list: + attr_list['handle_list'] = list(attr_list['handle_list']) + + return STARTUPINFO(dwFlags=self.dwFlags, + hStdInput=self.hStdInput, + hStdOutput=self.hStdOutput, + hStdError=self.hStdError, + wShowWindow=self.wShowWindow, + lpAttributeList=attr_list) + + + class Handle(int): + closed = False + + def Close(self, CloseHandle=_winapi.CloseHandle): + if not self.closed: + self.closed = True + CloseHandle(self) + + def Detach(self): + if not self.closed: + self.closed = True + return int(self) + raise ValueError("already closed") + + def __repr__(self): + return "%s(%d)" % (self.__class__.__name__, int(self)) + + __del__ = Close +else: + # When select or poll has indicated that the file is writable, + # we can write up to _PIPE_BUF bytes without risk of blocking. + # POSIX defines PIPE_BUF as >= 512. + _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) + + # poll/select have the advantage of not requiring any extra file + # descriptor, contrarily to epoll/kqueue (also, they require a single + # syscall). + if hasattr(selectors, 'PollSelector'): + _PopenSelector = selectors.PollSelector + else: + _PopenSelector = selectors.SelectSelector + + +if _mswindows: + # On Windows we just need to close `Popen._handle` when we no longer need + # it, so that the kernel can free it. `Popen._handle` gets closed + # implicitly when the `Popen` instance is finalized (see `Handle.__del__`, + # which is calling `CloseHandle` as requested in [1]), so there is nothing + # for `_cleanup` to do. + # + # [1] https://docs.microsoft.com/en-us/windows/desktop/ProcThread/ + # creating-processes + _active = None + + def _cleanup(): + pass +else: + # This lists holds Popen instances for which the underlying process had not + # exited at the time its __del__ method got called: those processes are + # wait()ed for synchronously from _cleanup() when a new Popen object is + # created, to avoid zombie processes. + _active = [] + + def _cleanup(): + if _active is None: + return + for inst in _active[:]: + res = inst._internal_poll(_deadstate=sys.maxsize) + if res is not None: + try: + _active.remove(inst) + except ValueError: + # This can happen if two threads create a new Popen instance. + # It's harmless that it was already removed, so ignore. + pass PIPE = -1 STDOUT = -2 @@ -175,9 +278,7 @@ def _args_from_interpreter_flags(): # 'inspect': 'i', # 'interactive': 'i', 'dont_write_bytecode': 'B', - 'no_user_site': 's', 'no_site': 'S', - 'ignore_environment': 'E', 'verbose': 'v', 'bytes_warning': 'b', 'quiet': 'q', @@ -189,6 +290,14 @@ def _args_from_interpreter_flags(): if v > 0: args.append('-' + opt * v) + if sys.flags.isolated: + args.append('-I') + else: + if sys.flags.ignore_environment: + args.append('-E') + if sys.flags.no_user_site: + args.append('-s') + # -W options warnopts = sys.warnoptions[:] bytes_warning = sys.flags.bytes_warning @@ -286,7 +395,7 @@ def check_output(*popenargs, timeout=None, **kwargs): b'when in the course of barman events\n' By default, all communication is in bytes, and therefore any "input" - should be bytes, and the return value wil be bytes. If in text mode, + should be bytes, and the return value will be bytes. If in text mode, any "input" should be a string, and the return value will be a string decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. @@ -366,12 +475,12 @@ def run(*popenargs, The other arguments are the same as for the Popen constructor. """ if input is not None: - if 'stdin' in kwargs: + if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: - if ('stdout' in kwargs) or ('stderr' in kwargs): + if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE @@ -380,11 +489,20 @@ def run(*popenargs, with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) - except TimeoutExpired: + except TimeoutExpired as exc: process.kill() - stdout, stderr = process.communicate() - raise TimeoutExpired(process.args, timeout, output=stdout, - stderr=stderr) + if _mswindows: + # Windows accumulates the output in a single blocking + # read() call run on child threads, with the timeout + # being done in a join() on those threads. communicate() + # _after_ kill() is required to collect that and add it + # to the exception. + exc.stdout, exc.stderr = process.communicate() + else: + # POSIX _communicate already populated the output so + # far into the TimeoutExpired exception. + process.wait() + raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. @@ -428,7 +546,7 @@ def list2cmdline(seq): # "Parsing C++ Command-Line Arguments" result = [] needquote = False - for arg in seq: + for arg in map(os.fsdecode, seq): bs_buf = [] # Add a space to separate this argument from the others @@ -511,3 +629,1326 @@ def getoutput(cmd): '/bin/ls' """ return getstatusoutput(cmd)[1] + + +def _use_posix_spawn(): + """Check if posix_spawn() can be used for subprocess. + + subprocess requires a posix_spawn() implementation that properly reports + errors to the parent process, & sets errno on the following failures: + + * Process attribute actions failed. + * File actions failed. + * exec() failed. + + Prefer an implementation which can use vfork() in some cases for best + performance. + """ + if _mswindows or not hasattr(os, 'posix_spawn'): + # os.posix_spawn() is not available + return False + + if sys.platform == 'darwin': + # posix_spawn() is a syscall on macOS and properly reports errors + return True + + # Check libc name and runtime libc version + try: + ver = os.confstr('CS_GNU_LIBC_VERSION') + # parse 'glibc 2.28' as ('glibc', (2, 28)) + parts = ver.split(maxsplit=1) + if len(parts) != 2: + # reject unknown format + raise ValueError + libc = parts[0] + version = tuple(map(int, parts[1].split('.'))) + + if sys.platform == 'linux' and libc == 'glibc' and version >= (2, 24): + # glibc 2.24 has a new Linux posix_spawn implementation using vfork + # which properly reports errors to the parent process. + return True + # Note: Don't use the implementation in earlier glibc because it doesn't + # use vfork (even if glibc 2.26 added a pipe to properly report errors + # to the parent process). + except (AttributeError, ValueError, OSError): + # os.confstr() or CS_GNU_LIBC_VERSION value not available + pass + + # By default, assume that posix_spawn() does not properly report errors. + return False + + +_USE_POSIX_SPAWN = _use_posix_spawn() + + +class Popen(object): + """ Execute a child program in a new process. + + For a complete description of the arguments see the Python documentation. + + Arguments: + args: A string, or a sequence of program arguments. + + bufsize: supplied as the buffering argument to the open() function when + creating the stdin/stdout/stderr pipe file objects + + executable: A replacement program to execute. + + stdin, stdout and stderr: These specify the executed programs' standard + input, standard output and standard error file handles, respectively. + + preexec_fn: (POSIX only) An object to be called in the child process + just before the child is executed. + + close_fds: Controls closing or inheriting of file descriptors. + + shell: If true, the command will be executed through the shell. + + cwd: Sets the current directory before the child is executed. + + env: Defines the environment variables for the new process. + + text: If true, decode stdin, stdout and stderr using the given encoding + (if set) or the system default otherwise. + + universal_newlines: Alias of text, provided for backwards compatibility. + + startupinfo and creationflags (Windows only) + + restore_signals (POSIX only) + + start_new_session (POSIX only) + + pass_fds (POSIX only) + + encoding and errors: Text mode encoding and error handling to use for + file objects stdin, stdout and stderr. + + Attributes: + stdin, stdout, stderr, pid, returncode + """ + _child_created = False # Set here since __del__ checks it + + def __init__(self, args, bufsize=-1, executable=None, + stdin=None, stdout=None, stderr=None, + preexec_fn=None, close_fds=True, + shell=False, cwd=None, env=None, universal_newlines=None, + startupinfo=None, creationflags=0, + restore_signals=True, start_new_session=False, + pass_fds=(), *, encoding=None, errors=None, text=None): + """Create new Popen instance.""" + _cleanup() + # Held while anything is calling waitpid before returncode has been + # updated to prevent clobbering returncode if wait() or poll() are + # called from multiple threads at once. After acquiring the lock, + # code must re-check self.returncode to see if another thread just + # finished a waitpid() call. + self._waitpid_lock = threading.Lock() + + self._input = None + self._communication_started = False + if bufsize is None: + bufsize = -1 # Restore default + if not isinstance(bufsize, int): + raise TypeError("bufsize must be an integer") + + if _mswindows: + if preexec_fn is not None: + raise ValueError("preexec_fn is not supported on Windows " + "platforms") + else: + # POSIX + if pass_fds and not close_fds: + warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) + close_fds = True + if startupinfo is not None: + raise ValueError("startupinfo is only supported on Windows " + "platforms") + if creationflags != 0: + raise ValueError("creationflags is only supported on Windows " + "platforms") + + self.args = args + self.stdin = None + self.stdout = None + self.stderr = None + self.pid = None + self.returncode = None + self.encoding = encoding + self.errors = errors + + # Validate the combinations of text and universal_newlines + if (text is not None and universal_newlines is not None + and bool(universal_newlines) != bool(text)): + raise SubprocessError('Cannot disambiguate when both text ' + 'and universal_newlines are supplied but ' + 'different. Pass one or the other.') + + # Input and output objects. The general principle is like + # this: + # + # Parent Child + # ------ ----- + # p2cwrite ---stdin---> p2cread + # c2pread <--stdout--- c2pwrite + # errread <--stderr--- errwrite + # + # On POSIX, the child objects are file descriptors. On + # Windows, these are Windows file handles. The parent objects + # are file descriptors on both platforms. The parent objects + # are -1 when not using PIPEs. The child objects are -1 + # when not redirecting. + + (p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) = self._get_handles(stdin, stdout, stderr) + + # We wrap OS handles *before* launching the child, otherwise a + # quickly terminating child could make our fds unwrappable + # (see #8458). + + # XXX RustPython TODO: have fds for fs functions be actual CRT fds on windows, not handles + # if _mswindows: + # if p2cwrite != -1: + # p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) + # if c2pread != -1: + # c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) + # if errread != -1: + # errread = msvcrt.open_osfhandle(errread.Detach(), 0) + + self.text_mode = encoding or errors or text or universal_newlines + + # How long to resume waiting on a child after the first ^C. + # There is no right value for this. The purpose is to be polite + # yet remain good for interactive users trying to exit a tool. + self._sigint_wait_secs = 0.25 # 1/xkcd221.getRandomNumber() + + self._closed_child_pipe_fds = False + + if self.text_mode: + if bufsize == 1: + line_buffering = True + # Use the default buffer size for the underlying binary streams + # since they don't support line buffering. + bufsize = -1 + else: + line_buffering = False + + try: + if p2cwrite != -1: + self.stdin = io.open(p2cwrite, 'wb', bufsize) + if self.text_mode: + self.stdin = io.TextIOWrapper(self.stdin, write_through=True, + line_buffering=line_buffering, + encoding=encoding, errors=errors) + if c2pread != -1: + self.stdout = io.open(c2pread, 'rb', bufsize) + if self.text_mode: + self.stdout = io.TextIOWrapper(self.stdout, + encoding=encoding, errors=errors) + if errread != -1: + self.stderr = io.open(errread, 'rb', bufsize) + if self.text_mode: + self.stderr = io.TextIOWrapper(self.stderr, + encoding=encoding, errors=errors) + + self._execute_child(args, executable, preexec_fn, close_fds, + pass_fds, cwd, env, + startupinfo, creationflags, shell, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite, + restore_signals, start_new_session) + except: + # Cleanup if the child failed starting. + for f in filter(None, (self.stdin, self.stdout, self.stderr)): + try: + f.close() + except OSError: + pass # Ignore EBADF or other errors. + + if not self._closed_child_pipe_fds: + to_close = [] + if stdin == PIPE: + to_close.append(p2cread) + if stdout == PIPE: + to_close.append(c2pwrite) + if stderr == PIPE: + to_close.append(errwrite) + if hasattr(self, '_devnull'): + to_close.append(self._devnull) + for fd in to_close: + try: + if _mswindows and isinstance(fd, Handle): + fd.Close() + else: + os.close(fd) + except OSError: + pass + + raise + + @property + def universal_newlines(self): + # universal_newlines as retained as an alias of text_mode for API + # compatibility. bpo-31756 + return self.text_mode + + @universal_newlines.setter + def universal_newlines(self, universal_newlines): + self.text_mode = bool(universal_newlines) + + def _translate_newlines(self, data, encoding, errors): + data = data.decode(encoding, errors) + return data.replace("\r\n", "\n").replace("\r", "\n") + + def __enter__(self): + return self + + def __exit__(self, exc_type, value, traceback): + if self.stdout: + self.stdout.close() + if self.stderr: + self.stderr.close() + try: # Flushing a BufferedWriter may raise an error + if self.stdin: + self.stdin.close() + finally: + if exc_type == KeyboardInterrupt: + # https://bugs.python.org/issue25942 + # In the case of a KeyboardInterrupt we assume the SIGINT + # was also already sent to our child processes. We can't + # block indefinitely as that is not user friendly. + # If we have not already waited a brief amount of time in + # an interrupted .wait() or .communicate() call, do so here + # for consistency. + if self._sigint_wait_secs > 0: + try: + self._wait(timeout=self._sigint_wait_secs) + except TimeoutExpired: + pass + self._sigint_wait_secs = 0 # Note that this has been done. + return # resume the KeyboardInterrupt + + # Wait for the process to terminate, to avoid zombies. + self.wait() + + def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn): + if not self._child_created: + # We didn't get to successfully create a child process. + return + if self.returncode is None: + # Not reading subprocess exit status creates a zombie process which + # is only destroyed at the parent python process exit + _warn("subprocess %s is still running" % self.pid, + ResourceWarning, source=self) + # In case the child hasn't been waited on, check if it's done. + self._internal_poll(_deadstate=_maxsize) + if self.returncode is None and _active is not None: + # Child is still running, keep us alive until we can wait on it. + _active.append(self) + + def _get_devnull(self): + if not hasattr(self, '_devnull'): + self._devnull = os.open(os.devnull, os.O_RDWR) + return self._devnull + + def _stdin_write(self, input): + if input: + try: + self.stdin.write(input) + except BrokenPipeError: + pass # communicate() must ignore broken pipe errors. + except OSError as exc: + if exc.errno == errno.EINVAL: + # bpo-19612, bpo-30418: On Windows, stdin.write() fails + # with EINVAL if the child process exited or if the child + # process is still running but closed the pipe. + pass + else: + raise + + try: + self.stdin.close() + except BrokenPipeError: + pass # communicate() must ignore broken pipe errors. + except OSError as exc: + if exc.errno == errno.EINVAL: + pass + else: + raise + + def communicate(self, input=None, timeout=None): + """Interact with process: Send data to stdin and close it. + Read data from stdout and stderr, until end-of-file is + reached. Wait for process to terminate. + + The optional "input" argument should be data to be sent to the + child process, or None, if no data should be sent to the child. + communicate() returns a tuple (stdout, stderr). + + By default, all communication is in bytes, and therefore any + "input" should be bytes, and the (stdout, stderr) will be bytes. + If in text mode (indicated by self.text_mode), any "input" should + be a string, and (stdout, stderr) will be strings decoded + according to locale encoding, or by "encoding" if set. Text mode + is triggered by setting any of text, encoding, errors or + universal_newlines. + """ + + if self._communication_started and input: + raise ValueError("Cannot send input after starting communication") + + # Optimization: If we are not worried about timeouts, we haven't + # started communicating, and we have one or zero pipes, using select() + # or threads is unnecessary. + if (timeout is None and not self._communication_started and + [self.stdin, self.stdout, self.stderr].count(None) >= 2): + stdout = None + stderr = None + if self.stdin: + self._stdin_write(input) + elif self.stdout: + stdout = self.stdout.read() + self.stdout.close() + elif self.stderr: + stderr = self.stderr.read() + self.stderr.close() + self.wait() + else: + if timeout is not None: + endtime = _time() + timeout + else: + endtime = None + + try: + stdout, stderr = self._communicate(input, endtime, timeout) + except KeyboardInterrupt: + # https://bugs.python.org/issue25942 + # See the detailed comment in .wait(). + if timeout is not None: + sigint_timeout = min(self._sigint_wait_secs, + self._remaining_time(endtime)) + else: + sigint_timeout = self._sigint_wait_secs + self._sigint_wait_secs = 0 # nothing else should wait. + try: + self._wait(timeout=sigint_timeout) + except TimeoutExpired: + pass + raise # resume the KeyboardInterrupt + + finally: + self._communication_started = True + + sts = self.wait(timeout=self._remaining_time(endtime)) + + return (stdout, stderr) + + + def poll(self): + """Check if child process has terminated. Set and return returncode + attribute.""" + return self._internal_poll() + + + def _remaining_time(self, endtime): + """Convenience for _communicate when computing timeouts.""" + if endtime is None: + return None + else: + return endtime - _time() + + + def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, + skip_check_and_raise=False): + """Convenience for checking if a timeout has expired.""" + if endtime is None: + return + if skip_check_and_raise or _time() > endtime: + raise TimeoutExpired( + self.args, orig_timeout, + output=b''.join(stdout_seq) if stdout_seq else None, + stderr=b''.join(stderr_seq) if stderr_seq else None) + + + def wait(self, timeout=None): + """Wait for child process to terminate; returns self.returncode.""" + if timeout is not None: + endtime = _time() + timeout + try: + return self._wait(timeout=timeout) + except KeyboardInterrupt: + # https://bugs.python.org/issue25942 + # The first keyboard interrupt waits briefly for the child to + # exit under the common assumption that it also received the ^C + # generated SIGINT and will exit rapidly. + if timeout is not None: + sigint_timeout = min(self._sigint_wait_secs, + self._remaining_time(endtime)) + else: + sigint_timeout = self._sigint_wait_secs + self._sigint_wait_secs = 0 # nothing else should wait. + try: + self._wait(timeout=sigint_timeout) + except TimeoutExpired: + pass + raise # resume the KeyboardInterrupt + + def _close_pipe_fds(self, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite): + # self._devnull is not always defined. + devnull_fd = getattr(self, '_devnull', None) + + with contextlib.ExitStack() as stack: + if _mswindows: + if p2cread != -1: + stack.callback(p2cread.Close) + if c2pwrite != -1: + stack.callback(c2pwrite.Close) + if errwrite != -1: + stack.callback(errwrite.Close) + else: + if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd: + stack.callback(os.close, p2cread) + if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd: + stack.callback(os.close, c2pwrite) + if errwrite != -1 and errread != -1 and errwrite != devnull_fd: + stack.callback(os.close, errwrite) + + if devnull_fd is not None: + stack.callback(os.close, devnull_fd) + + # Prevent a double close of these handles/fds from __init__ on error. + self._closed_child_pipe_fds = True + + if _mswindows: + # + # Windows methods + # + def _get_handles(self, stdin, stdout, stderr): + """Construct and return tuple with IO objects: + p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite + """ + if stdin is None and stdout is None and stderr is None: + return (-1, -1, -1, -1, -1, -1) + + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 + + if stdin is None: + p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE) + if p2cread is None: + p2cread, _ = _winapi.CreatePipe(None, 0) + p2cread = Handle(p2cread) + _winapi.CloseHandle(_) + elif stdin == PIPE: + p2cread, p2cwrite = _winapi.CreatePipe(None, 0) + p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite) + elif stdin == DEVNULL: + p2cread = msvcrt.get_osfhandle(self._get_devnull()) + elif isinstance(stdin, int): + p2cread = msvcrt.get_osfhandle(stdin) + else: + # Assuming file-like object + p2cread = msvcrt.get_osfhandle(stdin.fileno()) + # XXX RUSTPYTHON TODO: figure out why closing these old, non-inheritable + # pipe handles is necessary for us, but not CPython + old = p2cread + p2cread = self._make_inheritable(p2cread) + if stdin == PIPE: _winapi.CloseHandle(old) + + if stdout is None: + c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE) + if c2pwrite is None: + _, c2pwrite = _winapi.CreatePipe(None, 0) + c2pwrite = Handle(c2pwrite) + _winapi.CloseHandle(_) + elif stdout == PIPE: + c2pread, c2pwrite = _winapi.CreatePipe(None, 0) + c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite) + elif stdout == DEVNULL: + c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) + elif isinstance(stdout, int): + c2pwrite = msvcrt.get_osfhandle(stdout) + else: + # Assuming file-like object + c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) + # XXX RUSTPYTHON TODO: figure out why closing these old, non-inheritable + # pipe handles is necessary for us, but not CPython + old = c2pwrite + c2pwrite = self._make_inheritable(c2pwrite) + if stdout == PIPE: _winapi.CloseHandle(old) + + if stderr is None: + errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE) + if errwrite is None: + _, errwrite = _winapi.CreatePipe(None, 0) + errwrite = Handle(errwrite) + _winapi.CloseHandle(_) + elif stderr == PIPE: + errread, errwrite = _winapi.CreatePipe(None, 0) + errread, errwrite = Handle(errread), Handle(errwrite) + elif stderr == STDOUT: + errwrite = c2pwrite + elif stderr == DEVNULL: + errwrite = msvcrt.get_osfhandle(self._get_devnull()) + elif isinstance(stderr, int): + errwrite = msvcrt.get_osfhandle(stderr) + else: + # Assuming file-like object + errwrite = msvcrt.get_osfhandle(stderr.fileno()) + # XXX RUSTPYTHON TODO: figure out why closing these old, non-inheritable + # pipe handles is necessary for us, but not CPython + old = errwrite + errwrite = self._make_inheritable(errwrite) + if stderr == PIPE: _winapi.CloseHandle(old) + + return (p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + + + def _make_inheritable(self, handle): + """Return a duplicate of handle, which is inheritable""" + h = _winapi.DuplicateHandle( + _winapi.GetCurrentProcess(), handle, + _winapi.GetCurrentProcess(), 0, 1, + _winapi.DUPLICATE_SAME_ACCESS) + return Handle(h) + + + def _filter_handle_list(self, handle_list): + """Filter out console handles that can't be used + in lpAttributeList["handle_list"] and make sure the list + isn't empty. This also removes duplicate handles.""" + # An handle with it's lowest two bits set might be a special console + # handle that if passed in lpAttributeList["handle_list"], will + # cause it to fail. + return list({handle for handle in handle_list + if handle & 0x3 != 0x3 + or _winapi.GetFileType(handle) != + _winapi.FILE_TYPE_CHAR}) + + + def _execute_child(self, args, executable, preexec_fn, close_fds, + pass_fds, cwd, env, + startupinfo, creationflags, shell, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite, + unused_restore_signals, unused_start_new_session): + """Execute program (MS Windows version)""" + + assert not pass_fds, "pass_fds not supported on Windows." + + if isinstance(args, str): + pass + elif isinstance(args, bytes): + if shell: + raise TypeError('bytes args is not allowed on Windows') + args = list2cmdline([args]) + elif isinstance(args, os.PathLike): + if shell: + raise TypeError('path-like args is not allowed when ' + 'shell is true') + args = list2cmdline([args]) + else: + args = list2cmdline(args) + + if executable is not None: + executable = os.fsdecode(executable) + + # Process startup details + if startupinfo is None: + startupinfo = STARTUPINFO() + else: + # bpo-34044: Copy STARTUPINFO since it is modified above, + # so the caller can reuse it multiple times. + startupinfo = startupinfo.copy() + + use_std_handles = -1 not in (p2cread, c2pwrite, errwrite) + if use_std_handles: + startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES + startupinfo.hStdInput = p2cread + startupinfo.hStdOutput = c2pwrite + startupinfo.hStdError = errwrite + + attribute_list = startupinfo.lpAttributeList + have_handle_list = bool(attribute_list and + "handle_list" in attribute_list and + attribute_list["handle_list"]) + + # If we were given an handle_list or need to create one + if have_handle_list or (use_std_handles and close_fds): + if attribute_list is None: + attribute_list = startupinfo.lpAttributeList = {} + handle_list = attribute_list["handle_list"] = \ + list(attribute_list.get("handle_list", [])) + + if use_std_handles: + handle_list += [int(p2cread), int(c2pwrite), int(errwrite)] + + handle_list[:] = self._filter_handle_list(handle_list) + + if handle_list: + if not close_fds: + warnings.warn("startupinfo.lpAttributeList['handle_list'] " + "overriding close_fds", RuntimeWarning) + + # When using the handle_list we always request to inherit + # handles but the only handles that will be inherited are + # the ones in the handle_list + close_fds = False + + if shell: + startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW + startupinfo.wShowWindow = _winapi.SW_HIDE + comspec = os.environ.get("COMSPEC", "cmd.exe") + args = '{} /c "{}"'.format (comspec, args) + + if cwd is not None: + cwd = os.fsdecode(cwd) + + sys.audit("subprocess.Popen", executable, args, cwd, env) + + # Start the process + try: + hp, ht, pid, tid = _winapi.CreateProcess(executable, args, + # no special security + None, None, + int(not close_fds), + creationflags, + env, + cwd, + startupinfo) + finally: + # Child is launched. Close the parent's copy of those pipe + # handles that only the child should have open. You need + # to make sure that no handles to the write end of the + # output pipe are maintained in this process or else the + # pipe will not close when the child process exits and the + # ReadFile will hang. + self._close_pipe_fds(p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + + # Retain the process handle, but close the thread handle + self._child_created = True + self._handle = Handle(hp) + self.pid = pid + _winapi.CloseHandle(ht) + + def _internal_poll(self, _deadstate=None, + _WaitForSingleObject=_winapi.WaitForSingleObject, + _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0, + _GetExitCodeProcess=_winapi.GetExitCodeProcess): + """Check if child process has terminated. Returns returncode + attribute. + + This method is called by __del__, so it can only refer to objects + in its local scope. + + """ + if self.returncode is None: + if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0: + self.returncode = _GetExitCodeProcess(self._handle) + return self.returncode + + + def _wait(self, timeout): + """Internal implementation of wait() on Windows.""" + if timeout is None: + timeout_millis = _winapi.INFINITE + else: + timeout_millis = int(timeout * 1000) + if self.returncode is None: + # API note: Returns immediately if timeout_millis == 0. + result = _winapi.WaitForSingleObject(self._handle, + timeout_millis) + if result == _winapi.WAIT_TIMEOUT: + raise TimeoutExpired(self.args, timeout) + self.returncode = _winapi.GetExitCodeProcess(self._handle) + return self.returncode + + + def _readerthread(self, fh, buffer): + buffer.append(fh.read()) + fh.close() + + + def _communicate(self, input, endtime, orig_timeout): + # Start reader threads feeding into a list hanging off of this + # object, unless they've already been started. + if self.stdout and not hasattr(self, "_stdout_buff"): + self._stdout_buff = [] + self.stdout_thread = \ + threading.Thread(target=self._readerthread, + args=(self.stdout, self._stdout_buff)) + self.stdout_thread.daemon = True + self.stdout_thread.start() + if self.stderr and not hasattr(self, "_stderr_buff"): + self._stderr_buff = [] + self.stderr_thread = \ + threading.Thread(target=self._readerthread, + args=(self.stderr, self._stderr_buff)) + self.stderr_thread.daemon = True + self.stderr_thread.start() + + if self.stdin: + self._stdin_write(input) + + # Wait for the reader threads, or time out. If we time out, the + # threads remain reading and the fds left open in case the user + # calls communicate again. + if self.stdout is not None: + self.stdout_thread.join(self._remaining_time(endtime)) + if self.stdout_thread.is_alive(): + raise TimeoutExpired(self.args, orig_timeout) + if self.stderr is not None: + self.stderr_thread.join(self._remaining_time(endtime)) + if self.stderr_thread.is_alive(): + raise TimeoutExpired(self.args, orig_timeout) + + # Collect the output from and close both pipes, now that we know + # both have been read successfully. + stdout = None + stderr = None + if self.stdout: + stdout = self._stdout_buff + self.stdout.close() + if self.stderr: + stderr = self._stderr_buff + self.stderr.close() + + # All data exchanged. Translate lists into strings. + if stdout is not None: + stdout = stdout[0] + if stderr is not None: + stderr = stderr[0] + + return (stdout, stderr) + + def send_signal(self, sig): + """Send a signal to the process.""" + # Don't signal a process that we know has already died. + if self.returncode is not None: + return + if sig == signal.SIGTERM: + self.terminate() + elif sig == signal.CTRL_C_EVENT: + os.kill(self.pid, signal.CTRL_C_EVENT) + elif sig == signal.CTRL_BREAK_EVENT: + os.kill(self.pid, signal.CTRL_BREAK_EVENT) + else: + raise ValueError("Unsupported signal: {}".format(sig)) + + def terminate(self): + """Terminates the process.""" + # Don't terminate a process that we know has already died. + if self.returncode is not None: + return + try: + _winapi.TerminateProcess(self._handle, 1) + except PermissionError: + # ERROR_ACCESS_DENIED (winerror 5) is received when the + # process already died. + rc = _winapi.GetExitCodeProcess(self._handle) + if rc == _winapi.STILL_ACTIVE: + raise + self.returncode = rc + + kill = terminate + + else: + # + # POSIX methods + # + def _get_handles(self, stdin, stdout, stderr): + """Construct and return tuple with IO objects: + p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite + """ + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 + + if stdin is None: + pass + elif stdin == PIPE: + p2cread, p2cwrite = os.pipe() + elif stdin == DEVNULL: + p2cread = self._get_devnull() + elif isinstance(stdin, int): + p2cread = stdin + else: + # Assuming file-like object + p2cread = stdin.fileno() + + if stdout is None: + pass + elif stdout == PIPE: + c2pread, c2pwrite = os.pipe() + elif stdout == DEVNULL: + c2pwrite = self._get_devnull() + elif isinstance(stdout, int): + c2pwrite = stdout + else: + # Assuming file-like object + c2pwrite = stdout.fileno() + + if stderr is None: + pass + elif stderr == PIPE: + errread, errwrite = os.pipe() + elif stderr == STDOUT: + if c2pwrite != -1: + errwrite = c2pwrite + else: # child's stdout is not set, use parent's stdout + errwrite = sys.__stdout__.fileno() + elif stderr == DEVNULL: + errwrite = self._get_devnull() + elif isinstance(stderr, int): + errwrite = stderr + else: + # Assuming file-like object + errwrite = stderr.fileno() + + return (p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + + + def _posix_spawn(self, args, executable, env, restore_signals, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite): + """Execute program using os.posix_spawn().""" + if env is None: + env = os.environ + + kwargs = {} + if restore_signals: + # See _Py_RestoreSignals() in Python/pylifecycle.c + sigset = [] + for signame in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'): + signum = getattr(signal, signame, None) + if signum is not None: + sigset.append(signum) + kwargs['setsigdef'] = sigset + + file_actions = [] + for fd in (p2cwrite, c2pread, errread): + if fd != -1: + file_actions.append((os.POSIX_SPAWN_CLOSE, fd)) + for fd, fd2 in ( + (p2cread, 0), + (c2pwrite, 1), + (errwrite, 2), + ): + if fd != -1: + file_actions.append((os.POSIX_SPAWN_DUP2, fd, fd2)) + if file_actions: + kwargs['file_actions'] = file_actions + + self.pid = os.posix_spawn(executable, args, env, **kwargs) + self._child_created = True + + self._close_pipe_fds(p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + + def _execute_child(self, args, executable, preexec_fn, close_fds, + pass_fds, cwd, env, + startupinfo, creationflags, shell, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite, + restore_signals, start_new_session): + """Execute program (POSIX version)""" + + if isinstance(args, (str, bytes)): + args = [args] + elif isinstance(args, os.PathLike): + if shell: + raise TypeError('path-like args is not allowed when ' + 'shell is true') + args = [args] + else: + args = list(args) + + if shell: + # On Android the default shell is at '/system/bin/sh'. + unix_shell = ('/system/bin/sh' if + hasattr(sys, 'getandroidapilevel') else '/bin/sh') + args = [unix_shell, "-c"] + args + if executable: + args[0] = executable + + if executable is None: + executable = args[0] + + sys.audit("subprocess.Popen", executable, args, cwd, env) + + if (_USE_POSIX_SPAWN + and os.path.dirname(executable) + and preexec_fn is None + and not close_fds + and not pass_fds + and cwd is None + and (p2cread == -1 or p2cread > 2) + and (c2pwrite == -1 or c2pwrite > 2) + and (errwrite == -1 or errwrite > 2) + and not start_new_session): + self._posix_spawn(args, executable, env, restore_signals, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + return + + orig_executable = executable + + # For transferring possible exec failure from child to parent. + # Data format: "exception name:hex errno:description" + # Pickle is not used; it is complex and involves memory allocation. + errpipe_read, errpipe_write = os.pipe() + # errpipe_write must not be in the standard io 0, 1, or 2 fd range. + low_fds_to_close = [] + while errpipe_write < 3: + low_fds_to_close.append(errpipe_write) + errpipe_write = os.dup(errpipe_write) + for low_fd in low_fds_to_close: + os.close(low_fd) + try: + try: + # We must avoid complex work that could involve + # malloc or free in the child process to avoid + # potential deadlocks, thus we do all this here. + # and pass it to fork_exec() + + if env is not None: + env_list = [] + for k, v in env.items(): + k = os.fsencode(k) + if b'=' in k: + raise ValueError("illegal environment variable name") + env_list.append(k + b'=' + os.fsencode(v)) + else: + env_list = None # Use execv instead of execve. + executable = os.fsencode(executable) + if os.path.dirname(executable): + executable_list = (executable,) + else: + # This matches the behavior of os._execvpe(). + executable_list = tuple( + os.path.join(os.fsencode(dir), executable) + for dir in os.get_exec_path(env)) + fds_to_keep = set(pass_fds) + fds_to_keep.add(errpipe_write) + self.pid = _posixsubprocess.fork_exec( + args, executable_list, + close_fds, tuple(sorted(map(int, fds_to_keep))), + cwd, env_list, + p2cread, p2cwrite, c2pread, c2pwrite, + errread, errwrite, + errpipe_read, errpipe_write, + restore_signals, start_new_session, preexec_fn) + self._child_created = True + finally: + # be sure the FD is closed no matter what + os.close(errpipe_write) + + self._close_pipe_fds(p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + + # Wait for exec to fail or succeed; possibly raising an + # exception (limited in size) + errpipe_data = bytearray() + while True: + part = os.read(errpipe_read, 50000) + errpipe_data += part + if not part or len(errpipe_data) > 50000: + break + finally: + # be sure the FD is closed no matter what + os.close(errpipe_read) + + if errpipe_data: + try: + pid, sts = os.waitpid(self.pid, 0) + if pid == self.pid: + self._handle_exitstatus(sts) + else: + self.returncode = sys.maxsize + except ChildProcessError: + pass + + try: + exception_name, hex_errno, err_msg = ( + errpipe_data.split(b':', 2)) + # The encoding here should match the encoding + # written in by the subprocess implementations + # like _posixsubprocess + err_msg = err_msg.decode() + except ValueError: + exception_name = b'SubprocessError' + hex_errno = b'0' + err_msg = 'Bad exception data from child: {!r}'.format( + bytes(errpipe_data)) + child_exception_type = getattr( + builtins, exception_name.decode('ascii'), + SubprocessError) + if issubclass(child_exception_type, OSError) and hex_errno: + errno_num = int(hex_errno, 16) + child_exec_never_called = (err_msg == "noexec") + if child_exec_never_called: + err_msg = "" + # The error must be from chdir(cwd). + err_filename = cwd + else: + err_filename = orig_executable + if errno_num != 0: + err_msg = os.strerror(errno_num) + raise child_exception_type(errno_num, err_msg, err_filename) + raise child_exception_type(err_msg) + + + def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED, + _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED, + _WEXITSTATUS=os.WEXITSTATUS, _WIFSTOPPED=os.WIFSTOPPED, + _WSTOPSIG=os.WSTOPSIG): + """All callers to this function MUST hold self._waitpid_lock.""" + # This method is called (indirectly) by __del__, so it cannot + # refer to anything outside of its local scope. + if _WIFSIGNALED(sts): + self.returncode = -_WTERMSIG(sts) + elif _WIFEXITED(sts): + self.returncode = _WEXITSTATUS(sts) + elif _WIFSTOPPED(sts): + self.returncode = -_WSTOPSIG(sts) + else: + # Should never happen + raise SubprocessError("Unknown child exit status!") + + + def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid, + _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD): + """Check if child process has terminated. Returns returncode + attribute. + + This method is called by __del__, so it cannot reference anything + outside of the local scope (nor can any methods it calls). + + """ + if self.returncode is None: + if not self._waitpid_lock.acquire(False): + # Something else is busy calling waitpid. Don't allow two + # at once. We know nothing yet. + return None + try: + if self.returncode is not None: + return self.returncode # Another thread waited. + pid, sts = _waitpid(self.pid, _WNOHANG) + if pid == self.pid: + self._handle_exitstatus(sts) + except OSError as e: + if _deadstate is not None: + self.returncode = _deadstate + elif e.errno == _ECHILD: + # This happens if SIGCLD is set to be ignored or + # waiting for child processes has otherwise been + # disabled for our process. This child is dead, we + # can't get the status. + # http://bugs.python.org/issue15756 + self.returncode = 0 + finally: + self._waitpid_lock.release() + return self.returncode + + + def _try_wait(self, wait_flags): + """All callers to this function MUST hold self._waitpid_lock.""" + try: + (pid, sts) = os.waitpid(self.pid, wait_flags) + except ChildProcessError: + # This happens if SIGCLD is set to be ignored or waiting + # for child processes has otherwise been disabled for our + # process. This child is dead, we can't get the status. + pid = self.pid + sts = 0 + return (pid, sts) + + + def _wait(self, timeout): + """Internal implementation of wait() on POSIX.""" + if self.returncode is not None: + return self.returncode + + if timeout is not None: + endtime = _time() + timeout + # Enter a busy loop if we have a timeout. This busy loop was + # cribbed from Lib/threading.py in Thread.wait() at r71065. + delay = 0.0005 # 500 us -> initial delay of 1 ms + while True: + if self._waitpid_lock.acquire(False): + try: + if self.returncode is not None: + break # Another thread waited. + (pid, sts) = self._try_wait(os.WNOHANG) + assert pid == self.pid or pid == 0 + if pid == self.pid: + self._handle_exitstatus(sts) + break + finally: + self._waitpid_lock.release() + remaining = self._remaining_time(endtime) + if remaining <= 0: + raise TimeoutExpired(self.args, timeout) + delay = min(delay * 2, remaining, .05) + time.sleep(delay) + else: + while self.returncode is None: + with self._waitpid_lock: + if self.returncode is not None: + break # Another thread waited. + (pid, sts) = self._try_wait(0) + # Check the pid and loop as waitpid has been known to + # return 0 even without WNOHANG in odd situations. + # http://bugs.python.org/issue14396. + if pid == self.pid: + self._handle_exitstatus(sts) + return self.returncode + + + def _communicate(self, input, endtime, orig_timeout): + if self.stdin and not self._communication_started: + # Flush stdio buffer. This might block, if the user has + # been writing to .stdin in an uncontrolled fashion. + try: + self.stdin.flush() + except BrokenPipeError: + pass # communicate() must ignore BrokenPipeError. + if not input: + try: + self.stdin.close() + except BrokenPipeError: + pass # communicate() must ignore BrokenPipeError. + + stdout = None + stderr = None + + # Only create this mapping if we haven't already. + if not self._communication_started: + self._fileobj2output = {} + if self.stdout: + self._fileobj2output[self.stdout] = [] + if self.stderr: + self._fileobj2output[self.stderr] = [] + + if self.stdout: + stdout = self._fileobj2output[self.stdout] + if self.stderr: + stderr = self._fileobj2output[self.stderr] + + self._save_input(input) + + if self._input: + input_view = memoryview(self._input) + + with _PopenSelector() as selector: + if self.stdin and input: + selector.register(self.stdin, selectors.EVENT_WRITE) + if self.stdout and not self.stdout.closed: + selector.register(self.stdout, selectors.EVENT_READ) + if self.stderr and not self.stderr.closed: + selector.register(self.stderr, selectors.EVENT_READ) + + while selector.get_map(): + timeout = self._remaining_time(endtime) + if timeout is not None and timeout < 0: + self._check_timeout(endtime, orig_timeout, + stdout, stderr, + skip_check_and_raise=True) + raise RuntimeError( # Impossible :) + '_check_timeout(..., skip_check_and_raise=True) ' + 'failed to raise TimeoutExpired.') + + ready = selector.select(timeout) + self._check_timeout(endtime, orig_timeout, stdout, stderr) + + # XXX Rewrite these to use non-blocking I/O on the file + # objects; they are no longer using C stdio! + + for key, events in ready: + if key.fileobj is self.stdin: + chunk = input_view[self._input_offset : + self._input_offset + _PIPE_BUF] + try: + self._input_offset += os.write(key.fd, chunk) + except BrokenPipeError: + selector.unregister(key.fileobj) + key.fileobj.close() + else: + if self._input_offset >= len(self._input): + selector.unregister(key.fileobj) + key.fileobj.close() + elif key.fileobj in (self.stdout, self.stderr): + data = os.read(key.fd, 32768) + if not data: + selector.unregister(key.fileobj) + key.fileobj.close() + self._fileobj2output[key.fileobj].append(data) + + self.wait(timeout=self._remaining_time(endtime)) + + # All data exchanged. Translate lists into strings. + if stdout is not None: + stdout = b''.join(stdout) + if stderr is not None: + stderr = b''.join(stderr) + + # Translate newlines, if requested. + # This also turns bytes into strings. + if self.text_mode: + if stdout is not None: + stdout = self._translate_newlines(stdout, + self.stdout.encoding, + self.stdout.errors) + if stderr is not None: + stderr = self._translate_newlines(stderr, + self.stderr.encoding, + self.stderr.errors) + + return (stdout, stderr) + + + def _save_input(self, input): + # This method is called from the _communicate_with_*() methods + # so that if we time out while communicating, we can continue + # sending input if we retry. + if self.stdin and self._input is None: + self._input_offset = 0 + self._input = input + if input is not None and self.text_mode: + self._input = self._input.encode(self.stdin.encoding, + self.stdin.errors) + + + def send_signal(self, sig): + """Send a signal to the process.""" + # Skip signalling a process that we know has already died. + if self.returncode is None: + os.kill(self.pid, sig) + + def terminate(self): + """Terminate the process with SIGTERM + """ + self.send_signal(signal.SIGTERM) + + def kill(self): + """Kill the process with SIGKILL + """ + self.send_signal(signal.SIGKILL) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index ee644b126..175f34945 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2442,7 +2442,8 @@ def strip_python_stderr(stderr): This will typically be run on the result of the communicate() method of a subprocess.Popen object. """ - stderr = re.sub(br"\[\d+ refs, \d+ blocks\]\r?\n?", b"", stderr).strip() + # XXX RustPython TODO: bytes regexes + # stderr = re.sub(br"\[\d+ refs, \d+ blocks\]\r?\n?", b"", stderr).strip() return stderr requires_type_collecting = unittest.skipIf(hasattr(sys, 'getcounts'), diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index fba046fb3..d8cc54050 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -7,7 +7,7 @@ from test import support from test.support.script_helper import assert_python_ok -@unittest.skip("TODO: RUSTPYTHON") # Need to fix subprocess to take env +@unittest.skip("TODO: RUSTPYTHON, rustpython -X faulthandler") class TestTool(unittest.TestCase): data = """ diff --git a/tests/snippets/stdlib_subprocess.py b/tests/snippets/stdlib_subprocess.py index bbfdd88ca..96ead7658 100644 --- a/tests/snippets/stdlib_subprocess.py +++ b/tests/snippets/stdlib_subprocess.py @@ -5,7 +5,20 @@ import signal from testutils import assert_raises -p = subprocess.Popen(["echo", "test"]) +is_unix = not sys.platform.startswith("win") +if is_unix: + def echo(text): + return ["echo", text] + def sleep(secs): + return ["sleep", str(secs)] +else: + def echo(text): + return ["cmd", "/C", f"echo {text}"] + def sleep(secs): + # TODO: make work in a non-unixy environment (something with timeout.exe?) + return ["sleep", str(secs)] + +p = subprocess.Popen(echo("test")) time.sleep(0.1) @@ -14,7 +27,7 @@ assert p.returncode is None assert p.poll() == 0 assert p.returncode == 0 -p = subprocess.Popen(["sleep", "2"]) +p = subprocess.Popen(sleep(2)) assert p.poll() is None @@ -25,14 +38,13 @@ p.wait() assert p.returncode == 0 -p = subprocess.Popen(["echo", "test"], stdout=subprocess.PIPE) +p = subprocess.Popen(echo("test"), stdout=subprocess.PIPE) p.wait() -is_unix = "win" not in sys.platform or "darwin" in sys.platform assert p.stdout.read().strip() == b"test" -p = subprocess.Popen(["sleep", "2"]) +p = subprocess.Popen(sleep(2)) p.terminate() p.wait() if is_unix: @@ -40,7 +52,7 @@ if is_unix: else: assert p.returncode == 1 -p = subprocess.Popen(["sleep", "2"]) +p = subprocess.Popen(sleep(2)) p.kill() p.wait() if is_unix: @@ -48,10 +60,10 @@ if is_unix: else: assert p.returncode == 1 -p = subprocess.Popen(["echo", "test"], stdout=subprocess.PIPE) +p = subprocess.Popen(echo("test"), stdout=subprocess.PIPE) (stdout, stderr) = p.communicate() assert stdout.strip() == b"test" -p = subprocess.Popen(["sleep", "5"], stdout=subprocess.PIPE) +p = subprocess.Popen(sleep(5), stdout=subprocess.PIPE) with assert_raises(subprocess.TimeoutExpired): p.communicate(timeout=1) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 4ba6d0220..753f18663 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -73,6 +73,7 @@ crossbeam-utils = "0.7" generational-arena = "0.2" parking_lot = "0.11" thread_local = "1.0" +cfg-if = "0.1.10" ## unicode stuff unicode_names2 = "0.4" @@ -101,7 +102,6 @@ uname = "0.1.1" crc32fast = "1.2.0" adler32 = "1.0.3" gethostname = "0.2.0" -subprocess = "0.2.2" socket2 = "0.3" rustyline = "6.0" openssl = { version = "0.10", features = ["vendored"], optional = true } @@ -122,7 +122,13 @@ schannel = "0.1" [target.'cfg(windows)'.dependencies.winapi] version = "0.3" -features = ["winsock2", "handleapi", "ws2def", "std", "winbase", "wincrypt", "fileapi", "wincon"] +features = [ + "winsock2", "handleapi", "ws2def", "std", "winbase", "wincrypt", "fileapi", "processenv", + "namedpipeapi", "winnt", "processthreadsapi", "errhandlingapi", "winuser", "synchapi", +] [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen = "0.2" + +[build-dependencies] +itertools = "0.8" diff --git a/vm/build.rs b/vm/build.rs index f58285fd3..5adcc3ebf 100644 --- a/vm/build.rs +++ b/vm/build.rs @@ -1,3 +1,7 @@ +use itertools::Itertools; +use std::env; +use std::io::prelude::*; +use std::path::PathBuf; use std::process::Command; fn main() { @@ -8,6 +12,21 @@ fn main() { ); println!("cargo:rustc-env=RUSTPYTHON_GIT_TAG={}", git_tag()); println!("cargo:rustc-env=RUSTPYTHON_GIT_BRANCH={}", git_branch()); + + println!( + "cargo:rustc-env=RUSTPYTHON_TARGET_TRIPLE={}", + env::var("TARGET").unwrap() + ); + + let mut env_path = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + env_path.push("env_vars.rs"); + let mut f = std::fs::File::create(env_path).unwrap(); + write!( + f, + "hashmap! {{ {} }}", + std::env::vars_os().format_with(", ", |(k, v), f| f(&format_args!("{:?} => {:?}", k, v))) + ) + .unwrap(); } fn git_hash() -> String { diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 7de6cb29d..2d105cb73 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -183,6 +183,9 @@ impl PyDictRef { pub fn len(self) -> usize { self.entries.len() } + pub fn is_empty(&self) -> bool { + self.entries.len() == 0 + } #[pymethod(magic)] fn sizeof(self) -> usize { diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 4f82b554d..0aa7440c6 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1383,6 +1383,23 @@ where pub type PyComparisonValue = PyArithmaticValue; +#[derive(Clone)] +pub struct PySequence(Vec); + +impl PySequence { + pub fn into_vec(self) -> Vec { + self.0 + } + pub fn as_slice(&self) -> &[T] { + &self.0 + } +} +impl TryFromObject for PySequence { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + vm.extract_elements(&obj).map(Self) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 2f830ed2c..71ff0d8be 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -30,6 +30,7 @@ pub mod socket; mod string; #[cfg(feature = "rustpython-compiler")] mod symtable; +mod sysconfigdata; #[cfg(not(target_arch = "wasm32"))] mod thread; mod time_module; @@ -49,6 +50,8 @@ mod faulthandler; mod msvcrt; #[cfg(not(target_arch = "wasm32"))] mod multiprocessing; +#[cfg(unix)] +mod posixsubprocess; #[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))] mod pwd; #[cfg(not(target_arch = "wasm32"))] @@ -57,8 +60,6 @@ mod select; pub mod signal; #[cfg(all(not(target_arch = "wasm32"), feature = "ssl"))] mod ssl; -#[cfg(not(target_arch = "wasm32"))] -mod subprocess; #[cfg(windows)] mod winapi; #[cfg(windows)] @@ -95,6 +96,7 @@ pub fn get_module_inits() -> HashMap { "_imp".to_owned() => Box::new(imp::make_module), "unicodedata".to_owned() => Box::new(unicodedata::make_module), "_warnings".to_owned() => Box::new(warnings::make_module), + crate::sysmodule::sysconfigdata_name() => Box::new(sysconfigdata::make_module), }; // Insert parser related modules: @@ -129,7 +131,6 @@ pub fn get_module_inits() -> HashMap { modules.insert("select".to_owned(), Box::new(select::make_module)); #[cfg(feature = "ssl")] modules.insert("_ssl".to_owned(), Box::new(ssl::make_module)); - modules.insert("_subprocess".to_owned(), Box::new(subprocess::make_module)); modules.insert("_thread".to_owned(), Box::new(thread::make_module)); #[cfg(not(target_os = "redox"))] modules.insert("zlib".to_owned(), Box::new(zlib::make_module)); @@ -145,6 +146,14 @@ pub fn get_module_inits() -> HashMap { modules.insert("pwd".to_owned(), Box::new(pwd::make_module)); } + #[cfg(unix)] + { + modules.insert( + "_posixsubprocess".to_owned(), + Box::new(posixsubprocess::make_module), + ); + } + // Windows-only #[cfg(windows)] { diff --git a/vm/src/stdlib/msvcrt.rs b/vm/src/stdlib/msvcrt.rs index 367eff4f1..3a5e4568d 100644 --- a/vm/src/stdlib/msvcrt.rs +++ b/vm/src/stdlib/msvcrt.rs @@ -59,9 +59,22 @@ fn msvcrt_setmode(fd: i32, flags: i32, vm: &VirtualMachine) -> PyResult { } } +extern "C" { + fn _open_osfhandle(osfhandle: isize, flags: i32) -> i32; +} + +fn msvcrt_open_osfhandle(handle: isize, flags: i32, vm: &VirtualMachine) -> PyResult { + let ret = unsafe { suppress_iph!(_open_osfhandle(handle, flags)) }; + if ret == -1 { + Err(errno_err(vm)) + } else { + Ok(ret) + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; - py_module!(vm, "_msvcrt", { + py_module!(vm, "msvcrt", { "getch" => ctx.new_function(msvcrt_getch), "getwch" => ctx.new_function(msvcrt_getwch), "getche" => ctx.new_function(msvcrt_getche), @@ -69,5 +82,6 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "putch" => ctx.new_function(msvcrt_putch), "putwch" => ctx.new_function(msvcrt_putwch), "setmode" => ctx.new_function(msvcrt_setmode), + "open_osfhandle" => ctx.new_function(msvcrt_open_osfhandle), }) } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 59bccb4a3..2a1af9a38 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -28,7 +28,7 @@ use crate::exceptions::PyBaseExceptionRef; use crate::function::{IntoPyNativeFunc, OptionalArg, PyFuncArgs}; use crate::obj::objbyteinner::PyBytesLike; use crate::obj::objbytes::{PyBytes, PyBytesRef}; -use crate::obj::objdict::{PyDictRef, PyMapping}; +use crate::obj::objdict::PyDictRef; use crate::obj::objint::PyIntRef; use crate::obj::objiter; use crate::obj::objset::PySet; @@ -43,6 +43,8 @@ use crate::vm::VirtualMachine; // just to avoid unused import warnings #[cfg(unix)] +use crate::obj::objdict::PyMapping; +#[cfg(unix)] use crate::pyobject::PyIterable; #[cfg(unix)] use std::convert::TryFrom; @@ -119,12 +121,12 @@ enum OutputMode { fn output_by_mode(val: String, mode: OutputMode, vm: &VirtualMachine) -> PyObjectRef { match mode { OutputMode::String => vm.ctx.new_str(val), - OutputMode::Bytes => vm.ctx.new_bytes(val.as_bytes().to_vec()), + OutputMode::Bytes => vm.ctx.new_bytes(val.into_bytes()), } } pub struct PyPathLike { - path: ffi::OsString, + pub path: ffi::OsString, mode: OutputMode, } @@ -1036,6 +1038,19 @@ fn os_get_inheritable(fd: RawFd, vm: &VirtualMachine) -> PyResult { } } +#[cfg(unix)] +pub(crate) fn set_inheritable(fd: RawFd, inheritable: bool) -> nix::Result<()> { + use nix::fcntl; + + let flags = fcntl::FdFlag::from_bits_truncate(fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD)?); + let mut new_flags = flags; + new_flags.set(fcntl::FdFlag::FD_CLOEXEC, !inheritable); + if flags != new_flags { + fcntl::fcntl(fd, fcntl::FcntlArg::F_SETFD(new_flags))?; + } + Ok(()) +} + fn os_set_inheritable(fd: i64, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> { #[cfg(not(any(unix, windows)))] { @@ -1043,21 +1058,7 @@ fn os_set_inheritable(fd: i64, inheritable: bool, vm: &VirtualMachine) -> PyResu } #[cfg(unix)] { - let fd = fd as RawFd; - let _set_flag = || { - use nix::fcntl::fcntl; - use nix::fcntl::FcntlArg; - use nix::fcntl::FdFlag; - - let flags = FdFlag::from_bits_truncate(fcntl(fd, FcntlArg::F_GETFD)?); - let mut new_flags = flags; - new_flags.set(FdFlag::from_bits_truncate(libc::FD_CLOEXEC), !inheritable); - if flags != new_flags { - fcntl(fd, FcntlArg::F_SETFD(new_flags))?; - } - Ok(()) - }; - _set_flag().or_else(|err| Err(convert_nix_error(vm, err))) + set_inheritable(fd as RawFd, inheritable).map_err(|err| convert_nix_error(vm, err)) } #[cfg(windows)] { @@ -1860,6 +1861,12 @@ fn os_kill(pid: i32, sig: isize, vm: &VirtualMachine) -> PyResult<()> { } } +fn os_strerror(e: i32) -> String { + unsafe { ffi::CStr::from_ptr(libc::strerror(e)) } + .to_string_lossy() + .into_owned() +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -1970,6 +1977,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "set_inheritable" => ctx.new_function(os_set_inheritable), "link" => ctx.new_function(os_link), "kill" => ctx.new_function(os_kill), + "strerror" => ctx.new_function(os_strerror), "O_RDONLY" => ctx.new_int(libc::O_RDONLY), "O_WRONLY" => ctx.new_int(libc::O_WRONLY), diff --git a/vm/src/stdlib/posixsubprocess.rs b/vm/src/stdlib/posixsubprocess.rs new file mode 100644 index 000000000..42cb887f2 --- /dev/null +++ b/vm/src/stdlib/posixsubprocess.rs @@ -0,0 +1,204 @@ +use nix::{dir, errno::Errno, fcntl, unistd}; +use std::convert::Infallible as Never; +use std::ffi::{CStr, CString}; +use std::io::{self, prelude::*}; +use std::os::unix::ffi::OsStringExt; + +use crate::pyobject::{PyObjectRef, PyResult, PySequence, TryFromObject}; +use crate::VirtualMachine; + +use super::os::{convert_nix_error, set_inheritable, PyPathLike}; + +macro_rules! gen_args { + ($($field:ident: $t:ty),*$(,)?) => { + #[derive(FromArgs)] + struct ForkExecArgs { + $(#[pyarg(positional_only)] $field: $t,)* + } + }; +} + +struct CStrPathLike { + s: CString, +} +impl TryFromObject for CStrPathLike { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + let s = PyPathLike::try_from_object(vm, obj)?.path.into_vec(); + let s = CString::new(s) + .map_err(|_| vm.new_value_error("embedded null character".to_owned()))?; + Ok(CStrPathLike { s }) + } +} + +gen_args! { + args: PySequence /* list */, exec_list: PySequence /* list */, + close_fds: bool, fds_to_keep: PySequence, + cwd: Option, env_list: Option>, + p2cread: i32, p2cwrite: i32, c2pread: i32, c2pwrite: i32, + errread: i32, errwrite: i32, errpipe_read: i32, errpipe_write: i32, + restore_signals: bool, call_setsid: bool, preexec_fn: Option, +} + +fn _posixsubprocess_fork_exec(args: ForkExecArgs, vm: &VirtualMachine) -> PyResult { + if args.preexec_fn.is_some() { + return Err(vm.new_not_implemented_error("preexec_fn not supported yet".to_owned())); + } + let cstrs_to_ptrs = |cstrs: &[CStrPathLike]| { + cstrs + .iter() + .map(|s| s.s.as_ptr()) + .chain(std::iter::once(std::ptr::null())) + .collect::>() + }; + let argv = cstrs_to_ptrs(args.args.as_slice()); + let argv = &argv; + let envp = args.env_list.as_ref().map(|s| cstrs_to_ptrs(s.as_slice())); + let envp = envp.as_deref(); + match nix::unistd::fork().map_err(|e| convert_nix_error(vm, e))? { + nix::unistd::ForkResult::Child => exec(&args, ProcArgs { argv, envp }), + nix::unistd::ForkResult::Parent { child } => Ok(child.as_raw()), + } +} + +// can't reallocate inside of exec(), so we reallocate prior to fork() and pass this along +struct ProcArgs<'a> { + argv: &'a [*const libc::c_char], + envp: Option<&'a [*const libc::c_char]>, +} + +fn exec(args: &ForkExecArgs, procargs: ProcArgs) -> ! { + match exec_inner(args, procargs) { + Ok(x) => match x {}, + Err(e) => { + let e = e.as_errno().expect("got a non-errno nix error"); + let buf: &mut [u8] = &mut [0; 256]; + let mut cur = io::Cursor::new(&mut *buf); + // TODO: check if reached preexec, if not then have "noexec" after + let _ = write!(cur, "OSError:{}:", e as i32); + let pos = cur.position(); + let _ = unistd::write(args.errpipe_write, &buf[..pos as usize]); + std::process::exit(255) + } + } +} + +fn exec_inner(args: &ForkExecArgs, procargs: ProcArgs) -> nix::Result { + for &fd in args.fds_to_keep.as_slice() { + if fd != args.errpipe_write { + set_inheritable(fd, true)? + } + } + + for &fd in &[args.p2cwrite, args.c2pread, args.errread] { + if fd != -1 { + unistd::close(fd)?; + } + } + unistd::close(args.errpipe_read)?; + + let c2pwrite = if args.c2pwrite == 0 { + let fd = unistd::dup(args.c2pwrite)?; + set_inheritable(fd, true)?; + fd + } else { + args.c2pwrite + }; + + let mut errwrite = args.errwrite; + while errwrite == 0 || errwrite == 1 { + errwrite = unistd::dup(errwrite)?; + set_inheritable(errwrite, true)?; + } + + let dup_into_stdio = |fd, io_fd| { + if fd == io_fd { + set_inheritable(fd, true) + } else if fd != -1 { + unistd::dup2(fd, io_fd).map(drop) + } else { + Ok(()) + } + }; + dup_into_stdio(args.p2cread, 0)?; + dup_into_stdio(c2pwrite, 1)?; + dup_into_stdio(errwrite, 2)?; + + if let Some(ref cwd) = args.cwd { + unistd::chdir(cwd.s.as_c_str())? + } + + if args.restore_signals { + // TODO: restore signals SIGPIPE, SIGXFZ, SIGXFSZ to SIG_DFL + } + + if args.call_setsid { + unistd::setsid()?; + } + + if args.close_fds { + close_fds(3, args.fds_to_keep.as_slice())?; + } + + let mut first_err = None; + for exec in args.exec_list.as_slice() { + if let Some(envp) = procargs.envp { + unsafe { libc::execve(exec.s.as_ptr(), procargs.argv.as_ptr(), envp.as_ptr()) }; + } else { + unsafe { libc::execv(exec.s.as_ptr(), procargs.argv.as_ptr()) }; + } + let e = Errno::last(); + if e != Errno::ENOENT && e != Errno::ENOTDIR && first_err.is_none() { + first_err = Some(e) + } + } + Err(first_err.unwrap_or_else(Errno::last).into()) +} + +fn close_fds(above: i32, keep: &[i32]) -> nix::Result<()> { + // TODO: close fds by brute force if readdir doesn't work: + // https://github.com/python/cpython/blob/3.8/Modules/_posixsubprocess.c#L220 + let path = unsafe { CStr::from_bytes_with_nul_unchecked(FD_DIR_NAME) }; + let mut dir = dir::Dir::open( + path, + fcntl::OFlag::O_RDONLY | fcntl::OFlag::O_DIRECTORY, + nix::sys::stat::Mode::empty(), + )?; + for e in dir.iter() { + if let Some(fd) = pos_int_from_ascii(e?.file_name()) { + if fd > above && !keep.contains(&fd) { + unistd::close(fd)? + } + } + } + Ok(()) +} + +#[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "macos", +))] +const FD_DIR_NAME: &[u8] = b"/dev/fd\0"; + +#[cfg(target_os = "linux")] +const FD_DIR_NAME: &[u8] = b"/proc/self/fd\0"; + +fn pos_int_from_ascii(name: &CStr) -> Option { + let mut num = 0; + for c in name.to_bytes() { + if !c.is_ascii_digit() { + return None; + } + num = num * 10 + i32::from(c - b'0') + } + Some(num) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + py_module!(vm, "_posixsubprocess", { + "fork_exec" => named_function!(ctx, _posixsubprocess, fork_exec), + }) +} diff --git a/vm/src/stdlib/subprocess.rs b/vm/src/stdlib/subprocess.rs deleted file mode 100644 index 96714398c..000000000 --- a/vm/src/stdlib/subprocess.rs +++ /dev/null @@ -1,295 +0,0 @@ -use std::ffi::OsString; -use std::fs::File; -use std::io::ErrorKind; -use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard}; -use std::time::Duration; - -use crate::function::OptionalArg; -use crate::obj::objbytes::PyBytesRef; -use crate::obj::objlist::PyListRef; -use crate::obj::objstr::{self, PyStringRef}; -use crate::obj::objtype::PyClassRef; -use crate::pyobject::{Either, IntoPyObject, PyObjectRef, PyRef, PyResult, PyValue}; -use crate::stdlib::io::io_open; -use crate::stdlib::os::{convert_io_error, raw_file_number, rust_file}; -use crate::vm::VirtualMachine; - -#[derive(Debug)] -struct Popen { - process: RwLock, - args: PyObjectRef, -} - -// Remove once https://github.com/hniksic/rust-subprocess/issues/42 is resolved -#[cfg(windows)] -unsafe impl Sync for Popen {} - -impl PyValue for Popen { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.class("_subprocess", "Popen") - } -} - -type PopenRef = PyRef; - -#[derive(FromArgs)] -#[allow(dead_code)] -struct PopenArgs { - #[pyarg(positional_only)] - args: Either, - #[pyarg(positional_or_keyword, default = "None")] - stdin: Option, - #[pyarg(positional_or_keyword, default = "None")] - stdout: Option, - #[pyarg(positional_or_keyword, default = "None")] - stderr: Option, - #[pyarg(positional_or_keyword, default = "None")] - close_fds: Option, // TODO: use these unused options - #[pyarg(positional_or_keyword, default = "None")] - cwd: Option, - #[pyarg(positional_or_keyword, default = "None")] - start_new_session: Option, -} - -#[derive(FromArgs)] -struct PopenWaitArgs { - #[pyarg(positional_or_keyword, default = "None")] - timeout: Option, -} - -impl IntoPyObject for subprocess::ExitStatus { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - let status: i32 = match self { - subprocess::ExitStatus::Exited(status) => status as i32, - subprocess::ExitStatus::Signaled(status) => -i32::from(status), - subprocess::ExitStatus::Other(status) => status as i32, - _ => return Err(vm.new_os_error("Unknown exist status".to_owned())), - }; - Ok(vm.new_int(status)) - } -} - -#[cfg(windows)] -const NULL_DEVICE: &str = "nul"; -#[cfg(unix)] -const NULL_DEVICE: &str = "/dev/null"; - -fn convert_redirection(arg: Option, vm: &VirtualMachine) -> PyResult { - match arg { - Some(fd) => match fd { - -1 => Ok(subprocess::Redirection::Pipe), - -2 => Ok(subprocess::Redirection::Merge), - -3 => Ok(subprocess::Redirection::File( - File::open(NULL_DEVICE).unwrap(), - )), - fd => { - if fd < 0 { - Err(vm.new_value_error(format!("Invalid fd: {}", fd))) - } else { - Ok(subprocess::Redirection::File(rust_file(fd))) - } - } - }, - None => Ok(subprocess::Redirection::None), - } -} - -fn convert_to_file_io(file: &Option, mode: &str, vm: &VirtualMachine) -> PyResult { - match file { - Some(ref stdin) => io_open( - vm.new_int(raw_file_number(stdin.try_clone().unwrap())), - Some(mode), - Default::default(), - vm, - ), - None => Ok(vm.get_none()), - } -} - -impl PopenRef { - fn borrow_process(&self) -> RwLockReadGuard<'_, subprocess::Popen> { - self.process.read().unwrap() - } - - fn borrow_process_mut(&self) -> RwLockWriteGuard<'_, subprocess::Popen> { - self.process.write().unwrap() - } - - fn new(cls: PyClassRef, args: PopenArgs, vm: &VirtualMachine) -> PyResult { - let stdin = convert_redirection(args.stdin, vm)?; - let stdout = convert_redirection(args.stdout, vm)?; - let stderr = convert_redirection(args.stderr, vm)?; - let command_list = match &args.args { - Either::A(command) => vec![command.as_str().to_owned()], - Either::B(command_list) => command_list - .borrow_elements() - .iter() - .map(|x| objstr::clone_value(x)) - .collect(), - }; - let cwd = args.cwd.map(|x| OsString::from(x.as_str())); - - let process = subprocess::Popen::create( - &command_list, - subprocess::PopenConfig { - stdin, - stdout, - stderr, - cwd, - ..Default::default() - }, - ) - .map_err(|s| vm.new_os_error(format!("Could not start program: {}", s)))?; - - Popen { - process: RwLock::new(process), - args: args.args.into_object(), - } - .into_ref_with_type(vm, cls) - } - - fn poll(self) -> Option { - self.borrow_process_mut().poll() - } - - fn return_code(self) -> Option { - self.borrow_process().exit_status() - } - - fn wait(self, args: PopenWaitArgs, vm: &VirtualMachine) -> PyResult { - let timeout = match args.timeout { - Some(timeout) => self - .borrow_process_mut() - .wait_timeout(Duration::new(timeout, 0)), - None => self.borrow_process_mut().wait().map(Some), - } - .map_err(|s| vm.new_os_error(format!("Could not start program: {}", s)))?; - if let Some(exit) = timeout { - use subprocess::ExitStatus::*; - Ok(match exit { - Exited(i) => i.into(), - Signaled(s) => -i64::from(s), - _ => unreachable!("should not occur in normal operation"), - }) - } else { - let timeout_expired = vm.try_class("_subprocess", "TimeoutExpired")?; - Err(vm.new_exception_msg(timeout_expired, "Timeout".to_owned())) - } - } - - fn stdin(self, vm: &VirtualMachine) -> PyResult { - convert_to_file_io(&self.borrow_process().stdin, "wb", vm) - } - - fn stdout(self, vm: &VirtualMachine) -> PyResult { - convert_to_file_io(&self.borrow_process().stdout, "rb", vm) - } - - fn stderr(self, vm: &VirtualMachine) -> PyResult { - convert_to_file_io(&self.borrow_process().stderr, "rb", vm) - } - - fn terminate(self, vm: &VirtualMachine) -> PyResult<()> { - self.borrow_process_mut() - .terminate() - .map_err(|err| convert_io_error(vm, err)) - } - - fn kill(self, vm: &VirtualMachine) -> PyResult<()> { - self.borrow_process_mut() - .kill() - .map_err(|err| convert_io_error(vm, err)) - } - - #[allow(clippy::type_complexity)] - fn communicate( - self, - args: PopenCommunicateArgs, - vm: &VirtualMachine, - ) -> PyResult<(Option>, Option>)> { - let bytes = match args.input { - OptionalArg::Present(ref bytes) => Some(bytes.get_value().to_vec()), - OptionalArg::Missing => None, - }; - let mut communicator = self.borrow_process_mut().communicate_start(bytes); - if let OptionalArg::Present(timeout) = args.timeout { - communicator = communicator.limit_time(Duration::new(timeout, 0)); - } - communicator.read().map_err(|err| { - if err.error.kind() == ErrorKind::TimedOut { - let timeout_expired = vm.try_class("_subprocess", "TimeoutExpired").unwrap(); - vm.new_exception_msg(timeout_expired, "Timeout".to_owned()) - } else { - convert_io_error(vm, err.error) - } - }) - } - - fn pid(self) -> Option { - self.borrow_process().pid() - } - - fn enter(self) -> Self { - self - } - - fn exit( - self, - _exception_type: PyObjectRef, - _exception_value: PyObjectRef, - _traceback: PyObjectRef, - ) { - let mut process = self.borrow_process_mut(); - process.stdout.take(); - process.stdin.take(); - process.stderr.take(); - } - - fn args(self) -> PyObjectRef { - self.args.clone() - } -} - -#[derive(FromArgs)] -#[allow(dead_code)] -struct PopenCommunicateArgs { - #[pyarg(positional_or_keyword, optional = true)] - input: OptionalArg, - #[pyarg(positional_or_keyword, optional = true)] - timeout: OptionalArg, -} - -pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { - let ctx = &vm.ctx; - - let subprocess_error = ctx.new_class("SubprocessError", ctx.exceptions.exception_type.clone()); - let timeout_expired = ctx.new_class("TimeoutExpired", subprocess_error.clone()); - let called_process_error = ctx.new_class("CalledProcessError", subprocess_error.clone()); - - let popen = py_class!(ctx, "Popen", ctx.object(), { - (slot new) => PopenRef::new, - "poll" => ctx.new_method(PopenRef::poll), - "returncode" => ctx.new_readonly_getset("returncode", PopenRef::return_code), - "wait" => ctx.new_method(PopenRef::wait), - "stdin" => ctx.new_readonly_getset("stdin", PopenRef::stdin), - "stdout" => ctx.new_readonly_getset("stdout", PopenRef::stdout), - "stderr" => ctx.new_readonly_getset("stderr", PopenRef::stderr), - "terminate" => ctx.new_method(PopenRef::terminate), - "kill" => ctx.new_method(PopenRef::kill), - "communicate" => ctx.new_method(PopenRef::communicate), - "pid" => ctx.new_readonly_getset("pid", PopenRef::pid), - "__enter__" => ctx.new_method(PopenRef::enter), - "__exit__" => ctx.new_method(PopenRef::exit), - "args" => ctx.new_readonly_getset("args", PopenRef::args), - }); - - py_module!(vm, "_subprocess", { - "Popen" => popen, - "SubprocessError" => subprocess_error, - "TimeoutExpired" => timeout_expired, - "CalledProcessError" => called_process_error, - "PIPE" => ctx.new_int(-1), - "STDOUT" => ctx.new_int(-2), - "DEVNULL" => ctx.new_int(-3), - }) -} diff --git a/vm/src/stdlib/sysconfigdata.rs b/vm/src/stdlib/sysconfigdata.rs new file mode 100644 index 000000000..ae531bfe2 --- /dev/null +++ b/vm/src/stdlib/sysconfigdata.rs @@ -0,0 +1,16 @@ +use crate::pyobject::{ItemProtocol, PyObjectRef}; +use crate::VirtualMachine; + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let vars = vm.ctx.new_dict(); + macro_rules! hashmap { + ($($key:literal => $value:literal),*) => {{ + $(vars.set_item($key, vm.new_str($value.to_owned()), vm).unwrap();)* + }}; + } + include!(concat!(env!("OUT_DIR"), "/env_vars.rs")); + + py_module!(vm, "_sysconfigdata", { + "build_time_vars" => vars, + }) +} diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index 62b8dc94c..11956445a 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -1,30 +1,393 @@ #![allow(non_snake_case)] +use std::ptr::{null, null_mut}; + use winapi::shared::winerror; use winapi::um::winnt::HANDLE; -use winapi::um::{handleapi, winbase}; +use winapi::um::{ + fileapi, handleapi, namedpipeapi, processenv, processthreadsapi, synchapi, winbase, winnt, + winuser, +}; -use super::os; -use crate::pyobject::{PyObjectRef, PyResult}; +use super::os::errno_err; +use crate::function::OptionalArg; +use crate::obj::objdict::{PyDictRef, PyMapping}; +use crate::obj::objstr::PyStringRef; +use crate::pyobject::{PyObjectRef, PyResult, PySequence, TryFromObject}; use crate::VirtualMachine; -fn winapi_CloseHandle(handle: usize, vm: &VirtualMachine) -> PyResult<()> { - let res = unsafe { handleapi::CloseHandle(handle as HANDLE) }; - if res == 0 { - Err(os::errno_err(vm)) - } else { - Ok(()) +fn GetLastError() -> u32 { + unsafe { winapi::um::errhandlingapi::GetLastError() } +} + +fn husize(h: HANDLE) -> usize { + h as usize +} + +trait Convertable { + fn is_err(&self) -> bool; +} + +impl Convertable for HANDLE { + fn is_err(&self) -> bool { + *self == handleapi::INVALID_HANDLE_VALUE } } +impl Convertable for i32 { + fn is_err(&self) -> bool { + *self == 0 + } +} + +fn cvt(vm: &VirtualMachine, res: T) -> PyResult { + if res.is_err() { + Err(errno_err(vm)) + } else { + Ok(res) + } +} + +fn _winapi_CloseHandle(handle: usize, vm: &VirtualMachine) -> PyResult<()> { + cvt(vm, unsafe { handleapi::CloseHandle(handle as HANDLE) }).map(drop) +} + +fn _winapi_GetStdHandle(std_handle: u32, vm: &VirtualMachine) -> PyResult { + cvt(vm, unsafe { processenv::GetStdHandle(std_handle) }).map(husize) +} + +fn _winapi_CreatePipe( + _pipe_attrs: PyObjectRef, + size: u32, + vm: &VirtualMachine, +) -> PyResult<(usize, usize)> { + let mut read = null_mut(); + let mut write = null_mut(); + cvt(vm, unsafe { + namedpipeapi::CreatePipe(&mut read, &mut write, null_mut(), size) + })?; + Ok((read as usize, write as usize)) +} + +fn _winapi_DuplicateHandle( + (src_process, src): (usize, usize), + target_process: usize, + access: u32, + inherit: i32, + options: OptionalArg, + vm: &VirtualMachine, +) -> PyResult { + let mut target = null_mut(); + cvt(vm, unsafe { + handleapi::DuplicateHandle( + src_process as _, + src as _, + target_process as _, + &mut target, + access, + inherit, + options.unwrap_or(0), + ) + })?; + Ok(target as usize) +} + +fn _winapi_GetCurrentProcess() -> usize { + unsafe { processthreadsapi::GetCurrentProcess() as usize } +} + +fn _winapi_GetFileType(h: usize, vm: &VirtualMachine) -> PyResult { + let ret = unsafe { fileapi::GetFileType(h as _) }; + if ret == 0 && GetLastError() != 0 { + Err(errno_err(vm)) + } else { + Ok(ret) + } +} + +#[derive(FromArgs)] +struct CreateProcessArgs { + #[pyarg(positional_only)] + name: Option, + #[pyarg(positional_only)] + command_line: Option, + #[pyarg(positional_only)] + _proc_attrs: PyObjectRef, + #[pyarg(positional_only)] + _thread_attrs: PyObjectRef, + #[pyarg(positional_only)] + inherit_handles: i32, + #[pyarg(positional_only)] + creation_flags: u32, + #[pyarg(positional_only)] + env_mapping: Option, + #[pyarg(positional_only)] + current_dir: Option, + #[pyarg(positional_only)] + startup_info: PyObjectRef, +} + +fn _winapi_CreateProcess( + args: CreateProcessArgs, + vm: &VirtualMachine, +) -> PyResult<(usize, usize, u32, u32)> { + use winbase::STARTUPINFOEXW; + let mut si: STARTUPINFOEXW = unsafe { std::mem::zeroed() }; + si.StartupInfo.cb = std::mem::size_of::() as _; + + macro_rules! si_attr { + ($attr:ident, $t:ty) => {{ + si.StartupInfo.$attr = >::try_from_object( + vm, + vm.get_attribute(args.startup_info.clone(), stringify!($attr))?, + )? + .unwrap_or(0) as _ + }}; + ($attr:ident) => {{ + si.StartupInfo.$attr = >::try_from_object( + vm, + vm.get_attribute(args.startup_info.clone(), stringify!($attr))?, + )? + .unwrap_or(0) + }}; + } + si_attr!(dwFlags); + si_attr!(wShowWindow); + si_attr!(hStdInput, usize); + si_attr!(hStdOutput, usize); + si_attr!(hStdError, usize); + + let mut env = args + .env_mapping + .map(|m| getenvironment(m.into_dict(), vm)) + .transpose()?; + let env = env.as_mut().map_or_else(null_mut, |v| v.as_mut_ptr()); + + let mut attrlist = getattributelist( + vm.get_attribute(args.startup_info.clone(), "lpAttributeList")?, + vm, + )?; + si.lpAttributeList = attrlist + .as_mut() + .map_or_else(null_mut, |l| l.attrlist.as_mut_ptr() as _); + + let wstr = |s: PyStringRef| { + if s.as_str().contains('\0') { + Err(vm.new_value_error("embedded null character".to_owned())) + } else { + Ok(s.as_str() + .encode_utf16() + .chain(std::iter::once(0)) + .collect::>()) + } + }; + + let app_name = args.name.map(wstr).transpose()?; + let app_name = app_name.as_ref().map_or_else(null, |w| w.as_ptr()); + + let mut command_line = args.command_line.map(wstr).transpose()?; + let command_line = command_line + .as_mut() + .map_or_else(null_mut, |w| w.as_mut_ptr()); + + let mut current_dir = args.current_dir.map(wstr).transpose()?; + let current_dir = current_dir + .as_mut() + .map_or_else(null_mut, |w| w.as_mut_ptr()); + + let mut procinfo = unsafe { std::mem::zeroed() }; + let ret = unsafe { + processthreadsapi::CreateProcessW( + app_name, + command_line, + null_mut(), + null_mut(), + args.inherit_handles, + args.creation_flags + | winbase::EXTENDED_STARTUPINFO_PRESENT + | winbase::CREATE_UNICODE_ENVIRONMENT, + env as _, + current_dir, + &mut si as *mut STARTUPINFOEXW as _, + &mut procinfo, + ) + }; + + if ret == 0 { + return Err(errno_err(vm)); + } + + Ok(( + procinfo.hProcess as usize, + procinfo.hThread as usize, + procinfo.dwProcessId, + procinfo.dwThreadId, + )) +} + +fn getenvironment(env: PyDictRef, vm: &VirtualMachine) -> PyResult> { + let mut out = vec![]; + for (k, v) in env { + let k = PyStringRef::try_from_object(vm, k)?; + let k = k.as_str(); + let v = PyStringRef::try_from_object(vm, v)?; + let v = v.as_str(); + if k.contains('\0') || v.contains('\0') { + return Err(vm.new_value_error("embedded null character".to_owned())); + } + if k.len() == 0 || k[1..].contains('=') { + return Err(vm.new_value_error("illegal environment variable name".to_owned())); + } + out.extend(k.encode_utf16()); + out.push(b'=' as u16); + out.extend(v.encode_utf16()); + out.push(b'\0' as u16); + } + out.push(b'\0' as u16); + Ok(out) +} + +struct AttrList { + handlelist: Option>, + attrlist: Vec, +} +impl Drop for AttrList { + fn drop(&mut self) { + unsafe { + processthreadsapi::DeleteProcThreadAttributeList(self.attrlist.as_mut_ptr() as _) + }; + } +} + +fn getattributelist(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult> { + >::try_from_object(vm, obj)? + .map(|d| { + let d = d.into_dict(); + let handlelist = d + .get_item_option("handle_list", vm)? + .and_then(|obj| { + >>::try_from_object(vm, obj) + .and_then(|s| match s { + Some(s) if !s.as_slice().is_empty() => Ok(Some(s.into_vec())), + _ => Ok(None), + }) + .transpose() + }) + .transpose()?; + let attr_count = handlelist.is_some() as u32; + let mut size = 0; + let ret = unsafe { + processthreadsapi::InitializeProcThreadAttributeList( + null_mut(), + attr_count, + 0, + &mut size, + ) + }; + if ret != 0 || GetLastError() != winerror::ERROR_INSUFFICIENT_BUFFER { + return Err(errno_err(vm)); + } + let mut attrlist = vec![0u8; size]; + let ret = unsafe { + processthreadsapi::InitializeProcThreadAttributeList( + attrlist.as_mut_ptr() as _, + attr_count, + 0, + &mut size, + ) + }; + if ret == 0 { + return Err(errno_err(vm)); + } + let mut attrs = AttrList { + handlelist, + attrlist, + }; + if let Some(ref mut handlelist) = attrs.handlelist { + let ret = unsafe { + processthreadsapi::UpdateProcThreadAttribute( + attrs.attrlist.as_mut_ptr() as _, + 0, + (2 & 0xffff) | 0x20000, // PROC_THREAD_ATTRIBUTE_HANDLE_LIST + handlelist.as_mut_ptr() as _, + (handlelist.len() * std::mem::size_of::()) as _, + null_mut(), + null_mut(), + ) + }; + if ret == 0 { + return Err(errno_err(vm)); + } + } + Ok(attrs) + }) + .transpose() +} + +fn _winapi_WaitForSingleObject(h: usize, ms: u32, vm: &VirtualMachine) -> PyResult { + let ret = unsafe { synchapi::WaitForSingleObject(h as _, ms) }; + if ret == winbase::WAIT_FAILED { + Err(errno_err(vm)) + } else { + Ok(ret) + } +} + +fn _winapi_GetExitCodeProcess(h: usize, vm: &VirtualMachine) -> PyResult { + let mut ec = 0; + cvt(vm, unsafe { + processthreadsapi::GetExitCodeProcess(h as _, &mut ec) + })?; + Ok(ec) +} + +fn _winapi_TerminateProcess(h: usize, exit_code: u32, vm: &VirtualMachine) -> PyResult<()> { + cvt(vm, unsafe { + processthreadsapi::TerminateProcess(h as _, exit_code) + }) + .map(drop) +} pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; py_module!(vm, "_winapi", { - "CloseHandle" => ctx.new_function(winapi_CloseHandle), + "CloseHandle" => named_function!(ctx, _winapi, CloseHandle), + "GetStdHandle" => named_function!(ctx, _winapi, GetStdHandle), + "CreatePipe" => named_function!(ctx, _winapi, CreatePipe), + "DuplicateHandle" => named_function!(ctx, _winapi, DuplicateHandle), + "GetCurrentProcess" => named_function!(ctx, _winapi, GetCurrentProcess), + "CreateProcess" => named_function!(ctx, _winapi, CreateProcess), + "WaitForSingleObject" => named_function!(ctx, _winapi, WaitForSingleObject), + "GetExitCodeProcess" => named_function!(ctx, _winapi, GetExitCodeProcess), + "TerminateProcess" => named_function!(ctx, _winapi, TerminateProcess), + "WAIT_OBJECT_0" => ctx.new_int(winbase::WAIT_OBJECT_0), "WAIT_ABANDONED" => ctx.new_int(winbase::WAIT_ABANDONED), "WAIT_ABANDONED_0" => ctx.new_int(winbase::WAIT_ABANDONED_0), "WAIT_TIMEOUT" => ctx.new_int(winerror::WAIT_TIMEOUT), "INFINITE" => ctx.new_int(winbase::INFINITE), + "CREATE_NEW_CONSOLE" => ctx.new_int(winbase::CREATE_NEW_CONSOLE), + "CREATE_NEW_PROCESS_GROUP" => ctx.new_int(winbase::CREATE_NEW_PROCESS_GROUP), + "STD_INPUT_HANDLE" => ctx.new_int(winbase::STD_INPUT_HANDLE), + "STD_OUTPUT_HANDLE" => ctx.new_int(winbase::STD_OUTPUT_HANDLE), + "STD_ERROR_HANDLE" => ctx.new_int(winbase::STD_ERROR_HANDLE), + "SW_HIDE" => ctx.new_int(winuser::SW_HIDE), + "STARTF_USESTDHANDLES" => ctx.new_int(winbase::STARTF_USESTDHANDLES), + "STARTF_USESHOWWINDOW" => ctx.new_int(winbase::STARTF_USESHOWWINDOW), + "ABOVE_NORMAL_PRIORITY_CLASS" => ctx.new_int(winbase::ABOVE_NORMAL_PRIORITY_CLASS), + "BELOW_NORMAL_PRIORITY_CLASS" => ctx.new_int(winbase::BELOW_NORMAL_PRIORITY_CLASS), + "HIGH_PRIORITY_CLASS" => ctx.new_int(winbase::HIGH_PRIORITY_CLASS), + "IDLE_PRIORITY_CLASS" => ctx.new_int(winbase::IDLE_PRIORITY_CLASS), + "NORMAL_PRIORITY_CLASS" => ctx.new_int(winbase::NORMAL_PRIORITY_CLASS), + "REALTIME_PRIORITY_CLASS" => ctx.new_int(winbase::REALTIME_PRIORITY_CLASS), + "CREATE_NO_WINDOW" => ctx.new_int(winbase::CREATE_NO_WINDOW), + "DETACHED_PROCESS" => ctx.new_int(winbase::DETACHED_PROCESS), + "CREATE_DEFAULT_ERROR_MODE" => ctx.new_int(winbase::CREATE_DEFAULT_ERROR_MODE), + "CREATE_BREAKAWAY_FROM_JOB" => ctx.new_int(winbase::CREATE_BREAKAWAY_FROM_JOB), + "DUPLICATE_SAME_ACCESS" => ctx.new_int(winnt::DUPLICATE_SAME_ACCESS), + "FILE_TYPE_CHAR" => ctx.new_int(winbase::FILE_TYPE_CHAR), + "FILE_TYPE_DISK" => ctx.new_int(winbase::FILE_TYPE_DISK), + "FILE_TYPE_PIPE" => ctx.new_int(winbase::FILE_TYPE_PIPE), + "FILE_TYPE_REMOTE" => ctx.new_int(winbase::FILE_TYPE_REMOTE), + "FILE_TYPE_UNKNOWN" => ctx.new_int(winbase::FILE_TYPE_UNKNOWN), }) } diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 007d7ef8e..0b11a5619 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -222,6 +222,32 @@ fn sys_displayhook(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { Ok(()) } +const PLATFORM: &str = { + cfg_if::cfg_if! { + if #[cfg(any(target_os = "linux", target_os = "android"))] { + // Android is linux as well. see https://bugs.python.org/issue32637 + "linux" + } else if #[cfg(target_os = "macos")] { + "darwin" + } else if #[cfg(windows)] { + "win32" + } else { + "unknown" + } + } +}; + +const ABIFLAGS: &str = ""; + +// not the same as CPython (e.g. rust's x86_x64-unknown-linux-gnu is just x86_64-linux-gnu) +// but hopefully that's just an implementation detail? TODO: copy CPython's multiarch exactly, +// https://github.com/python/cpython/blob/3.8/configure.ac#L725 +const MULTIARCH: &str = env!("RUSTPYTHON_TARGET_TRIPLE"); + +pub fn sysconfigdata_name() -> String { + format!("_sysconfigdata_{}_{}_{}", ABIFLAGS, PLATFORM, MULTIARCH) +} + pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectRef) { let ctx = &vm.ctx; @@ -244,6 +270,7 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectR let implementation = py_namespace!(vm, { "name" => ctx.new_str("rustpython".to_owned()), "cache_tag" => ctx.new_str("rustpython-01".to_owned()), + "_multiarch" => ctx.new_str(MULTIARCH.to_owned()), }); let path = ctx.new_list( @@ -255,19 +282,6 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectR .collect(), ); - let platform = if cfg!(target_os = "linux") { - "linux".to_owned() - } else if cfg!(target_os = "macos") { - "darwin".to_owned() - } else if cfg!(target_os = "windows") { - "win32".to_owned() - } else if cfg!(target_os = "android") { - // Linux as well. see https://bugs.python.org/issue32637 - "linux".to_owned() - } else { - "unknown".to_owned() - }; - let framework = "".to_owned(); // https://doc.rust-lang.org/reference/conditional-compilation.html#target_endian @@ -395,7 +409,7 @@ settrace() -- set the global debug tracing function "_getframe" => ctx.new_function(getframe), "modules" => modules.clone(), "warnoptions" => ctx.new_list(vec![]), - "platform" => ctx.new_str(platform), + "platform" => ctx.new_str(PLATFORM.to_owned()), "_framework" => ctx.new_str(framework), "meta_path" => ctx.new_list(vec![]), "path_hooks" => ctx.new_list(vec![]), @@ -414,7 +428,7 @@ settrace() -- set the global debug tracing function "exec_prefix" => ctx.new_str(exec_prefix.to_owned()), "base_exec_prefix" => ctx.new_str(base_exec_prefix.to_owned()), "exit" => ctx.new_function(sys_exit), - "abiflags" => ctx.new_str("".to_owned()), + "abiflags" => ctx.new_str(ABIFLAGS.to_owned()), "audit" => ctx.new_function(sys_audit), "displayhook" => ctx.new_function(sys_displayhook), "__displayhook__" => ctx.new_function(sys_displayhook),