Add _posixsubprocess

vm/src/pyobject.rs

@@ -1383,6 +1383,23 @@ where
 
 pub type PyComparisonValue = PyArithmaticValue<bool>;
 
+#[derive(Clone)]
+pub struct PySequence<T = PyObjectRef>(Vec<T>);
+
+impl<T> PySequence<T> {
+    pub fn into_vec(self) -> Vec<T> {
+        self.0
+    }
+    pub fn as_slice(&self) -> &[T] {
+        &self.0
+    }
+}
+impl<T: TryFromObject> TryFromObject for PySequence<T> {
+    fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
+        vm.extract_elements(&obj).map(Self)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

vm/src/stdlib/mod.rs

@@ -49,6 +49,8 @@ mod faulthandler;
 mod msvcrt;
 #[cfg(not(target_arch = "wasm32"))]
 mod multiprocessing;
+#[cfg(unix)]
+mod posixsubprocess;
 #[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))]
 mod pwd;
 #[cfg(not(target_arch = "wasm32"))]
@@ -145,6 +147,14 @@ pub fn get_module_inits() -> HashMap<String, StdlibInitFunc> {
         modules.insert("pwd".to_owned(), Box::new(pwd::make_module));
     }
 
+    #[cfg(unix)]
+    {
+        modules.insert(
+            "_posixsubprocess".to_owned(),
+            Box::new(posixsubprocess::make_module),
+        );
+    }
+
     // Windows-only
     #[cfg(windows)]
     {

vm/src/stdlib/os.rs

@@ -119,7 +119,7 @@ enum OutputMode {
 fn output_by_mode(val: String, mode: OutputMode, vm: &VirtualMachine) -> PyObjectRef {
     match mode {
         OutputMode::String => vm.ctx.new_str(val),
-        OutputMode::Bytes => vm.ctx.new_bytes(val.as_bytes().to_vec()),
+        OutputMode::Bytes => vm.ctx.new_bytes(val.into_bytes()),
     }
 }
 
@@ -140,6 +140,9 @@ impl PyPathLike {
         use std::os::windows::ffi::OsStrExt;
         self.path.encode_wide().chain(std::iter::once(0)).collect()
     }
+    pub fn into_os_string(self) -> ffi::OsString {
+        self.path
+    }
 }
 
 impl TryFromObject for PyPathLike {
@@ -1036,6 +1039,19 @@ fn os_get_inheritable(fd: RawFd, vm: &VirtualMachine) -> PyResult<bool> {
     }
 }
 
+#[cfg(unix)]
+pub(crate) fn set_inheritable(fd: RawFd, inheritable: bool) -> nix::Result<()> {
+    use nix::fcntl;
+
+    let flags = fcntl::FdFlag::from_bits_truncate(fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD)?);
+    let mut new_flags = flags;
+    new_flags.set(fcntl::FdFlag::FD_CLOEXEC, !inheritable);
+    if flags != new_flags {
+        fcntl::fcntl(fd, fcntl::FcntlArg::F_SETFD(new_flags))?;
+    }
+    Ok(())
+}
+
 fn os_set_inheritable(fd: i64, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> {
     #[cfg(not(any(unix, windows)))]
     {
@@ -1043,21 +1059,7 @@ fn os_set_inheritable(fd: i64, inheritable: bool, vm: &VirtualMachine) -> PyResu
     }
     #[cfg(unix)]
     {
-        let fd = fd as RawFd;
-        let _set_flag = || {
-            use nix::fcntl::fcntl;
-            use nix::fcntl::FcntlArg;
-            use nix::fcntl::FdFlag;
-
-            let flags = FdFlag::from_bits_truncate(fcntl(fd, FcntlArg::F_GETFD)?);
-            let mut new_flags = flags;
-            new_flags.set(FdFlag::from_bits_truncate(libc::FD_CLOEXEC), !inheritable);
-            if flags != new_flags {
-                fcntl(fd, FcntlArg::F_SETFD(new_flags))?;
-            }
-            Ok(())
-        };
-        _set_flag().or_else(|err| Err(convert_nix_error(vm, err)))
+        set_inheritable(fd as RawFd, inheritable).map_err(|err| convert_nix_error(vm, err))
     }
     #[cfg(windows)]
     {
@@ -1860,6 +1862,12 @@ fn os_kill(pid: i32, sig: isize, vm: &VirtualMachine) -> PyResult<()> {
     }
 }
 
+fn os_strerror(e: i32) -> String {
+    unsafe { ffi::CStr::from_ptr(libc::strerror(e)) }
+        .to_string_lossy()
+        .into_owned()
+}
+
 pub fn make_module(vm: &VirtualMachine) -> PyObjectRef {
     let ctx = &vm.ctx;
 
@@ -1970,6 +1978,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef {
         "set_inheritable" => ctx.new_function(os_set_inheritable),
         "link" => ctx.new_function(os_link),
         "kill" => ctx.new_function(os_kill),
+        "strerror" => ctx.new_function(os_strerror),
 
         "O_RDONLY" => ctx.new_int(libc::O_RDONLY),
         "O_WRONLY" => ctx.new_int(libc::O_WRONLY),

vm/src/stdlib/posixsubprocess.rs

@@ -0,0 +1,206 @@
+use nix::{dir, errno::Errno, fcntl, unistd};
+use std::convert::Infallible as Never;
+use std::ffi::{CStr, CString};
+use std::io::{self, prelude::*};
+use std::os::unix::ffi::OsStringExt;
+
+use crate::pyobject::{PyObjectRef, PyResult, PySequence, TryFromObject};
+use crate::VirtualMachine;
+
+use super::os::{convert_nix_error, set_inheritable, PyPathLike};
+
+macro_rules! gen_args {
+    ($($field:ident: $t:ty),*$(,)?) => {
+        #[derive(FromArgs)]
+        struct ForkExecArgs {
+            $(#[pyarg(positional_only)] $field: $t,)*
+        }
+    };
+}
+
+struct CStrPathLike {
+    s: CString,
+}
+impl TryFromObject for CStrPathLike {
+    fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
+        let s = PyPathLike::try_from_object(vm, obj)?
+            .into_os_string()
+            .into_vec();
+        let s = CString::new(s)
+            .map_err(|_| vm.new_value_error("embedded null character".to_owned()))?;
+        Ok(CStrPathLike { s })
+    }
+}
+
+gen_args! {
+    args: PySequence<CStrPathLike> /* list */, exec_list: PySequence<CStrPathLike> /* list */,
+    close_fds: bool, fds_to_keep: PySequence<i32>,
+    cwd: Option<CStrPathLike>, env_list: Option<PySequence<CStrPathLike>>,
+    p2cread: i32, p2cwrite: i32, c2pread: i32, c2pwrite: i32,
+    errread: i32, errwrite: i32, errpipe_read: i32, errpipe_write: i32,
+    restore_signals: bool, call_setsid: bool, preexec_fn: Option<PyObjectRef>,
+}
+
+fn _posixsubprocess_fork_exec(args: ForkExecArgs, vm: &VirtualMachine) -> PyResult<libc::pid_t> {
+    if args.preexec_fn.is_some() {
+        return Err(vm.new_not_implemented_error("preexec_fn not supported yet".to_owned()));
+    }
+    let cstrs_to_ptrs = |cstrs: &[CStrPathLike]| {
+        cstrs
+            .iter()
+            .map(|s| s.s.as_ptr())
+            .chain(std::iter::once(std::ptr::null()))
+            .collect::<Vec<_>>()
+    };
+    let argv = cstrs_to_ptrs(args.args.as_slice());
+    let argv = &argv;
+    let envp = args.env_list.as_ref().map(|s| cstrs_to_ptrs(s.as_slice()));
+    let envp = envp.as_deref();
+    match nix::unistd::fork().map_err(|e| convert_nix_error(vm, e))? {
+        nix::unistd::ForkResult::Child => exec(&args, ProcArgs { argv, envp }),
+        nix::unistd::ForkResult::Parent { child } => Ok(child.as_raw()),
+    }
+}
+
+// can't reallocate inside of exec(), so we reallocate prior to fork() and pass this along
+struct ProcArgs<'a> {
+    argv: &'a [*const libc::c_char],
+    envp: Option<&'a [*const libc::c_char]>,
+}
+
+fn exec(args: &ForkExecArgs, procargs: ProcArgs) -> ! {
+    match exec_inner(args, procargs) {
+        Ok(x) => match x {},
+        Err(e) => {
+            let e = e.as_errno().expect("got a non-errno nix error");
+            let buf: &mut [u8] = &mut [0; 256];
+            let mut cur = io::Cursor::new(&mut *buf);
+            // TODO: check if reached preexec, if not then have "noexec" after
+            let _ = write!(cur, "OSError:{}:", e as i32);
+            let pos = cur.position();
+            let _ = unistd::write(args.errpipe_write, &buf[..pos as usize]);
+            std::process::exit(255)
+        }
+    }
+}
+
+fn exec_inner(args: &ForkExecArgs, procargs: ProcArgs) -> nix::Result<Never> {
+    for &fd in args.fds_to_keep.as_slice() {
+        if fd != args.errpipe_write {
+            set_inheritable(fd, true)?
+        }
+    }
+
+    for &fd in &[args.p2cwrite, args.c2pread, args.errread] {
+        if fd != -1 {
+            unistd::close(fd)?;
+        }
+    }
+    unistd::close(args.errpipe_read)?;
+
+    let c2pwrite = if args.c2pwrite == 0 {
+        let fd = unistd::dup(args.c2pwrite)?;
+        set_inheritable(fd, true)?;
+        fd
+    } else {
+        args.c2pwrite
+    };
+
+    let mut errwrite = args.errwrite;
+    while errwrite == 0 || errwrite == 1 {
+        errwrite = unistd::dup(errwrite)?;
+        set_inheritable(errwrite, true)?;
+    }
+
+    let dup_into_stdio = |fd, io_fd| {
+        if fd == io_fd {
+            set_inheritable(fd, true)
+        } else if fd != -1 {
+            unistd::dup2(fd, io_fd).map(drop)
+        } else {
+            Ok(())
+        }
+    };
+    dup_into_stdio(args.p2cread, 0)?;
+    dup_into_stdio(c2pwrite, 1)?;
+    dup_into_stdio(errwrite, 2)?;
+
+    if let Some(ref cwd) = args.cwd {
+        unistd::chdir(cwd.s.as_c_str())?
+    }
+
+    if args.restore_signals {
+        // TODO: restore signals SIGPIPE, SIGXFZ, SIGXFSZ to SIG_DFL
+    }
+
+    if args.call_setsid {
+        unistd::setsid()?;
+    }
+
+    if args.close_fds {
+        close_fds(3, args.fds_to_keep.as_slice())?;
+    }
+
+    let mut first_err = None;
+    for exec in args.exec_list.as_slice() {
+        if let Some(envp) = procargs.envp {
+            unsafe { libc::execve(exec.s.as_ptr(), procargs.argv.as_ptr(), envp.as_ptr()) };
+        } else {
+            unsafe { libc::execv(exec.s.as_ptr(), procargs.argv.as_ptr()) };
+        }
+        let e = Errno::last();
+        if e != Errno::ENOENT && e != Errno::ENOTDIR && first_err.is_none() {
+            first_err = Some(e)
+        }
+    }
+    Err(first_err.unwrap_or_else(Errno::last).into())
+}
+
+fn close_fds(above: i32, keep: &[i32]) -> nix::Result<()> {
+    // TODO: close fds by brute force if readdir doesn't work:
+    // https://github.com/python/cpython/blob/3.8/Modules/_posixsubprocess.c#L220
+    let path = unsafe { CStr::from_bytes_with_nul_unchecked(FD_DIR_NAME) };
+    let mut dir = dir::Dir::open(
+        path,
+        fcntl::OFlag::O_RDONLY | fcntl::OFlag::O_DIRECTORY,
+        nix::sys::stat::Mode::empty(),
+    )?;
+    for e in dir.iter() {
+        if let Some(fd) = pos_int_from_ascii(e?.file_name()) {
+            if fd > above && !keep.contains(&fd) {
+                unistd::close(fd)?
+            }
+        }
+    }
+    Ok(())
+}
+
+#[cfg(any(
+    target_os = "dragonfly",
+    target_os = "freebsd",
+    target_os = "netbsd",
+    target_os = "openbsd",
+    target_os = "macos",
+))]
+const FD_DIR_NAME: &[u8] = b"/dev/fd\0";
+
+#[cfg(target_os = "linux")]
+const FD_DIR_NAME: &[u8] = b"/proc/self/fd\0";
+
+fn pos_int_from_ascii(name: &CStr) -> Option<i32> {
+    let mut num = 0;
+    for c in name.to_bytes() {
+        if !c.is_ascii_digit() {
+            return None;
+        }
+        num = num * 10 + i32::from(c - b'0')
+    }
+    Some(num)
+}
+
+pub fn make_module(vm: &VirtualMachine) -> PyObjectRef {
+    let ctx = &vm.ctx;
+    py_module!(vm, "_posixsubprocess", {
+        "fork_exec" => named_function!(ctx, _posixsubprocess, fork_exec),
+    })
+}