Significant improvements to new function definition style

- PyRef<T> type for accepting references to payloads.
- Args<T> type for consuming remaining positional args
  (mirrors python `*args`).
- KwArgs<T> type for consuming remaining keyword args
  (mirrors python `*kwargs`).
- OptArg<T> type for consuming remaining keyword args
  (no python code equivalent, only possible in native functions like in cpython).
- PyIterable<T> for accepting an iterator over a sequence of Ts.
- Arity checking (but TypeError messages need work)

vm/src/function.rs

@@ -0,0 +1,41 @@
+use std::marker::PhantomData;
+use std::ops::Deref;
+
+use crate::obj::objtype;
+use crate::pyobject::{PyObjectPayload2, PyObjectRef, PyResult, TryFromObject};
+use crate::vm::VirtualMachine;
+
+// TODO: Move PyFuncArgs, FromArgs, etc. here
+
+pub struct PyRef<T> {
+    // invariant: this obj must always have payload of type T
+    obj: PyObjectRef,
+    _payload: PhantomData<T>,
+}
+
+impl<T> Deref for PyRef<T>
+where
+    T: PyObjectPayload2,
+{
+    type Target = T;
+
+    fn deref(&self) -> &T {
+        self.obj.payload().expect("unexpected payload for type")
+    }
+}
+
+impl<T> TryFromObject for PyRef<T>
+where
+    T: PyObjectPayload2,
+{
+    fn try_from_object(vm: &mut VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
+        if objtype::isinstance(&obj, &T::required_type(&vm.ctx)) {
+            Ok(PyRef {
+                obj,
+                _payload: PhantomData,
+            })
+        } else {
+            Err(vm.new_type_error("wrong type".to_string())) // TODO: better message
+        }
+    }
+}

vm/src/lib.rs

@@ -38,6 +38,7 @@ pub mod eval;
 mod exceptions;
 pub mod format;
 mod frame;
+pub mod function;
 pub mod import;
 pub mod obj;
 pub mod pyobject;

vm/src/obj/objint.rs

@@ -4,7 +4,7 @@ use super::objtype;
 use crate::format::FormatSpec;
 use crate::pyobject::{
     FromPyObjectRef, IntoPyObject, PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef,
-    PyResult, TypeProtocol,
+    PyResult, TryFromObject, TypeProtocol,
 };
 use crate::vm::VirtualMachine;
 use num_bigint::{BigInt, ToBigInt};
@@ -31,6 +31,26 @@ impl IntoPyObject for usize {
     }
 }
 
+impl TryFromObject for usize {
+    fn try_from_object(vm: &mut VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
+        // FIXME: don't use get_value
+        match get_value(&obj).to_usize() {
+            Some(value) => Ok(value),
+            None => Err(vm.new_overflow_error("Int value cannot fit into Rust usize".to_string())),
+        }
+    }
+}
+
+impl TryFromObject for isize {
+    fn try_from_object(vm: &mut VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
+        // FIXME: don't use get_value
+        match get_value(&obj).to_isize() {
+            Some(value) => Ok(value),
+            None => Err(vm.new_overflow_error("Int value cannot fit into Rust isize".to_string())),
+        }
+    }
+}
+
 fn int_repr(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
     arg_check!(vm, args, required = [(int, Some(vm.ctx.int_type()))]);
     let v = get_value(int);

vm/src/obj/objrange.rs

@@ -4,8 +4,7 @@ use std::ops::Mul;
 use super::objint;
 use super::objtype;
 use crate::pyobject::{
-    FromPyObject, PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, PyResult,
-    TypeProtocol,
+    PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, PyResult, TypeProtocol,
 };
 use crate::vm::VirtualMachine;
 use num_bigint::{BigInt, Sign};
@@ -21,18 +20,6 @@ pub struct RangeType {
     pub step: BigInt,
 }
 
-type PyRange = RangeType;
-
-impl FromPyObject for PyRange {
-    fn typ(ctx: &PyContext) -> Option<PyObjectRef> {
-        Some(ctx.range_type())
-    }
-
-    fn from_pyobject(obj: PyObjectRef) -> PyResult<Self> {
-        Ok(get_value(&obj))
-    }
-}
-
 impl RangeType {
     #[inline]
     pub fn try_len(&self) -> Option<usize> {
@@ -360,12 +347,22 @@ fn range_bool(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
     Ok(vm.ctx.new_bool(len > 0))
 }
 
-fn range_contains(vm: &mut VirtualMachine, zelf: PyRange, needle: PyObjectRef) -> bool {
-    if objtype::isinstance(&needle, &vm.ctx.int_type()) {
-        zelf.contains(&objint::get_value(&needle))
+fn range_contains(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
+    arg_check!(
+        vm,
+        args,
+        required = [(zelf, Some(vm.ctx.range_type())), (needle, None)]
+    );
+
+    let range = get_value(zelf);
+
+    let result = if objtype::isinstance(needle, &vm.ctx.int_type()) {
+        range.contains(&objint::get_value(needle))
     } else {
         false
-    }
+    };
+
+    Ok(vm.ctx.new_bool(result))
 }
 
 fn range_index(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {

vm/src/obj/objstr.rs

@@ -3,7 +3,7 @@ use super::objsequence::PySliceableSequence;
 use super::objtype;
 use crate::format::{FormatParseError, FormatPart, FormatString};
 use crate::pyobject::{
-    FromPyObject, PyContext, PyFuncArgs, PyObjectPayload, PyObjectRef, PyResult, TypeProtocol,
+    PyContext, PyFuncArgs, PyObjectPayload, PyObjectRef, PyResult, TypeProtocol,
 };
 use crate::vm::VirtualMachine;
 use num_traits::ToPrimitive;
@@ -16,16 +16,6 @@ extern crate unicode_segmentation;
 
 use self::unicode_segmentation::UnicodeSegmentation;
 
-impl FromPyObject for String {
-    fn typ(ctx: &PyContext) -> Option<PyObjectRef> {
-        Some(ctx.str_type())
-    }
-
-    fn from_pyobject(obj: PyObjectRef) -> PyResult<Self> {
-        Ok(get_value(&obj))
-    }
-}
-
 pub fn init(context: &PyContext) {
     let str_type = &context.str_type;
     context.set_attr(&str_type, "__add__", context.new_rustfunc(str_add));
@@ -477,8 +467,15 @@ fn str_rstrip(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
     Ok(vm.ctx.new_str(value))
 }
 
-fn str_endswith(_vm: &mut VirtualMachine, zelf: String, suffix: String) -> bool {
-    zelf.ends_with(&suffix)
+fn str_endswith(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
+    arg_check!(
+        vm,
+        args,
+        required = [(s, Some(vm.ctx.str_type())), (pat, Some(vm.ctx.str_type()))]
+    );
+    let value = get_value(&s);
+    let pat = get_value(&pat);
+    Ok(vm.ctx.new_bool(value.ends_with(pat.as_str())))
 }
 
 fn str_isidentifier(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {

vm/src/pyobject.rs

@@ -1,3 +1,9 @@
+use std::cell::{Cell, RefCell};
+use std::collections::HashMap;
+use std::fmt;
+use std::iter;
+use std::rc::{Rc, Weak};
+
 use crate::bytecode;
 use crate::exceptions;
 use crate::frame::Frame;
@@ -35,10 +41,6 @@ use num_bigint::BigInt;
 use num_bigint::ToBigInt;
 use num_complex::Complex64;
 use num_traits::{One, Zero};
-use std::cell::{Cell, RefCell};
-use std::collections::HashMap;
-use std::fmt;
-use std::rc::{Rc, Weak};
 
 /* Python objects and references.
 
@@ -575,7 +577,7 @@ impl PyContext {
     {
         PyObject::new(
             PyObjectPayload::RustFunction {
-                function: factory.create(self),
+                function: factory.create(),
             },
             self.builtin_function_or_method_type(),
         )
@@ -957,24 +959,191 @@ impl PyFuncArgs {
             None => Ok(None),
         }
     }
-}
 
-pub trait FromPyObject: Sized {
-    fn typ(ctx: &PyContext) -> Option<PyObjectRef>;
-
-    fn from_pyobject(obj: PyObjectRef) -> PyResult<Self>;
-}
-
-impl FromPyObject for PyObjectRef {
-    fn typ(_ctx: &PyContext) -> Option<PyObjectRef> {
-        None
+    fn into_iter(self) -> impl Iterator<Item = PyArg> {
+        self.args.into_iter().map(PyArg::Positional).chain(
+            self.kwargs
+                .into_iter()
+                .map(|(name, value)| PyArg::Keyword(name, value)),
+        )
     }
 
-    fn from_pyobject(obj: PyObjectRef) -> PyResult<Self> {
+    fn bind<T: FromArgs>(self, vm: &mut VirtualMachine) -> PyResult<T> {
+        let mut args = self.into_iter().peekable();
+        let bound = T::from_args(vm, &mut args)?;
+
+        if args.next().is_none() {
+            Ok(bound)
+        } else {
+            Err(vm.new_type_error("too many args".to_string())) // TODO: improve error message
+        }
+    }
+}
+
+pub trait FromArgs: Sized {
+    fn from_args<I>(vm: &mut VirtualMachine, args: &mut iter::Peekable<I>) -> PyResult<Self>
+    where
+        I: Iterator<Item = PyArg>;
+}
+
+pub struct PyIterable<T> {
+    obj: PyObjectRef,
+    method: PyObjectRef,
+    _item: std::marker::PhantomData<T>,
+}
+
+impl<T> PyIterable<T> {
+    pub fn iter<'a>(&self, vm: &'a mut VirtualMachine) -> PyResult<PyIterator<'a, T>> {
+        let iter_obj = vm.invoke(
+            self.method.clone(),
+            PyFuncArgs {
+                args: vec![],
+                kwargs: vec![],
+            },
+        )?;
+
+        Ok(PyIterator {
+            vm,
+            obj: iter_obj,
+            _item: std::marker::PhantomData,
+        })
+    }
+}
+
+pub struct PyIterator<'a, T> {
+    vm: &'a mut VirtualMachine,
+    obj: PyObjectRef,
+    _item: std::marker::PhantomData<T>,
+}
+
+impl<'a, T> Iterator for PyIterator<'a, T>
+where
+    T: TryFromObject,
+{
+    type Item = PyResult<T>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.vm.call_method(&self.obj, "__next__", vec![]) {
+            Ok(value) => Some(T::try_from_object(self.vm, value)),
+            Err(ref err) if objtype::isinstance(err, &self.vm.ctx.exceptions.stop_iteration) => {
+                None
+            }
+            Err(err) => Some(Err(err)),
+        }
+    }
+}
+
+impl<T> TryFromObject for PyIterable<T>
+where
+    T: TryFromObject,
+{
+    fn try_from_object(vm: &mut VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
+        Ok(PyIterable {
+            obj: obj.clone(),
+            method: vm.get_method(obj, "__iter__")?,
+            _item: std::marker::PhantomData,
+        })
+    }
+}
+
+impl TryFromObject for PyObjectRef {
+    fn try_from_object(_vm: &mut VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
         Ok(obj)
     }
 }
 
+pub struct KwArgs<T>(HashMap<String, T>);
+
+impl<T> FromArgs for KwArgs<T>
+where
+    T: TryFromObject,
+{
+    fn from_args<I>(vm: &mut VirtualMachine, args: &mut iter::Peekable<I>) -> PyResult<Self>
+    where
+        I: Iterator<Item = PyArg>,
+    {
+        let mut kwargs = HashMap::new();
+        while let Some(PyArg::Keyword(name, value)) = args.next() {
+            kwargs.insert(name, T::try_from_object(vm, value)?);
+        }
+        Ok(KwArgs(kwargs))
+    }
+}
+
+pub struct Args<T>(Vec<T>);
+
+impl<T> FromArgs for Args<T>
+where
+    T: TryFromObject,
+{
+    fn from_args<I>(vm: &mut VirtualMachine, args: &mut iter::Peekable<I>) -> PyResult<Self>
+    where
+        I: Iterator<Item = PyArg>,
+    {
+        let mut varargs = Vec::new();
+        while let Some(PyArg::Positional(value)) = args.next() {
+            varargs.push(T::try_from_object(vm, value)?);
+        }
+        Ok(Args(varargs))
+    }
+}
+
+impl<T> FromArgs for T
+where
+    T: TryFromObject,
+{
+    fn from_args<I>(vm: &mut VirtualMachine, args: &mut iter::Peekable<I>) -> PyResult<Self>
+    where
+        I: Iterator<Item = PyArg>,
+    {
+        if let Some(PyArg::Positional(value)) = args.next() {
+            Ok(T::try_from_object(vm, value)?)
+        } else {
+            Err(vm.new_type_error("not enough args".to_string())) // TODO: improve error message
+        }
+    }
+}
+
+pub struct OptArg<T>(Option<T>);
+
+impl<T> std::ops::Deref for OptArg<T> {
+    type Target = Option<T>;
+
+    fn deref(&self) -> &Option<T> {
+        &self.0
+    }
+}
+
+impl<T> FromArgs for OptArg<T>
+where
+    T: TryFromObject,
+{
+    fn from_args<I>(vm: &mut VirtualMachine, args: &mut iter::Peekable<I>) -> PyResult<Self>
+    where
+        I: Iterator<Item = PyArg>,
+    {
+        Ok(OptArg(if let Some(PyArg::Positional(_)) = args.peek() {
+            let value = if let Some(PyArg::Positional(value)) = args.next() {
+                value
+            } else {
+                unreachable!()
+            };
+            Some(T::try_from_object(vm, value)?)
+        } else {
+            None
+        }))
+    }
+}
+
+pub enum PyArg {
+    Positional(PyObjectRef),
+    Keyword(String, PyObjectRef),
+}
+
+pub trait TryFromObject: Sized {
+    fn try_from_object(vm: &mut VirtualMachine, obj: PyObjectRef) -> PyResult<Self>;
+}
+
 pub trait IntoPyObject {
     fn into_pyobject(self, ctx: &PyContext) -> PyResult;
 }
@@ -985,30 +1154,46 @@ impl IntoPyObject for PyObjectRef {
     }
 }
 
-impl IntoPyObject for PyResult {
-    fn into_pyobject(self, _ctx: &PyContext) -> PyResult {
-        self
+impl<T> IntoPyObject for PyResult<T>
+where
+    T: IntoPyObject,
+{
+    fn into_pyobject(self, ctx: &PyContext) -> PyResult {
+        self.and_then(|res| T::into_pyobject(res, ctx))
     }
 }
 
-pub trait FromPyFuncArgs: Sized {
-    fn required_params(ctx: &PyContext) -> Vec<Parameter>;
+impl IntoPyObject for () {
+    fn into_pyobject(self, ctx: &PyContext) -> PyResult {
+        Ok(ctx.none())
+    }
+}
 
-    fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult<Self>;
+impl<T> IntoPyObject for T
+where
+    T: PyObjectPayload2 + Sized,
+{
+    fn into_pyobject(self, ctx: &PyContext) -> PyResult {
+        Ok(PyObject::new(
+            PyObjectPayload::AnyRustValue {
+                value: Box::new(self),
+            },
+            T::required_type(ctx),
+        ))
+    }
 }
 
 macro_rules! tuple_from_py_func_args {
     ($($T:ident),+) => {
-        impl<$($T),+> FromPyFuncArgs for ($($T,)+)
+        impl<$($T),+> FromArgs for ($($T,)+)
         where
-            $($T: FromPyFuncArgs),+
+            $($T: FromArgs),+
         {
-            fn required_params(ctx: &PyContext) -> Vec<Parameter> {
-                vec![$($T::required_params(ctx),)+].into_iter().flatten().collect()
-            }
-
-            fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult<Self> {
-                Ok(($($T::from_py_func_args(args)?,)+))
+            fn from_args<I>(vm: &mut VirtualMachine, args: &mut iter::Peekable<I>) -> PyResult<Self>
+            where
+                I: Iterator<Item = PyArg>
+            {
+                Ok(($($T::from_args(vm, args)?,)+))
             }
         }
     };
@@ -1020,141 +1205,46 @@ tuple_from_py_func_args!(A, B, C);
 tuple_from_py_func_args!(A, B, C, D);
 tuple_from_py_func_args!(A, B, C, D, E);
 
-impl<T> FromPyFuncArgs for T
-where
-    T: FromPyObject,
-{
-    fn required_params(ctx: &PyContext) -> Vec<Parameter> {
-        vec![Parameter {
-            kind: PositionalOnly,
-            typ: T::typ(ctx),
-        }]
-    }
-
-    fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult<Self> {
-        Self::from_pyobject(args.shift())
-    }
-}
-
 pub type PyNativeFunc = Box<dyn Fn(&mut VirtualMachine, PyFuncArgs) -> PyResult>;
 
 pub trait PyNativeFuncFactory<T, R> {
-    fn create(self, ctx: &PyContext) -> PyNativeFunc;
+    fn create(self) -> PyNativeFunc;
 }
 
 impl<F> PyNativeFuncFactory<PyFuncArgs, PyResult> for F
 where
     F: Fn(&mut VirtualMachine, PyFuncArgs) -> PyResult + 'static,
 {
-    fn create(self, _ctx: &PyContext) -> PyNativeFunc {
+    fn create(self) -> PyNativeFunc {
         Box::new(self)
     }
 }
 
-macro_rules! tuple_py_native_func_factory {
-    ($($T:ident),+) => {
+macro_rules! py_native_func_factory_tuple {
+    ($(($n:tt, $T:ident)),+) => {
         impl<F, $($T,)+ R> PyNativeFuncFactory<($($T,)+), R> for F
         where
-            F: Fn(&mut VirtualMachine, $($T),+) -> R + 'static,
-            $($T: FromPyFuncArgs,)+
+            F: Fn($($T,)+ &mut VirtualMachine) -> R + 'static,
+            $($T: FromArgs,)+
+            ($($T,)+): FromArgs,
             R: IntoPyObject,
         {
-            fn create(self, ctx: &PyContext) -> PyNativeFunc {
-                let parameters = vec![$($T::required_params(ctx)),+]
-                    .into_iter()
-                    .flatten()
-                    .collect();
-                let signature = Signature::new(parameters);
+            fn create(self) -> PyNativeFunc {
+                Box::new(move |vm, args| {
+                    let ($($n,)+) = args.bind::<($($T,)+)>(vm)?;
 
-                Box::new(move |vm, mut args| {
-                    signature.check(vm, &mut args)?;
-
-                    (self)(vm, $($T::from_py_func_args(&mut args)?,)+)
-                        .into_pyobject(&vm.ctx)
+                    (self)($($n,)+ vm).into_pyobject(&vm.ctx)
                 })
             }
         }
     };
 }
 
-tuple_py_native_func_factory!(A);
-tuple_py_native_func_factory!(A, B);
-tuple_py_native_func_factory!(A, B, C);
-tuple_py_native_func_factory!(A, B, C, D);
-tuple_py_native_func_factory!(A, B, C, D, E);
-
-#[derive(Debug)]
-pub struct Signature {
-    positional_params: Vec<Parameter>,
-    keyword_params: HashMap<String, Parameter>,
-}
-
-impl Signature {
-    fn new(params: Vec<Parameter>) -> Self {
-        let mut positional_params = Vec::new();
-        let mut keyword_params = HashMap::new();
-        for param in params {
-            match param.kind {
-                PositionalOnly => {
-                    positional_params.push(param);
-                }
-                KeywordOnly { ref name } => {
-                    keyword_params.insert(name.clone(), param);
-                }
-            }
-        }
-
-        Self {
-            positional_params,
-            keyword_params,
-        }
-    }
-
-    fn arg_type(&self, pos: usize) -> Option<&PyObjectRef> {
-        self.positional_params[pos].typ.as_ref()
-    }
-
-    #[allow(unused)]
-    fn kwarg_type(&self, name: &str) -> Option<&PyObjectRef> {
-        self.keyword_params[name].typ.as_ref()
-    }
-
-    fn check(&self, vm: &mut VirtualMachine, args: &PyFuncArgs) -> PyResult<()> {
-        // TODO: check arity
-
-        for (pos, arg) in args.args.iter().enumerate() {
-            if let Some(expected_type) = self.arg_type(pos) {
-                if !objtype::isinstance(arg, expected_type) {
-                    let arg_typ = arg.typ();
-                    let expected_type_name = vm.to_pystr(&expected_type)?;
-                    let actual_type = vm.to_pystr(&arg_typ)?;
-                    return Err(vm.new_type_error(format!(
-                        "argument of type {} is required for parameter {} (got: {})",
-                        expected_type_name,
-                        pos + 1,
-                        actual_type
-                    )));
-                }
-            }
-        }
-
-        Ok(())
-    }
-}
-
-#[derive(Debug)]
-pub struct Parameter {
-    typ: Option<PyObjectRef>,
-    kind: ParameterKind,
-}
-
-#[derive(Debug)]
-pub enum ParameterKind {
-    PositionalOnly,
-    KeywordOnly { name: String },
-}
-
-use self::ParameterKind::*;
+py_native_func_factory_tuple!((a, A));
+py_native_func_factory_tuple!((a, A), (b, B));
+py_native_func_factory_tuple!((a, A), (b, B), (c, C));
+py_native_func_factory_tuple!((a, A), (b, B), (c, C), (d, D));
+py_native_func_factory_tuple!((a, A), (b, B), (c, C), (d, D), (e, E));
 
 /// Rather than determining the type of a python object, this enum is more
 /// a holder for the rust payload of a python object. It is more a carrier
@@ -1182,9 +1272,6 @@ pub enum PyObjectPayload {
     Dict {
         elements: RefCell<objdict::DictContentType>,
     },
-    Set {
-        elements: RefCell<HashMap<u64, PyObjectRef>>,
-    },
     Iterator {
         position: Cell<usize>,
         iterated_obj: PyObjectRef,
@@ -1247,6 +1334,9 @@ pub enum PyObjectPayload {
         dict: RefCell<PyAttributes>,
         mro: Vec<PyObjectRef>,
     },
+    Set {
+        elements: RefCell<HashMap<u64, PyObjectRef>>,
+    },
     WeakRef {
         referent: PyObjectWeakRef,
     },
@@ -1322,6 +1412,20 @@ impl PyObject {
     pub fn into_ref(self) -> PyObjectRef {
         Rc::new(self)
     }
+
+    pub fn payload<T: PyObjectPayload2>(&self) -> Option<&T> {
+        if let PyObjectPayload::AnyRustValue { ref value } = self.payload {
+            value.downcast_ref()
+        } else {
+            None
+        }
+    }
+}
+
+// The intention is for this to replace `PyObjectPayload` once everything is
+// converted to use `PyObjectPayload::AnyRustvalue`.
+pub trait PyObjectPayload2: std::any::Any + fmt::Debug {
+    fn required_type(ctx: &PyContext) -> PyObjectRef;
 }
 
 #[cfg(test)]