From b83443cea9fe2fe30b780323e510af0e0f5b35a1 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 5 Jan 2020 00:17:45 -0600 Subject: [PATCH 01/10] Partially implement async generators --- compiler/src/compile.rs | 57 ++++++- compiler/src/error.rs | 6 + vm/src/frame.rs | 14 +- vm/src/obj/mod.rs | 1 + vm/src/obj/objasyncgenerator.rs | 259 ++++++++++++++++++++++++++++++++ vm/src/obj/objcoroinner.rs | 28 +++- vm/src/obj/objfunction.rs | 14 +- vm/src/types.rs | 16 ++ 8 files changed, 378 insertions(+), 17 deletions(-) create mode 100644 vm/src/obj/objasyncgenerator.rs diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index 034e0f9a1..f4d088c42 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -39,7 +39,7 @@ struct CompileContext { func: FunctionContext, } -#[derive(Clone, Copy)] +#[derive(Clone, Copy, PartialEq)] enum FunctionContext { NoFunction, Function, @@ -562,6 +562,17 @@ impl Compiler { } match value { Some(v) => { +<<<<<<< Updated upstream +======= + if self.ctx.func == FunctionContext::AsyncFunction + && self.current_output().is_generator() + { + return Err(self.error_loc( + CompileErrorType::AsyncReturnValue, + statement.location.clone(), + )); + } +>>>>>>> Stashed changes self.compile_expression(v)?; } None => { @@ -1543,6 +1554,8 @@ impl Compiler { self.set_source_location(expression.location); use ast::ExpressionType::*; + #[allow(unused_imports)] // not unused, overrides ast::ExpressionType::None + use Option::None; match &expression.node { Call { function, @@ -1637,13 +1650,25 @@ impl Compiler { self.mark_generator(); match value { Some(expression) => self.compile_expression(expression)?, - Option::None => self.emit(Instruction::LoadConst { + None => self.emit(Instruction::LoadConst { value: bytecode::Constant::None, }), }; self.emit(Instruction::YieldValue); } Await { value } => { + if self.ctx.func != FunctionContext::AsyncFunction { +<<<<<<< Updated upstream + return Err(CompileError { + statement: None, + error: CompileErrorType::InvalidAwait, + location: self.current_source_location.clone(), + source_path: None, + }); +======= + return Err(self.error(CompileErrorType::InvalidAwait)); +>>>>>>> Stashed changes + } self.compile_expression(value)?; self.emit(Instruction::GetAwaitable); self.emit(Instruction::LoadConst { @@ -1652,6 +1677,32 @@ impl Compiler { self.emit(Instruction::YieldFrom); } YieldFrom { value } => { + match self.ctx.func { + FunctionContext::NoFunction => { +<<<<<<< Updated upstream + return Err(CompileError { + statement: None, + error: CompileErrorType::InvalidYieldFrom, + location: self.current_source_location.clone(), + source_path: None, + }) + } + FunctionContext::AsyncFunction => { + return Err(CompileError { + statement: None, + error: CompileErrorType::AsyncYieldFrom, + location: self.current_source_location.clone(), + source_path: None, + }) +======= + return Err(self.error(CompileErrorType::InvalidYieldFrom)) + } + FunctionContext::AsyncFunction => { + return Err(self.error(CompileErrorType::AsyncYieldFrom)) +>>>>>>> Stashed changes + } + FunctionContext::Function => {} + } self.mark_generator(); self.compile_expression(value)?; self.emit(Instruction::GetIter); @@ -1670,7 +1721,7 @@ impl Compiler { value: bytecode::Constant::Boolean { value: false }, }); } - None => { + ast::ExpressionType::None => { self.emit(Instruction::LoadConst { value: bytecode::Constant::None, }); diff --git a/compiler/src/error.rs b/compiler/src/error.rs index 836e479ad..2c8250136 100644 --- a/compiler/src/error.rs +++ b/compiler/src/error.rs @@ -54,6 +54,9 @@ pub enum CompileErrorType { InvalidContinue, InvalidReturn, InvalidYield, + InvalidYieldFrom, + InvalidAwait, + AsyncYieldFrom, } impl CompileError { @@ -96,6 +99,9 @@ impl fmt::Display for CompileError { CompileErrorType::InvalidContinue => "'continue' outside loop".to_owned(), CompileErrorType::InvalidReturn => "'return' outside function".to_owned(), CompileErrorType::InvalidYield => "'yield' outside function".to_owned(), + CompileErrorType::InvalidYieldFrom => "'yield from' outside function".to_owned(), + CompileErrorType::InvalidAwait => "'await' outside async function".to_owned(), + CompileErrorType::AsyncYieldFrom => "'yield from' inside async function".to_owned(), }; if let Some(statement) = &self.statement { diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 21c54277a..922cf7cb4 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -7,6 +7,7 @@ use itertools::Itertools; use crate::bytecode; use crate::exceptions::{self, ExceptionCtor, PyBaseExceptionRef}; use crate::function::{single_or_tuple_any, PyFuncArgs}; +use crate::obj::objasyncgenerator::PyAsyncGenWrappedValue; use crate::obj::objbool; use crate::obj::objcode::PyCodeRef; use crate::obj::objcoroinner::Coro; @@ -118,11 +119,15 @@ impl ExecutionResult { } /// Turn an ExecutionResult into a PyResult that would be returned from a generator or coroutine - pub fn into_result(self, vm: &VirtualMachine) -> PyResult { + pub fn into_result(self, async_stopiter: bool, vm: &VirtualMachine) -> PyResult { match self { ExecutionResult::Yield(value) => Ok(value), ExecutionResult::Return(value) => { - let stop_iteration = vm.ctx.exceptions.stop_iteration.clone(); + let stop_iteration = if async_stopiter { + vm.ctx.exceptions.stop_async_iteration.clone() + } else { + vm.ctx.exceptions.stop_iteration.clone() + }; let args = if vm.is_none(&value) { vec![] } else { @@ -379,6 +384,11 @@ impl Frame { } bytecode::Instruction::YieldValue => { let value = self.pop_value(); + let value = if self.code.flags.contains(bytecode::CodeFlags::IS_COROUTINE) { + PyAsyncGenWrappedValue(value).into_ref(vm).into_object() + } else { + value + }; Ok(Some(ExecutionResult::Yield(value))) } bytecode::Instruction::YieldFrom => self.execute_yield_from(vm), diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index dd259c752..9a5523e46 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -1,5 +1,6 @@ //! This package contains the python basic/builtin types +pub mod objasyncgenerator; pub mod objbool; pub mod objbuiltinfunc; pub mod objbytearray; diff --git a/vm/src/obj/objasyncgenerator.rs b/vm/src/obj/objasyncgenerator.rs new file mode 100644 index 000000000..a5ed03873 --- /dev/null +++ b/vm/src/obj/objasyncgenerator.rs @@ -0,0 +1,259 @@ +use super::objcode::PyCodeRef; +use super::objcoroinner::Coro; +use super::objtype::PyClassRef; +use crate::frame::FrameRef; +use crate::function::OptionalArg; +use crate::pyobject::{ + IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, +}; +use crate::vm::VirtualMachine; + +use std::cell::RefCell; + +#[pyclass(name = "async_generator")] +#[derive(Debug)] +pub struct PyAsyncGen { + inner: Coro, +} +pub type PyAsyncGenRef = PyRef; + +impl PyValue for PyAsyncGen { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.types.async_generator.clone() + } +} + +#[pyimpl] +impl PyAsyncGen { + pub fn as_coro(&self) -> &Coro { + &self.inner + } + + pub fn new(frame: FrameRef, vm: &VirtualMachine) -> PyAsyncGenRef { + PyAsyncGen { + inner: Coro::new_async(frame), + } + .into_ref(vm) + } + + #[pymethod(name = "__aiter__")] + fn aiter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } + + #[pymethod(name = "__anext__")] + fn anext(zelf: PyRef, vm: &VirtualMachine) -> PyAsyncGenASend { + Self::asend(zelf, vm.get_none(), vm) + } + + #[pymethod] + fn asend(zelf: PyRef, value: PyObjectRef, _vm: &VirtualMachine) -> PyAsyncGenASend { + PyAsyncGenASend { + ag: zelf, + value: RefCell::new(Some(value)), + } + } + + #[pymethod] + fn athrow( + zelf: PyRef, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyAsyncGenAThrow { + PyAsyncGenAThrow { + ag: zelf, + value: RefCell::new(Some(( + exc_type, + exc_val.unwrap_or_else(|| vm.get_none()), + exc_tb.unwrap_or_else(|| vm.get_none()), + ))), + } + } + + #[pymethod] + fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + self.inner.close(vm) + } + + #[pyproperty] + fn ag_await(&self, _vm: &VirtualMachine) -> Option { + self.inner.frame().yield_from_target() + } + #[pyproperty] + fn ag_frame(&self, _vm: &VirtualMachine) -> FrameRef { + self.inner.frame() + } + #[pyproperty] + fn ag_running(&self, _vm: &VirtualMachine) -> bool { + self.inner.running() + } + #[pyproperty] + fn ag_code(&self, _vm: &VirtualMachine) -> PyCodeRef { + self.inner.frame().code.clone() + } +} + +#[pyclass(name = "async_generator_wrapped_value")] +#[derive(Debug)] +pub(crate) struct PyAsyncGenWrappedValue(pub PyObjectRef); +impl PyValue for PyAsyncGenWrappedValue { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.types.async_generator_wrapped_value.clone() + } +} + +impl PyAsyncGenWrappedValue { + fn unbox(val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(match val { + val @ Self => { + Err(vm.new_exception( + vm.ctx.exceptions.stop_iteration.clone(), + vec![val.0.clone()], + )) + } + val => Ok(val), + }) + } +} + +#[pyclass(name = "async_generator_asend")] +#[derive(Debug)] +struct PyAsyncGenASend { + ag: PyAsyncGenRef, + value: RefCell>, +} + +impl PyValue for PyAsyncGenASend { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.types.async_generator_asend.clone() + } +} + +#[pyimpl] +impl PyAsyncGenASend { + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } + #[pymethod(name = "__await__")] + fn r#await(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + self.send(vm.get_none(), vm) + } + + #[pymethod] + fn send(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let val = if self.ag.inner.started() { + val + } else if vm.is_none(&val) { + self.value.replace(None).unwrap_or_else(|| vm.get_none()) + } else { + return Err(vm.new_type_error( + "can't send non-None value to a just-started async generator".to_string(), + )); + }; + self.ag + .inner + .send(val, vm) + .and_then(|val| PyAsyncGenWrappedValue::unbox(val, vm)) + } + + #[pymethod] + fn throw( + &self, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.ag + .inner + .throw( + exc_type, + exc_val.unwrap_or_else(|| vm.get_none()), + exc_tb.unwrap_or_else(|| vm.get_none()), + vm, + ) + .and_then(|val| PyAsyncGenWrappedValue::unbox(val, vm)) + } +} + +#[pyclass(name = "async_generator_asend")] +#[derive(Debug)] +struct PyAsyncGenAThrow { + ag: PyAsyncGenRef, + value: RefCell>, +} + +impl PyValue for PyAsyncGenAThrow { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.types.async_generator_athrow.clone() + } +} + +#[pyimpl] +impl PyAsyncGenAThrow { + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } + #[pymethod(name = "__await__")] + fn r#await(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + self.send(vm.get_none(), vm) + } + + #[pymethod] + fn send(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let val = if self.ag.inner.started() { + val + } else if vm.is_none(&val) { + self.value + .replace(None) + .map_or_else(|| Ok(vm.get_none()), |val| val.into_pyobject(vm))? + } else { + return Err(vm.new_type_error( + "can't send non-None value to a just-started async generator".to_string(), + )); + }; + self.ag + .inner + .send(val, vm) + .and_then(|val| PyAsyncGenWrappedValue::unbox(val, vm)) + } + + #[pymethod] + fn throw( + &self, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.ag + .inner + .throw( + exc_type, + exc_val.unwrap_or_else(|| vm.get_none()), + exc_tb.unwrap_or_else(|| vm.get_none()), + vm, + ) + .and_then(|val| PyAsyncGenWrappedValue::unbox(val, vm)) + } +} + +pub fn init(ctx: &PyContext) { + PyAsyncGen::extend_class(ctx, &ctx.types.async_generator); + PyAsyncGenASend::extend_class(ctx, &ctx.types.async_generator_asend); + PyAsyncGenAThrow::extend_class(ctx, &ctx.types.async_generator_athrow); +} diff --git a/vm/src/obj/objcoroinner.rs b/vm/src/obj/objcoroinner.rs index dfb2ab264..83c104826 100644 --- a/vm/src/obj/objcoroinner.rs +++ b/vm/src/obj/objcoroinner.rs @@ -12,6 +12,8 @@ pub struct Coro { closed: Cell, running: Cell, exceptions: RefCell>, + started: Cell, + async_iter: bool, } impl Coro { @@ -21,6 +23,18 @@ impl Coro { closed: Cell::new(false), running: Cell::new(false), exceptions: RefCell::new(vec![]), + started: Cell::new(false), + async_iter: false, + } + } + pub fn new_async(frame: FrameRef) -> Self { + Coro { + frame, + closed: Cell::new(false), + running: Cell::new(false), + exceptions: RefCell::new(vec![]), + started: Cell::new(false), + async_iter: true, } } @@ -47,6 +61,7 @@ impl Coro { .split_off(curr_exception_stack_len), ); self.running.set(false); + self.started.set(true); result } @@ -54,13 +69,11 @@ impl Coro { if self.closed.get() { return Err(objiter::new_stop_iteration(vm)); } - self.frame.push_value(value.clone()); let result = self.run_with_context(|| vm.run_frame(self.frame.clone()), vm); self.maybe_close(&result); - result?.into_result(vm) + result?.into_result(self.async_iter, vm) } - pub fn throw( &self, exc_type: PyObjectRef, @@ -76,7 +89,7 @@ impl Coro { self.run_with_context(|| self.frame.gen_throw(vm, exc_type, exc_val, exc_tb), vm); self.maybe_close(&result); vm.frames.borrow_mut().pop(); - result?.into_result(vm) + result?.into_result(self.async_iter, vm) } pub fn close(&self, vm: &VirtualMachine) -> PyResult<()> { @@ -106,12 +119,15 @@ impl Coro { } } - pub fn closed(&self) -> bool { - self.closed.get() + pub fn started(&self) -> bool { + self.started.get() } pub fn running(&self) -> bool { self.running.get() } + pub fn closed(&self) -> bool { + self.closed.get() + } pub fn frame(&self) -> FrameRef { self.frame.clone() } diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index b9994712d..bb3af08f4 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -6,6 +6,7 @@ use super::objtype::PyClassRef; use crate::bytecode; use crate::frame::Frame; use crate::function::{OptionalArg, PyFuncArgs}; +use crate::obj::objasyncgenerator::PyAsyncGen; use crate::obj::objcoroutine::PyCoroutine; use crate::obj::objgenerator::PyGenerator; use crate::pyobject::{ @@ -221,12 +222,13 @@ impl PyFunction { let frame = Frame::new(code.clone(), scope).into_ref(vm); // If we have a generator, create a new generator - if code.flags.contains(bytecode::CodeFlags::IS_GENERATOR) { - Ok(PyGenerator::new(frame, vm).into_object()) - } else if code.flags.contains(bytecode::CodeFlags::IS_COROUTINE) { - Ok(PyCoroutine::new(frame, vm).into_object()) - } else { - vm.run_frame_full(frame) + let is_gen = code.flags.contains(bytecode::CodeFlags::IS_GENERATOR); + let is_coro = code.flags.contains(bytecode::CodeFlags::IS_COROUTINE); + match (is_gen, is_coro) { + (true, false) => Ok(PyGenerator::new(frame, vm).into_object()), + (false, true) => Ok(PyCoroutine::new(frame, vm).into_object()), + (true, true) => Ok(PyAsyncGen::new(frame, vm).into_object()), + (false, false) => vm.run_frame_full(frame), } } diff --git a/vm/src/types.rs b/vm/src/types.rs index fc8caec05..9e3b6a469 100644 --- a/vm/src/types.rs +++ b/vm/src/types.rs @@ -1,3 +1,4 @@ +use crate::obj::objasyncgenerator; use crate::obj::objbool; use crate::obj::objbuiltinfunc; use crate::obj::objbytearray; @@ -47,6 +48,10 @@ use std::rc::Rc; /// Holder of references to builtin types. #[derive(Debug)] pub struct TypeZoo { + pub async_generator: PyClassRef, + pub async_generator_asend: PyClassRef, + pub async_generator_athrow: PyClassRef, + pub async_generator_wrapped_value: PyClassRef, pub bytes_type: PyClassRef, pub bytesiterator_type: PyClassRef, pub bytearray_type: PyClassRef, @@ -134,6 +139,12 @@ impl TypeZoo { let generator_type = create_type("generator", &type_type, &object_type); let coroutine_type = create_type("coroutine", &type_type, &object_type); let coroutine_wrapper_type = create_type("coroutine_wrapper", &type_type, &object_type); + let async_generator = create_type("async_generator", &type_type, &object_type); + let async_generator_asend = create_type("async_generator_asend", &type_type, &object_type); + let async_generator_athrow = + create_type("async_generator_athrow", &type_type, &object_type); + let async_generator_wrapped_value = + create_type("async_generator_wrapped_value", &type_type, &object_type); let bound_method_type = create_type("method", &type_type, &object_type); let str_type = create_type("str", &type_type, &object_type); let list_type = create_type("list", &type_type, &object_type); @@ -175,6 +186,10 @@ impl TypeZoo { let traceback_type = create_type("traceback", &type_type, &object_type); Self { + async_generator, + async_generator_asend, + async_generator_athrow, + async_generator_wrapped_value, bool_type, memoryview_type, bytearray_type, @@ -379,6 +394,7 @@ pub fn initialize_types(context: &PyContext) { objclassmethod::init(&context); objgenerator::init(&context); objcoroutine::init(&context); + objasyncgenerator::init(&context); objint::init(&context); objfloat::init(&context); objcomplex::init(&context); From f0cc91d9ac38ff8c2620e07d4c214fe583b2738f Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 3 Mar 2020 00:57:41 -0600 Subject: [PATCH 02/10] Refactor to be more accurate to CPython --- vm/src/obj/objasyncgenerator.rs | 242 ++++++++++++++++++++++++-------- vm/src/obj/objcoroinner.rs | 7 +- 2 files changed, 189 insertions(+), 60 deletions(-) diff --git a/vm/src/obj/objasyncgenerator.rs b/vm/src/obj/objasyncgenerator.rs index a5ed03873..96b74ddc5 100644 --- a/vm/src/obj/objasyncgenerator.rs +++ b/vm/src/obj/objasyncgenerator.rs @@ -1,19 +1,18 @@ use super::objcode::PyCodeRef; use super::objcoroinner::Coro; -use super::objtype::PyClassRef; +use super::objtype::{self, PyClassRef}; use crate::frame::FrameRef; use crate::function::OptionalArg; -use crate::pyobject::{ - IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, -}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -use std::cell::RefCell; +use std::cell::Cell; #[pyclass(name = "async_generator")] #[derive(Debug)] pub struct PyAsyncGen { inner: Coro, + running_async: Cell, } pub type PyAsyncGenRef = PyRef; @@ -32,6 +31,7 @@ impl PyAsyncGen { pub fn new(frame: FrameRef, vm: &VirtualMachine) -> PyAsyncGenRef { PyAsyncGen { inner: Coro::new_async(frame), + running_async: Cell::new(false), } .into_ref(vm) } @@ -50,7 +50,8 @@ impl PyAsyncGen { fn asend(zelf: PyRef, value: PyObjectRef, _vm: &VirtualMachine) -> PyAsyncGenASend { PyAsyncGenASend { ag: zelf, - value: RefCell::new(Some(value)), + state: Cell::new(AwaitableState::Init), + value, } } @@ -64,17 +65,28 @@ impl PyAsyncGen { ) -> PyAsyncGenAThrow { PyAsyncGenAThrow { ag: zelf, - value: RefCell::new(Some(( + aclose: false, + state: Cell::new(AwaitableState::Init), + value: ( exc_type, exc_val.unwrap_or_else(|| vm.get_none()), exc_tb.unwrap_or_else(|| vm.get_none()), - ))), + ), } } #[pymethod] - fn close(&self, vm: &VirtualMachine) -> PyResult<()> { - self.inner.close(vm) + fn aclose(zelf: PyRef, vm: &VirtualMachine) -> PyAsyncGenAThrow { + PyAsyncGenAThrow { + ag: zelf, + aclose: true, + state: Cell::new(AwaitableState::Init), + value: ( + vm.ctx.exceptions.generator_exit.clone().into_object(), + vm.get_none(), + vm.get_none(), + ), + } } #[pyproperty] @@ -105,9 +117,20 @@ impl PyValue for PyAsyncGenWrappedValue { } impl PyAsyncGenWrappedValue { - fn unbox(val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn unbox(ag: &PyAsyncGen, val: PyResult, vm: &VirtualMachine) -> PyResult { + if let Err(ref e) = val { + if objtype::isinstance(&e, &vm.ctx.exceptions.stop_async_iteration) + || objtype::isinstance(&e, &vm.ctx.exceptions.generator_exit) + { + ag.inner.closed.set(true); + } + ag.running_async.set(false); + } + let val = val?; + match_class!(match val { val @ Self => { + ag.running_async.set(false); Err(vm.new_exception( vm.ctx.exceptions.stop_iteration.clone(), vec![val.0.clone()], @@ -118,11 +141,19 @@ impl PyAsyncGenWrappedValue { } } +#[derive(Debug, Clone, Copy)] +enum AwaitableState { + Init, + Iter, + Closed, +} + #[pyclass(name = "async_generator_asend")] #[derive(Debug)] struct PyAsyncGenASend { ag: PyAsyncGenRef, - value: RefCell>, + state: Cell, + value: PyObjectRef, } impl PyValue for PyAsyncGenASend { @@ -149,19 +180,34 @@ impl PyAsyncGenASend { #[pymethod] fn send(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let val = if self.ag.inner.started() { - val - } else if vm.is_none(&val) { - self.value.replace(None).unwrap_or_else(|| vm.get_none()) - } else { - return Err(vm.new_type_error( - "can't send non-None value to a just-started async generator".to_string(), - )); + let val = match self.state.get() { + AwaitableState::Closed => { + return Err(vm.new_runtime_error( + "cannot reuse already awaited __anext__()/asend()".to_owned(), + )) + } + AwaitableState::Iter => val, // already running, all good + AwaitableState::Init => { + if self.ag.running_async.get() { + return Err(vm.new_runtime_error( + "anext(): asynchronous generator is already running".to_owned(), + )); + } + self.ag.running_async.set(true); + self.state.set(AwaitableState::Iter); + if vm.is_none(&val) { + self.value.clone() + } else { + val + } + } }; - self.ag - .inner - .send(val, vm) - .and_then(|val| PyAsyncGenWrappedValue::unbox(val, vm)) + let res = self.ag.inner.send(val, vm); + let res = PyAsyncGenWrappedValue::unbox(&self.ag, res, vm); + if res.is_err() { + self.state.set(AwaitableState::Closed); + } + res } #[pymethod] @@ -172,23 +218,38 @@ impl PyAsyncGenASend { exc_tb: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - self.ag - .inner - .throw( - exc_type, - exc_val.unwrap_or_else(|| vm.get_none()), - exc_tb.unwrap_or_else(|| vm.get_none()), - vm, - ) - .and_then(|val| PyAsyncGenWrappedValue::unbox(val, vm)) + if let AwaitableState::Closed = self.state.get() { + return Err( + vm.new_runtime_error("cannot reuse already awaited __anext__()/asend()".to_owned()) + ); + } + + let res = self.ag.inner.throw( + exc_type, + exc_val.unwrap_or_else(|| vm.get_none()), + exc_tb.unwrap_or_else(|| vm.get_none()), + vm, + ); + let res = PyAsyncGenWrappedValue::unbox(&self.ag, res, vm); + if res.is_err() { + self.state.set(AwaitableState::Closed); + } + res + } + + #[pymethod] + fn close(&self) { + self.state.set(AwaitableState::Closed); } } -#[pyclass(name = "async_generator_asend")] +#[pyclass(name = "async_generator_athrow")] #[derive(Debug)] struct PyAsyncGenAThrow { ag: PyAsyncGenRef, - value: RefCell>, + aclose: bool, + state: Cell, + value: (PyObjectRef, PyObjectRef, PyObjectRef), } impl PyValue for PyAsyncGenAThrow { @@ -215,21 +276,53 @@ impl PyAsyncGenAThrow { #[pymethod] fn send(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let val = if self.ag.inner.started() { - val - } else if vm.is_none(&val) { - self.value - .replace(None) - .map_or_else(|| Ok(vm.get_none()), |val| val.into_pyobject(vm))? - } else { - return Err(vm.new_type_error( - "can't send non-None value to a just-started async generator".to_string(), - )); - }; - self.ag - .inner - .send(val, vm) - .and_then(|val| PyAsyncGenWrappedValue::unbox(val, vm)) + match self.state.get() { + AwaitableState::Closed => { + Err(vm + .new_runtime_error("cannot reuse already awaited aclose()/athrow()".to_owned())) + } + AwaitableState::Init => { + if self.ag.running_async.get() { + self.state.set(AwaitableState::Closed); + let msg = if self.aclose { + "aclose(): asynchronous generator is already running" + } else { + "athrow(): asynchronous generator is already running" + }; + return Err(vm.new_runtime_error(msg.to_owned())); + } + if self.ag.inner.closed() { + self.state.set(AwaitableState::Closed); + return Err(vm.new_exception_empty(vm.ctx.exceptions.stop_iteration.clone())); + } + if !vm.is_none(&val) { + return Err(vm.new_runtime_error( + "can't send non-None value to a just-started coroutine".to_owned(), + )); + } + self.state.set(AwaitableState::Iter); + self.ag.running_async.set(true); + + let (ty, val, tb) = self.value.clone(); + let ret = self.ag.inner.throw(ty, val, tb, vm); + let ret = if self.aclose { + self.check_no_ignore(&ret, vm)?; + ret + } else { + PyAsyncGenWrappedValue::unbox(&self.ag, ret, vm) + }; + self.check_error(ret, vm) + } + AwaitableState::Iter => { + let ret = self.ag.inner.send(val, vm); + if self.aclose { + self.check_no_ignore(&ret, vm)?; + self.check_error(ret, vm) + } else { + PyAsyncGenWrappedValue::unbox(&self.ag, ret, vm) + } + } + } } #[pymethod] @@ -240,15 +333,46 @@ impl PyAsyncGenAThrow { exc_tb: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - self.ag - .inner - .throw( - exc_type, - exc_val.unwrap_or_else(|| vm.get_none()), - exc_tb.unwrap_or_else(|| vm.get_none()), - vm, - ) - .and_then(|val| PyAsyncGenWrappedValue::unbox(val, vm)) + let ret = self.ag.inner.throw( + exc_type, + exc_val.unwrap_or_else(|| vm.get_none()), + exc_tb.unwrap_or_else(|| vm.get_none()), + vm, + ); + if self.aclose { + self.check_no_ignore(&ret, vm)?; + self.check_error(ret, vm) + } else { + PyAsyncGenWrappedValue::unbox(&self.ag, ret, vm) + } + } + + #[pymethod] + fn close(&self) { + self.state.set(AwaitableState::Closed); + } + + fn check_no_ignore(&self, res: &PyResult, vm: &VirtualMachine) -> PyResult<()> { + if let Ok(ref v) = res { + if v.payload_is::() { + return Err( + vm.new_runtime_error("async generator ignored GeneratorExit".to_owned()) + ); + } + } + Ok(()) + } + fn check_error(&self, res: PyResult, vm: &VirtualMachine) -> PyResult { + if self.aclose { + if let Err(ref e) = res { + if objtype::isinstance(&e, &vm.ctx.exceptions.stop_async_iteration) + || objtype::isinstance(&e, &vm.ctx.exceptions.generator_exit) + { + return Err(vm.new_exception_empty(vm.ctx.exceptions.stop_iteration.clone())); + } + } + } + res } } diff --git a/vm/src/obj/objcoroinner.rs b/vm/src/obj/objcoroinner.rs index 83c104826..768103446 100644 --- a/vm/src/obj/objcoroinner.rs +++ b/vm/src/obj/objcoroinner.rs @@ -9,7 +9,7 @@ use std::cell::{Cell, RefCell}; #[derive(Debug)] pub struct Coro { frame: FrameRef, - closed: Cell, + pub closed: Cell, running: Cell, exceptions: RefCell>, started: Cell, @@ -69,6 +69,11 @@ impl Coro { if self.closed.get() { return Err(objiter::new_stop_iteration(vm)); } + if !self.started.get() && !vm.is_none(&value) { + return Err(vm.new_type_error( + "can't send non-None value to a just-started coroutine".to_string(), + )); + } self.frame.push_value(value.clone()); let result = self.run_with_context(|| vm.run_frame(self.frame.clone()), vm); self.maybe_close(&result); From 8a30af83577df1732bccfb1b40cbbf20364f2f31 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 3 Mar 2020 00:57:56 -0600 Subject: [PATCH 03/10] Add test/test_asyncgen from CPython 3.6 --- Lib/test/test_asyncgen.py | 1103 +++++++++++++++++++++++++++++++++++++ 1 file changed, 1103 insertions(+) create mode 100644 Lib/test/test_asyncgen.py diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py new file mode 100644 index 000000000..4961170b7 --- /dev/null +++ b/Lib/test/test_asyncgen.py @@ -0,0 +1,1103 @@ +import inspect +import sys +import types +import unittest + +from unittest import mock + +from test.support import import_module +asyncio = import_module("asyncio") + + +class AwaitException(Exception): + pass + + +@types.coroutine +def awaitable(*, throw=False): + if throw: + yield ('throw',) + else: + yield ('result',) + + +def run_until_complete(coro): + exc = False + while True: + try: + if exc: + exc = False + fut = coro.throw(AwaitException) + else: + fut = coro.send(None) + except StopIteration as ex: + return ex.args[0] + + if fut == ('throw',): + exc = True + + +def to_list(gen): + async def iterate(): + res = [] + async for i in gen: + res.append(i) + return res + + return run_until_complete(iterate()) + + +class AsyncGenSyntaxTest(unittest.TestCase): + + def test_async_gen_syntax_01(self): + code = '''async def foo(): + await abc + yield from 123 + ''' + + with self.assertRaisesRegex(SyntaxError, 'yield from.*inside async'): + exec(code, {}, {}) + + def test_async_gen_syntax_02(self): + code = '''async def foo(): + yield from 123 + ''' + + with self.assertRaisesRegex(SyntaxError, 'yield from.*inside async'): + exec(code, {}, {}) + + def test_async_gen_syntax_03(self): + code = '''async def foo(): + await abc + yield + return 123 + ''' + + with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'): + exec(code, {}, {}) + + def test_async_gen_syntax_04(self): + code = '''async def foo(): + yield + return 123 + ''' + + with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'): + exec(code, {}, {}) + + def test_async_gen_syntax_05(self): + code = '''async def foo(): + if 0: + yield + return 12 + ''' + + with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'): + exec(code, {}, {}) + + +class AsyncGenTest(unittest.TestCase): + + def compare_generators(self, sync_gen, async_gen): + def sync_iterate(g): + res = [] + while True: + try: + res.append(g.__next__()) + except StopIteration: + res.append('STOP') + break + except Exception as ex: + res.append(str(type(ex))) + return res + + def async_iterate(g): + res = [] + while True: + an = g.__anext__() + try: + while True: + try: + an.__next__() + except StopIteration as ex: + if ex.args: + res.append(ex.args[0]) + break + else: + res.append('EMPTY StopIteration') + break + except StopAsyncIteration: + raise + except Exception as ex: + res.append(str(type(ex))) + break + except StopAsyncIteration: + res.append('STOP') + break + return res + + def async_iterate(g): + res = [] + while True: + try: + g.__anext__().__next__() + except StopAsyncIteration: + res.append('STOP') + break + except StopIteration as ex: + if ex.args: + res.append(ex.args[0]) + else: + res.append('EMPTY StopIteration') + break + except Exception as ex: + res.append(str(type(ex))) + return res + + sync_gen_result = sync_iterate(sync_gen) + async_gen_result = async_iterate(async_gen) + self.assertEqual(sync_gen_result, async_gen_result) + return async_gen_result + + def test_async_gen_iteration_01(self): + async def gen(): + await awaitable() + a = yield 123 + self.assertIs(a, None) + await awaitable() + yield 456 + await awaitable() + yield 789 + + self.assertEqual(to_list(gen()), [123, 456, 789]) + + def test_async_gen_iteration_02(self): + async def gen(): + await awaitable() + yield 123 + await awaitable() + + g = gen() + ai = g.__aiter__() + self.assertEqual(ai.__anext__().__next__(), ('result',)) + + try: + ai.__anext__().__next__() + except StopIteration as ex: + self.assertEqual(ex.args[0], 123) + else: + self.fail('StopIteration was not raised') + + self.assertEqual(ai.__anext__().__next__(), ('result',)) + + try: + ai.__anext__().__next__() + except StopAsyncIteration as ex: + self.assertFalse(ex.args) + else: + self.fail('StopAsyncIteration was not raised') + + def test_async_gen_exception_03(self): + async def gen(): + await awaitable() + yield 123 + await awaitable(throw=True) + yield 456 + + with self.assertRaises(AwaitException): + to_list(gen()) + + def test_async_gen_exception_04(self): + async def gen(): + await awaitable() + yield 123 + 1 / 0 + + g = gen() + ai = g.__aiter__() + self.assertEqual(ai.__anext__().__next__(), ('result',)) + + try: + ai.__anext__().__next__() + except StopIteration as ex: + self.assertEqual(ex.args[0], 123) + else: + self.fail('StopIteration was not raised') + + with self.assertRaises(ZeroDivisionError): + ai.__anext__().__next__() + + def test_async_gen_exception_05(self): + async def gen(): + yield 123 + raise StopAsyncIteration + + with self.assertRaisesRegex(RuntimeError, + 'async generator.*StopAsyncIteration'): + to_list(gen()) + + @unittest.skip("TODO: RUSTPYTHON") + def test_async_gen_exception_06(self): + async def gen(): + yield 123 + raise StopIteration + + with self.assertRaisesRegex(RuntimeError, + 'async generator.*StopIteration'): + to_list(gen()) + + def test_async_gen_exception_07(self): + def sync_gen(): + try: + yield 1 + 1 / 0 + finally: + yield 2 + yield 3 + + yield 100 + + async def async_gen(): + try: + yield 1 + 1 / 0 + finally: + yield 2 + yield 3 + + yield 100 + + self.compare_generators(sync_gen(), async_gen()) + + @unittest.skip("TODO: RUSTPYTHON") + def test_async_gen_exception_08(self): + def sync_gen(): + try: + yield 1 + finally: + yield 2 + 1 / 0 + yield 3 + + yield 100 + + async def async_gen(): + try: + yield 1 + await awaitable() + finally: + await awaitable() + yield 2 + 1 / 0 + yield 3 + + yield 100 + + self.compare_generators(sync_gen(), async_gen()) + + @unittest.skip("TODO: RUSTPYTHON") + def test_async_gen_exception_09(self): + def sync_gen(): + try: + yield 1 + 1 / 0 + finally: + yield 2 + yield 3 + + yield 100 + + async def async_gen(): + try: + await awaitable() + yield 1 + 1 / 0 + finally: + yield 2 + await awaitable() + yield 3 + + yield 100 + + self.compare_generators(sync_gen(), async_gen()) + + def test_async_gen_exception_10(self): + async def gen(): + yield 123 + with self.assertRaisesRegex(TypeError, + "non-None value .* async generator"): + gen().__anext__().send(100) + + def test_async_gen_exception_11(self): + def sync_gen(): + yield 10 + yield 20 + + def sync_gen_wrapper(): + yield 1 + sg = sync_gen() + sg.send(None) + try: + sg.throw(GeneratorExit()) + except GeneratorExit: + yield 2 + yield 3 + + async def async_gen(): + yield 10 + yield 20 + + async def async_gen_wrapper(): + yield 1 + asg = async_gen() + await asg.asend(None) + try: + await asg.athrow(GeneratorExit()) + except GeneratorExit: + yield 2 + yield 3 + + self.compare_generators(sync_gen_wrapper(), async_gen_wrapper()) + + def test_async_gen_api_01(self): + async def gen(): + yield 123 + + g = gen() + + self.assertEqual(g.__name__, 'gen') + g.__name__ = '123' + self.assertEqual(g.__name__, '123') + + self.assertIn('.gen', g.__qualname__) + g.__qualname__ = '123' + self.assertEqual(g.__qualname__, '123') + + self.assertIsNone(g.ag_await) + self.assertIsInstance(g.ag_frame, types.FrameType) + self.assertFalse(g.ag_running) + self.assertIsInstance(g.ag_code, types.CodeType) + + self.assertTrue(inspect.isawaitable(g.aclose())) + + +class AsyncGenAsyncioTest(unittest.TestCase): + + def setUp(self): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(None) + + def tearDown(self): + self.loop.close() + self.loop = None + + async def to_list(self, gen): + res = [] + async for i in gen: + res.append(i) + return res + + def test_async_gen_asyncio_01(self): + async def gen(): + yield 1 + await asyncio.sleep(0.01, loop=self.loop) + yield 2 + await asyncio.sleep(0.01, loop=self.loop) + return + yield 3 + + res = self.loop.run_until_complete(self.to_list(gen())) + self.assertEqual(res, [1, 2]) + + def test_async_gen_asyncio_02(self): + async def gen(): + yield 1 + await asyncio.sleep(0.01, loop=self.loop) + yield 2 + 1 / 0 + yield 3 + + with self.assertRaises(ZeroDivisionError): + self.loop.run_until_complete(self.to_list(gen())) + + def test_async_gen_asyncio_03(self): + loop = self.loop + + class Gen: + async def __aiter__(self): + yield 1 + await asyncio.sleep(0.01, loop=loop) + yield 2 + + res = loop.run_until_complete(self.to_list(Gen())) + self.assertEqual(res, [1, 2]) + + def test_async_gen_asyncio_anext_04(self): + async def foo(): + yield 1 + await asyncio.sleep(0.01, loop=self.loop) + try: + yield 2 + yield 3 + except ZeroDivisionError: + yield 1000 + await asyncio.sleep(0.01, loop=self.loop) + yield 4 + + async def run1(): + it = foo().__aiter__() + + self.assertEqual(await it.__anext__(), 1) + self.assertEqual(await it.__anext__(), 2) + self.assertEqual(await it.__anext__(), 3) + self.assertEqual(await it.__anext__(), 4) + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + + async def run2(): + it = foo().__aiter__() + + self.assertEqual(await it.__anext__(), 1) + self.assertEqual(await it.__anext__(), 2) + try: + it.__anext__().throw(ZeroDivisionError) + except StopIteration as ex: + self.assertEqual(ex.args[0], 1000) + else: + self.fail('StopIteration was not raised') + self.assertEqual(await it.__anext__(), 4) + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + + self.loop.run_until_complete(run1()) + self.loop.run_until_complete(run2()) + + def test_async_gen_asyncio_anext_05(self): + async def foo(): + v = yield 1 + v = yield v + yield v * 100 + + async def run(): + it = foo().__aiter__() + + try: + it.__anext__().send(None) + except StopIteration as ex: + self.assertEqual(ex.args[0], 1) + else: + self.fail('StopIteration was not raised') + + try: + it.__anext__().send(10) + except StopIteration as ex: + self.assertEqual(ex.args[0], 10) + else: + self.fail('StopIteration was not raised') + + try: + it.__anext__().send(12) + except StopIteration as ex: + self.assertEqual(ex.args[0], 1200) + else: + self.fail('StopIteration was not raised') + + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + + self.loop.run_until_complete(run()) + + def test_async_gen_asyncio_anext_06(self): + DONE = 0 + + # test synchronous generators + def foo(): + try: + yield + except: + pass + g = foo() + g.send(None) + with self.assertRaises(StopIteration): + g.send(None) + + # now with asynchronous generators + + async def gen(): + nonlocal DONE + try: + yield + except: + pass + DONE = 1 + + async def run(): + nonlocal DONE + g = gen() + await g.asend(None) + with self.assertRaises(StopAsyncIteration): + await g.asend(None) + DONE += 10 + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 11) + + def test_async_gen_asyncio_anext_tuple(self): + async def foo(): + try: + yield (1,) + except ZeroDivisionError: + yield (2,) + + async def run(): + it = foo().__aiter__() + + self.assertEqual(await it.__anext__(), (1,)) + with self.assertRaises(StopIteration) as cm: + it.__anext__().throw(ZeroDivisionError) + self.assertEqual(cm.exception.args[0], (2,)) + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + + self.loop.run_until_complete(run()) + + def test_async_gen_asyncio_anext_stopiteration(self): + async def foo(): + try: + yield StopIteration(1) + except ZeroDivisionError: + yield StopIteration(3) + + async def run(): + it = foo().__aiter__() + + v = await it.__anext__() + self.assertIsInstance(v, StopIteration) + self.assertEqual(v.value, 1) + with self.assertRaises(StopIteration) as cm: + it.__anext__().throw(ZeroDivisionError) + v = cm.exception.args[0] + self.assertIsInstance(v, StopIteration) + self.assertEqual(v.value, 3) + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + + self.loop.run_until_complete(run()) + + def test_async_gen_asyncio_aclose_06(self): + async def foo(): + try: + yield 1 + 1 / 0 + finally: + await asyncio.sleep(0.01, loop=self.loop) + yield 12 + + async def run(): + gen = foo() + it = gen.__aiter__() + await it.__anext__() + await gen.aclose() + + with self.assertRaisesRegex( + RuntimeError, + "async generator ignored GeneratorExit"): + self.loop.run_until_complete(run()) + + def test_async_gen_asyncio_aclose_07(self): + DONE = 0 + + async def foo(): + nonlocal DONE + try: + yield 1 + 1 / 0 + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE += 1 + DONE += 1000 + + async def run(): + gen = foo() + it = gen.__aiter__() + await it.__anext__() + await gen.aclose() + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_aclose_08(self): + DONE = 0 + + fut = asyncio.Future(loop=self.loop) + + async def foo(): + nonlocal DONE + try: + yield 1 + await fut + DONE += 1000 + yield 2 + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE += 1 + DONE += 1000 + + async def run(): + gen = foo() + it = gen.__aiter__() + self.assertEqual(await it.__anext__(), 1) + t = self.loop.create_task(it.__anext__()) + await asyncio.sleep(0.01, loop=self.loop) + await gen.aclose() + return t + + t = self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + # Silence ResourceWarnings + fut.cancel() + t.cancel() + self.loop.run_until_complete(asyncio.sleep(0.01, loop=self.loop)) + + def test_async_gen_asyncio_gc_aclose_09(self): + DONE = 0 + + async def gen(): + nonlocal DONE + try: + while True: + yield 1 + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE = 1 + + async def run(): + g = gen() + await g.__anext__() + await g.__anext__() + del g + + await asyncio.sleep(0.1, loop=self.loop) + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_aclose_10(self): + DONE = 0 + + # test synchronous generators + def foo(): + try: + yield + except: + pass + g = foo() + g.send(None) + g.close() + + # now with asynchronous generators + + async def gen(): + nonlocal DONE + try: + yield + except: + pass + DONE = 1 + + async def run(): + nonlocal DONE + g = gen() + await g.asend(None) + await g.aclose() + DONE += 10 + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 11) + + def test_async_gen_asyncio_aclose_11(self): + DONE = 0 + + # test synchronous generators + def foo(): + try: + yield + except: + pass + yield + g = foo() + g.send(None) + with self.assertRaisesRegex(RuntimeError, 'ignored GeneratorExit'): + g.close() + + # now with asynchronous generators + + async def gen(): + nonlocal DONE + try: + yield + except: + pass + yield + DONE += 1 + + async def run(): + nonlocal DONE + g = gen() + await g.asend(None) + with self.assertRaisesRegex(RuntimeError, 'ignored GeneratorExit'): + await g.aclose() + DONE += 10 + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 10) + + def test_async_gen_asyncio_asend_01(self): + DONE = 0 + + # Sanity check: + def sgen(): + v = yield 1 + yield v * 2 + sg = sgen() + v = sg.send(None) + self.assertEqual(v, 1) + v = sg.send(100) + self.assertEqual(v, 200) + + async def gen(): + nonlocal DONE + try: + await asyncio.sleep(0.01, loop=self.loop) + v = yield 1 + await asyncio.sleep(0.01, loop=self.loop) + yield v * 2 + await asyncio.sleep(0.01, loop=self.loop) + return + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE = 1 + + async def run(): + g = gen() + + v = await g.asend(None) + self.assertEqual(v, 1) + + v = await g.asend(100) + self.assertEqual(v, 200) + + with self.assertRaises(StopAsyncIteration): + await g.asend(None) + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_asend_02(self): + DONE = 0 + + async def sleep_n_crash(delay): + await asyncio.sleep(delay, loop=self.loop) + 1 / 0 + + async def gen(): + nonlocal DONE + try: + await asyncio.sleep(0.01, loop=self.loop) + v = yield 1 + await sleep_n_crash(0.01) + DONE += 1000 + yield v * 2 + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE = 1 + + async def run(): + g = gen() + + v = await g.asend(None) + self.assertEqual(v, 1) + + await g.asend(100) + + with self.assertRaises(ZeroDivisionError): + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_asend_03(self): + DONE = 0 + + async def sleep_n_crash(delay): + fut = asyncio.ensure_future(asyncio.sleep(delay, loop=self.loop), + loop=self.loop) + self.loop.call_later(delay / 2, lambda: fut.cancel()) + return await fut + + async def gen(): + nonlocal DONE + try: + await asyncio.sleep(0.01, loop=self.loop) + v = yield 1 + await sleep_n_crash(0.01) + DONE += 1000 + yield v * 2 + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE = 1 + + async def run(): + g = gen() + + v = await g.asend(None) + self.assertEqual(v, 1) + + await g.asend(100) + + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_athrow_01(self): + DONE = 0 + + class FooEr(Exception): + pass + + # Sanity check: + def sgen(): + try: + v = yield 1 + except FooEr: + v = 1000 + yield v * 2 + sg = sgen() + v = sg.send(None) + self.assertEqual(v, 1) + v = sg.throw(FooEr) + self.assertEqual(v, 2000) + with self.assertRaises(StopIteration): + sg.send(None) + + async def gen(): + nonlocal DONE + try: + await asyncio.sleep(0.01, loop=self.loop) + try: + v = yield 1 + except FooEr: + v = 1000 + await asyncio.sleep(0.01, loop=self.loop) + yield v * 2 + await asyncio.sleep(0.01, loop=self.loop) + # return + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE = 1 + + async def run(): + g = gen() + + v = await g.asend(None) + self.assertEqual(v, 1) + + v = await g.athrow(FooEr) + self.assertEqual(v, 2000) + + with self.assertRaises(StopAsyncIteration): + await g.asend(None) + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_athrow_02(self): + DONE = 0 + + class FooEr(Exception): + pass + + async def sleep_n_crash(delay): + fut = asyncio.ensure_future(asyncio.sleep(delay, loop=self.loop), + loop=self.loop) + self.loop.call_later(delay / 2, lambda: fut.cancel()) + return await fut + + async def gen(): + nonlocal DONE + try: + await asyncio.sleep(0.01, loop=self.loop) + try: + v = yield 1 + except FooEr: + await sleep_n_crash(0.01) + yield v * 2 + await asyncio.sleep(0.01, loop=self.loop) + # return + finally: + await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01, loop=self.loop) + DONE = 1 + + async def run(): + g = gen() + + v = await g.asend(None) + self.assertEqual(v, 1) + + try: + await g.athrow(FooEr) + except asyncio.CancelledError: + self.assertEqual(DONE, 1) + raise + else: + self.fail('CancelledError was not raised') + + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + + def test_async_gen_asyncio_athrow_03(self): + DONE = 0 + + # test synchronous generators + def foo(): + try: + yield + except: + pass + g = foo() + g.send(None) + with self.assertRaises(StopIteration): + g.throw(ValueError) + + # now with asynchronous generators + + async def gen(): + nonlocal DONE + try: + yield + except: + pass + DONE = 1 + + async def run(): + nonlocal DONE + g = gen() + await g.asend(None) + with self.assertRaises(StopAsyncIteration): + await g.athrow(ValueError) + DONE += 10 + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 11) + + def test_async_gen_asyncio_athrow_tuple(self): + async def gen(): + try: + yield 1 + except ZeroDivisionError: + yield (2,) + + async def run(): + g = gen() + v = await g.asend(None) + self.assertEqual(v, 1) + v = await g.athrow(ZeroDivisionError) + self.assertEqual(v, (2,)) + with self.assertRaises(StopAsyncIteration): + await g.asend(None) + + self.loop.run_until_complete(run()) + + def test_async_gen_asyncio_athrow_stopiteration(self): + async def gen(): + try: + yield 1 + except ZeroDivisionError: + yield StopIteration(2) + + async def run(): + g = gen() + v = await g.asend(None) + self.assertEqual(v, 1) + v = await g.athrow(ZeroDivisionError) + self.assertIsInstance(v, StopIteration) + self.assertEqual(v.value, 2) + with self.assertRaises(StopAsyncIteration): + await g.asend(None) + + self.loop.run_until_complete(run()) + + def test_async_gen_asyncio_shutdown_01(self): + finalized = 0 + + async def waiter(timeout): + nonlocal finalized + try: + await asyncio.sleep(timeout, loop=self.loop) + yield 1 + finally: + await asyncio.sleep(0, loop=self.loop) + finalized += 1 + + async def wait(): + async for _ in waiter(1): + pass + + t1 = self.loop.create_task(wait()) + t2 = self.loop.create_task(wait()) + + self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop)) + + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) + self.assertEqual(finalized, 2) + + # Silence warnings + t1.cancel() + t2.cancel() + self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop)) + + def test_async_gen_asyncio_shutdown_02(self): + logged = 0 + + def logger(loop, context): + nonlocal logged + self.assertIn('asyncgen', context) + expected = 'an error occurred during closing of asynchronous' + if expected in context['message']: + logged += 1 + + async def waiter(timeout): + try: + await asyncio.sleep(timeout, loop=self.loop) + yield 1 + finally: + 1 / 0 + + async def wait(): + async for _ in waiter(1): + pass + + t = self.loop.create_task(wait()) + self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop)) + + self.loop.set_exception_handler(logger) + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) + + self.assertEqual(logged, 1) + + # Silence warnings + t.cancel() + self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop)) + +if __name__ == "__main__": + unittest.main() From af25fbed39fac85d69cca0407ba5172f99d51538 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Thu, 5 Mar 2020 00:57:05 -0600 Subject: [PATCH 04/10] Fix syntax error tests --- compiler/src/compile.rs | 29 ----------------------------- compiler/src/error.rs | 4 ++++ compiler/src/output_stream.rs | 5 +++++ compiler/src/peephole.rs | 3 +++ parser/src/lexer.rs | 2 ++ 5 files changed, 14 insertions(+), 29 deletions(-) diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index f4d088c42..29a876f5f 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -562,8 +562,6 @@ impl Compiler { } match value { Some(v) => { -<<<<<<< Updated upstream -======= if self.ctx.func == FunctionContext::AsyncFunction && self.current_output().is_generator() { @@ -572,7 +570,6 @@ impl Compiler { statement.location.clone(), )); } ->>>>>>> Stashed changes self.compile_expression(v)?; } None => { @@ -1658,16 +1655,7 @@ impl Compiler { } Await { value } => { if self.ctx.func != FunctionContext::AsyncFunction { -<<<<<<< Updated upstream - return Err(CompileError { - statement: None, - error: CompileErrorType::InvalidAwait, - location: self.current_source_location.clone(), - source_path: None, - }); -======= return Err(self.error(CompileErrorType::InvalidAwait)); ->>>>>>> Stashed changes } self.compile_expression(value)?; self.emit(Instruction::GetAwaitable); @@ -1679,27 +1667,10 @@ impl Compiler { YieldFrom { value } => { match self.ctx.func { FunctionContext::NoFunction => { -<<<<<<< Updated upstream - return Err(CompileError { - statement: None, - error: CompileErrorType::InvalidYieldFrom, - location: self.current_source_location.clone(), - source_path: None, - }) - } - FunctionContext::AsyncFunction => { - return Err(CompileError { - statement: None, - error: CompileErrorType::AsyncYieldFrom, - location: self.current_source_location.clone(), - source_path: None, - }) -======= return Err(self.error(CompileErrorType::InvalidYieldFrom)) } FunctionContext::AsyncFunction => { return Err(self.error(CompileErrorType::AsyncYieldFrom)) ->>>>>>> Stashed changes } FunctionContext::Function => {} } diff --git a/compiler/src/error.rs b/compiler/src/error.rs index 2c8250136..6f28e3a33 100644 --- a/compiler/src/error.rs +++ b/compiler/src/error.rs @@ -57,6 +57,7 @@ pub enum CompileErrorType { InvalidYieldFrom, InvalidAwait, AsyncYieldFrom, + AsyncReturnValue, } impl CompileError { @@ -102,6 +103,9 @@ impl fmt::Display for CompileError { CompileErrorType::InvalidYieldFrom => "'yield from' outside function".to_owned(), CompileErrorType::InvalidAwait => "'await' outside async function".to_owned(), CompileErrorType::AsyncYieldFrom => "'yield from' inside async function".to_owned(), + CompileErrorType::AsyncReturnValue => { + "'return' with value inside async generator".to_owned() + } }; if let Some(statement) = &self.statement { diff --git a/compiler/src/output_stream.rs b/compiler/src/output_stream.rs index fa358817a..9cc49a944 100644 --- a/compiler/src/output_stream.rs +++ b/compiler/src/output_stream.rs @@ -7,6 +7,8 @@ pub trait OutputStream: From + Into { fn set_label(&mut self, label: Label); /// Mark the inner CodeObject as a generator fn mark_generator(&mut self); + /// Check to see if the inner CodeObject is a generator + fn is_generator(&self) -> bool; } pub struct CodeObjectStream { @@ -36,4 +38,7 @@ impl OutputStream for CodeObjectStream { fn mark_generator(&mut self) { self.code.flags |= CodeFlags::IS_GENERATOR; } + fn is_generator(&self) -> bool { + self.code.flags.contains(CodeFlags::IS_GENERATOR) + } } diff --git a/compiler/src/peephole.rs b/compiler/src/peephole.rs index b85266867..67a4b9efe 100644 --- a/compiler/src/peephole.rs +++ b/compiler/src/peephole.rs @@ -112,6 +112,9 @@ where fn mark_generator(&mut self) { self.inner.mark_generator() } + fn is_generator(&self) -> bool { + self.inner.is_generator() + } } impl OptimizationBuffer for PeepholeOptimizer { diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index d43dc37bb..b145588d6 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -685,6 +685,8 @@ where tabs = 0; } None => { + spaces = 0; + tabs = 0; break; } _ => { From 8f2c03c30792f74f05b426daaba0db081bc2da3a Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sat, 7 Mar 2020 08:29:06 -0600 Subject: [PATCH 05/10] Update to test_asyncgen from CPython 3.8 --- Lib/test/test_asyncgen.py | 322 +++++++++++++++++++++++++------------- 1 file changed, 217 insertions(+), 105 deletions(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 4961170b7..c3257b5d7 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1,10 +1,7 @@ import inspect -import sys import types import unittest -from unittest import mock - from test.support import import_module asyncio = import_module("asyncio") @@ -136,24 +133,6 @@ class AsyncGenTest(unittest.TestCase): break return res - def async_iterate(g): - res = [] - while True: - try: - g.__anext__().__next__() - except StopAsyncIteration: - res.append('STOP') - break - except StopIteration as ex: - if ex.args: - res.append(ex.args[0]) - else: - res.append('EMPTY StopIteration') - break - except Exception as ex: - res.append(str(type(ex))) - return res - sync_gen_result = sync_iterate(sync_gen) async_gen_result = async_iterate(async_gen) self.assertEqual(sync_gen_result, async_gen_result) @@ -179,19 +158,22 @@ class AsyncGenTest(unittest.TestCase): g = gen() ai = g.__aiter__() - self.assertEqual(ai.__anext__().__next__(), ('result',)) + + an = ai.__anext__() + self.assertEqual(an.__next__(), ('result',)) try: - ai.__anext__().__next__() + an.__next__() except StopIteration as ex: self.assertEqual(ex.args[0], 123) else: self.fail('StopIteration was not raised') - self.assertEqual(ai.__anext__().__next__(), ('result',)) + an = ai.__anext__() + self.assertEqual(an.__next__(), ('result',)) try: - ai.__anext__().__next__() + an.__next__() except StopAsyncIteration as ex: self.assertFalse(ex.args) else: @@ -215,10 +197,11 @@ class AsyncGenTest(unittest.TestCase): g = gen() ai = g.__aiter__() - self.assertEqual(ai.__anext__().__next__(), ('result',)) + an = ai.__anext__() + self.assertEqual(an.__next__(), ('result',)) try: - ai.__anext__().__next__() + an.__next__() except StopIteration as ex: self.assertEqual(ex.args[0], 123) else: @@ -246,6 +229,7 @@ class AsyncGenTest(unittest.TestCase): 'async generator.*StopIteration'): to_list(gen()) + @unittest.skip("TODO: RUSTPYTHON") def test_async_gen_exception_07(self): def sync_gen(): try: @@ -390,6 +374,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): def tearDown(self): self.loop.close() self.loop = None + asyncio.set_event_loop_policy(None) async def to_list(self, gen): res = [] @@ -400,9 +385,9 @@ class AsyncGenAsyncioTest(unittest.TestCase): def test_async_gen_asyncio_01(self): async def gen(): yield 1 - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) yield 2 - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) return yield 3 @@ -412,7 +397,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): def test_async_gen_asyncio_02(self): async def gen(): yield 1 - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) yield 2 1 / 0 yield 3 @@ -426,7 +411,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): class Gen: async def __aiter__(self): yield 1 - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) yield 2 res = loop.run_until_complete(self.to_list(Gen())) @@ -435,13 +420,13 @@ class AsyncGenAsyncioTest(unittest.TestCase): def test_async_gen_asyncio_anext_04(self): async def foo(): yield 1 - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) try: yield 2 yield 3 except ZeroDivisionError: yield 1000 - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) yield 4 async def run1(): @@ -592,7 +577,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): yield 1 1 / 0 finally: - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) yield 12 async def run(): @@ -615,8 +600,8 @@ class AsyncGenAsyncioTest(unittest.TestCase): yield 1 1 / 0 finally: - await asyncio.sleep(0.01, loop=self.loop) - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) + await asyncio.sleep(0.01) DONE += 1 DONE += 1000 @@ -642,8 +627,8 @@ class AsyncGenAsyncioTest(unittest.TestCase): DONE += 1000 yield 2 finally: - await asyncio.sleep(0.01, loop=self.loop) - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) + await asyncio.sleep(0.01) DONE += 1 DONE += 1000 @@ -651,18 +636,14 @@ class AsyncGenAsyncioTest(unittest.TestCase): gen = foo() it = gen.__aiter__() self.assertEqual(await it.__anext__(), 1) - t = self.loop.create_task(it.__anext__()) - await asyncio.sleep(0.01, loop=self.loop) await gen.aclose() - return t - t = self.loop.run_until_complete(run()) + self.loop.run_until_complete(run()) self.assertEqual(DONE, 1) # Silence ResourceWarnings fut.cancel() - t.cancel() - self.loop.run_until_complete(asyncio.sleep(0.01, loop=self.loop)) + self.loop.run_until_complete(asyncio.sleep(0.01)) def test_async_gen_asyncio_gc_aclose_09(self): DONE = 0 @@ -673,8 +654,8 @@ class AsyncGenAsyncioTest(unittest.TestCase): while True: yield 1 finally: - await asyncio.sleep(0.01, loop=self.loop) - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) + await asyncio.sleep(0.01) DONE = 1 async def run(): @@ -683,7 +664,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): await g.__anext__() del g - await asyncio.sleep(0.1, loop=self.loop) + await asyncio.sleep(0.1) self.loop.run_until_complete(run()) self.assertEqual(DONE, 1) @@ -758,6 +739,33 @@ class AsyncGenAsyncioTest(unittest.TestCase): self.loop.run_until_complete(run()) self.assertEqual(DONE, 10) + def test_async_gen_asyncio_aclose_12(self): + DONE = 0 + + async def target(): + await asyncio.sleep(0.01) + 1 / 0 + + async def foo(): + nonlocal DONE + task = asyncio.create_task(target()) + try: + yield 1 + finally: + try: + await task + except ZeroDivisionError: + DONE = 1 + + async def run(): + gen = foo() + it = gen.__aiter__() + await it.__anext__() + await gen.aclose() + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + def test_async_gen_asyncio_asend_01(self): DONE = 0 @@ -774,15 +782,15 @@ class AsyncGenAsyncioTest(unittest.TestCase): async def gen(): nonlocal DONE try: - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) v = yield 1 - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) yield v * 2 - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) return finally: - await asyncio.sleep(0.01, loop=self.loop) - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) + await asyncio.sleep(0.01) DONE = 1 async def run(): @@ -804,20 +812,20 @@ class AsyncGenAsyncioTest(unittest.TestCase): DONE = 0 async def sleep_n_crash(delay): - await asyncio.sleep(delay, loop=self.loop) + await asyncio.sleep(delay) 1 / 0 async def gen(): nonlocal DONE try: - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) v = yield 1 await sleep_n_crash(0.01) DONE += 1000 yield v * 2 finally: - await asyncio.sleep(0.01, loop=self.loop) - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) + await asyncio.sleep(0.01) DONE = 1 async def run(): @@ -836,7 +844,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): DONE = 0 async def sleep_n_crash(delay): - fut = asyncio.ensure_future(asyncio.sleep(delay, loop=self.loop), + fut = asyncio.ensure_future(asyncio.sleep(delay), loop=self.loop) self.loop.call_later(delay / 2, lambda: fut.cancel()) return await fut @@ -844,14 +852,14 @@ class AsyncGenAsyncioTest(unittest.TestCase): async def gen(): nonlocal DONE try: - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) v = yield 1 await sleep_n_crash(0.01) DONE += 1000 yield v * 2 finally: - await asyncio.sleep(0.01, loop=self.loop) - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) + await asyncio.sleep(0.01) DONE = 1 async def run(): @@ -890,18 +898,18 @@ class AsyncGenAsyncioTest(unittest.TestCase): async def gen(): nonlocal DONE try: - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) try: v = yield 1 except FooEr: v = 1000 - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) yield v * 2 - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) # return finally: - await asyncio.sleep(0.01, loop=self.loop) - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) + await asyncio.sleep(0.01) DONE = 1 async def run(): @@ -926,7 +934,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): pass async def sleep_n_crash(delay): - fut = asyncio.ensure_future(asyncio.sleep(delay, loop=self.loop), + fut = asyncio.ensure_future(asyncio.sleep(delay), loop=self.loop) self.loop.call_later(delay / 2, lambda: fut.cancel()) return await fut @@ -934,17 +942,17 @@ class AsyncGenAsyncioTest(unittest.TestCase): async def gen(): nonlocal DONE try: - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) try: v = yield 1 except FooEr: await sleep_n_crash(0.01) yield v * 2 - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) # return finally: - await asyncio.sleep(0.01, loop=self.loop) - await asyncio.sleep(0.01, loop=self.loop) + await asyncio.sleep(0.01) + await asyncio.sleep(0.01) DONE = 1 async def run(): @@ -1043,10 +1051,10 @@ class AsyncGenAsyncioTest(unittest.TestCase): async def waiter(timeout): nonlocal finalized try: - await asyncio.sleep(timeout, loop=self.loop) + await asyncio.sleep(timeout) yield 1 finally: - await asyncio.sleep(0, loop=self.loop) + await asyncio.sleep(0) finalized += 1 async def wait(): @@ -1056,48 +1064,152 @@ class AsyncGenAsyncioTest(unittest.TestCase): t1 = self.loop.create_task(wait()) t2 = self.loop.create_task(wait()) - self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop)) - - self.loop.run_until_complete(self.loop.shutdown_asyncgens()) - self.assertEqual(finalized, 2) + self.loop.run_until_complete(asyncio.sleep(0.1)) # Silence warnings t1.cancel() t2.cancel() - self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop)) - def test_async_gen_asyncio_shutdown_02(self): - logged = 0 + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(t1) + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(t2) - def logger(loop, context): - nonlocal logged - self.assertIn('asyncgen', context) - expected = 'an error occurred during closing of asynchronous' - if expected in context['message']: - logged += 1 - - async def waiter(timeout): - try: - await asyncio.sleep(timeout, loop=self.loop) - yield 1 - finally: - 1 / 0 - - async def wait(): - async for _ in waiter(1): - pass - - t = self.loop.create_task(wait()) - self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop)) - - self.loop.set_exception_handler(logger) self.loop.run_until_complete(self.loop.shutdown_asyncgens()) - self.assertEqual(logged, 1) + self.assertEqual(finalized, 2) + + # TODO: RUSTPYTHON: async for gen expression compilation + # def test_async_gen_expression_01(self): + # async def arange(n): + # for i in range(n): + # await asyncio.sleep(0.01) + # yield i + + # def make_arange(n): + # # This syntax is legal starting with Python 3.7 + # return (i * 2 async for i in arange(n)) + + # async def run(): + # return [i async for i in make_arange(10)] + + # res = self.loop.run_until_complete(run()) + # self.assertEqual(res, [i * 2 for i in range(10)]) + + # def test_async_gen_expression_02(self): + # async def wrap(n): + # await asyncio.sleep(0.01) + # return n + + # def make_arange(n): + # # This syntax is legal starting with Python 3.7 + # return (i * 2 for i in range(n) if await wrap(i)) + + # async def run(): + # return [i async for i in make_arange(10)] + + # res = self.loop.run_until_complete(run()) + # self.assertEqual(res, [i * 2 for i in range(1, 10)]) + + def test_asyncgen_nonstarted_hooks_are_cancellable(self): + # See https://bugs.python.org/issue38013 + messages = [] + + def exception_handler(loop, context): + messages.append(context) + + async def async_iterate(): + yield 1 + yield 2 + + async def main(): + loop = asyncio.get_running_loop() + loop.set_exception_handler(exception_handler) + + async for i in async_iterate(): + break + + asyncio.run(main()) + + self.assertEqual([], messages) + + def test_async_gen_await_same_anext_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + nxt = it.__anext__() + await nxt + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)" + ): + await nxt + + await it.aclose() # prevent unfinished iterator warning + + self.loop.run_until_complete(run()) + + def test_async_gen_await_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + nxt = it.aclose() + await nxt + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + await nxt + + self.loop.run_until_complete(run()) + + def test_async_gen_aclose_twice_with_different_coros(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + await it.aclose() + await it.aclose() + + self.loop.run_until_complete(run()) + + def test_async_gen_aclose_after_exhaustion(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + async for _ in it: + pass + await it.aclose() + + self.loop.run_until_complete(run()) + + def test_async_gen_aclose_compatible_with_get_stack(self): + async def async_generator(): + yield object() + + async def run(): + ag = async_generator() + asyncio.create_task(ag.aclose()) + tasks = asyncio.all_tasks() + for task in tasks: + # No AttributeError raised + task.get_stack() + + self.loop.run_until_complete(run()) - # Silence warnings - t.cancel() - self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop)) if __name__ == "__main__": unittest.main() From 9507a9f7957e7e42d49bbca73df092b0ed5cf844 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sat, 7 Mar 2020 08:57:27 -0600 Subject: [PATCH 06/10] Add some functions from asyncio from Python 3.8 --- Lib/asyncio/events.py | 15 ++++++++++++++- Lib/asyncio/tasks.py | 23 ++++++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 28a45fc3c..466db6d9a 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -6,7 +6,8 @@ __all__ = ['AbstractEventLoopPolicy', 'get_event_loop_policy', 'set_event_loop_policy', 'get_event_loop', 'set_event_loop', 'new_event_loop', 'get_child_watcher', 'set_child_watcher', - '_set_running_loop', '_get_running_loop', + '_set_running_loop', 'get_running_loop', + '_get_running_loop', ] import functools @@ -614,6 +615,18 @@ class _RunningLoop(threading.local): _running_loop = _RunningLoop() +def get_running_loop(): + """Return the running event loop. Raise a RuntimeError if there is none. + + This function is thread-specific. + """ + # NOTE: this function is implemented in C (see _asynciomodule.c) + loop = _get_running_loop() + if loop is None: + raise RuntimeError('no running event loop') + return loop + + def _get_running_loop(): """Return the running event loop or None. diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index d31c0109f..8c4f3cca7 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -1,6 +1,6 @@ """Support for tasks, coroutines and the scheduler.""" -__all__ = ['Task', +__all__ = ['Task', 'create_task', 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', 'wait', 'wait_for', 'as_completed', 'sleep', 'async', 'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe', @@ -44,6 +44,16 @@ def _all_tasks_compat(loop=None): return {t for t in _all_tasks if futures._get_loop(t) is loop} +def _set_task_name(task, name): + if name is not None: + try: + set_name = task.set_name + except AttributeError: + pass + else: + set_name(name) + + class Task(futures.Future): """A coroutine wrapped in a Future.""" @@ -292,6 +302,17 @@ else: Task = _CTask = _asyncio.Task +def create_task(coro, *, name=None): + """Schedule the execution of a coroutine object in a spawn task. + + Return a Task object. + """ + loop = events.get_running_loop() + task = loop.create_task(coro) + _set_task_name(task, name) + return task + + # wait() and as_completed() similar to those in PEP 3148. FIRST_COMPLETED = concurrent.futures.FIRST_COMPLETED From c1bf5124b1b0e82337e398a3ffc45a41f6805c46 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sat, 7 Mar 2020 09:33:36 -0600 Subject: [PATCH 07/10] RuntimeError on coro raising StopIteration --- Lib/test/test_asyncgen.py | 1 - vm/src/obj/objasyncgenerator.rs | 4 +-- vm/src/obj/objcoroinner.rs | 55 ++++++++++++++++++++++++--------- vm/src/obj/objcoroutine.rs | 4 +-- vm/src/obj/objgenerator.rs | 4 +-- 5 files changed, 46 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index c3257b5d7..102089816 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -219,7 +219,6 @@ class AsyncGenTest(unittest.TestCase): 'async generator.*StopAsyncIteration'): to_list(gen()) - @unittest.skip("TODO: RUSTPYTHON") def test_async_gen_exception_06(self): async def gen(): yield 123 diff --git a/vm/src/obj/objasyncgenerator.rs b/vm/src/obj/objasyncgenerator.rs index 96b74ddc5..40bc4f144 100644 --- a/vm/src/obj/objasyncgenerator.rs +++ b/vm/src/obj/objasyncgenerator.rs @@ -1,5 +1,5 @@ use super::objcode::PyCodeRef; -use super::objcoroinner::Coro; +use super::objcoroinner::{Coro, Variant}; use super::objtype::{self, PyClassRef}; use crate::frame::FrameRef; use crate::function::OptionalArg; @@ -30,7 +30,7 @@ impl PyAsyncGen { pub fn new(frame: FrameRef, vm: &VirtualMachine) -> PyAsyncGenRef { PyAsyncGen { - inner: Coro::new_async(frame), + inner: Coro::new(frame, Variant::AsyncGen), running_async: Cell::new(false), } .into_ref(vm) diff --git a/vm/src/obj/objcoroinner.rs b/vm/src/obj/objcoroinner.rs index 768103446..51ab8f12d 100644 --- a/vm/src/obj/objcoroinner.rs +++ b/vm/src/obj/objcoroinner.rs @@ -6,6 +6,25 @@ use crate::vm::VirtualMachine; use std::cell::{Cell, RefCell}; +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum Variant { + Gen, + Coroutine, + AsyncGen, +} +impl Variant { + fn exec_result(self, res: ExecutionResult, vm: &VirtualMachine) -> PyResult { + res.into_result(self == Self::AsyncGen, vm) + } + fn name(self) -> &'static str { + match self { + Self::Gen => "generator", + Self::Coroutine => "coroutine", + Self::AsyncGen => "async generator", + } + } +} + #[derive(Debug)] pub struct Coro { frame: FrameRef, @@ -13,28 +32,18 @@ pub struct Coro { running: Cell, exceptions: RefCell>, started: Cell, - async_iter: bool, + variant: Variant, } impl Coro { - pub fn new(frame: FrameRef) -> Self { + pub fn new(frame: FrameRef, variant: Variant) -> Self { Coro { frame, closed: Cell::new(false), running: Cell::new(false), exceptions: RefCell::new(vec![]), started: Cell::new(false), - async_iter: false, - } - } - pub fn new_async(frame: FrameRef) -> Self { - Coro { - frame, - closed: Cell::new(false), - running: Cell::new(false), - exceptions: RefCell::new(vec![]), - started: Cell::new(false), - async_iter: true, + variant, } } @@ -62,6 +71,22 @@ impl Coro { ); self.running.set(false); self.started.set(true); + if let Err(ref e) = result { + if objtype::isinstance(e, &vm.ctx.exceptions.stop_iteration) { + let err = + vm.new_runtime_error(format!("{} raised StopIteration", self.variant.name())); + err.set_cause(Some(e.clone())); + return Err(err); + } + if self.variant == Variant::AsyncGen + && objtype::isinstance(e, &vm.ctx.exceptions.stop_async_iteration) + { + let err = + vm.new_runtime_error("async generator raise StopAsyncIteration".to_owned()); + err.set_cause(Some(e.clone())); + return Err(err); + } + } result } @@ -77,7 +102,7 @@ impl Coro { self.frame.push_value(value.clone()); let result = self.run_with_context(|| vm.run_frame(self.frame.clone()), vm); self.maybe_close(&result); - result?.into_result(self.async_iter, vm) + self.variant.exec_result(result?, vm) } pub fn throw( &self, @@ -94,7 +119,7 @@ impl Coro { self.run_with_context(|| self.frame.gen_throw(vm, exc_type, exc_val, exc_tb), vm); self.maybe_close(&result); vm.frames.borrow_mut().pop(); - result?.into_result(self.async_iter, vm) + self.variant.exec_result(result?, vm) } pub fn close(&self, vm: &VirtualMachine) -> PyResult<()> { diff --git a/vm/src/obj/objcoroutine.rs b/vm/src/obj/objcoroutine.rs index ac7a77fb2..0b2397878 100644 --- a/vm/src/obj/objcoroutine.rs +++ b/vm/src/obj/objcoroutine.rs @@ -1,5 +1,5 @@ use super::objcode::PyCodeRef; -use super::objcoroinner::Coro; +use super::objcoroinner::{Coro, Variant}; use super::objstr::PyStringRef; use super::objtype::PyClassRef; use crate::frame::FrameRef; @@ -29,7 +29,7 @@ impl PyCoroutine { pub fn new(frame: FrameRef, vm: &VirtualMachine) -> PyCoroutineRef { PyCoroutine { - inner: Coro::new(frame), + inner: Coro::new(frame, Variant::Coroutine), } .into_ref(vm) } diff --git a/vm/src/obj/objgenerator.rs b/vm/src/obj/objgenerator.rs index 45b81de99..1a2e7d62d 100644 --- a/vm/src/obj/objgenerator.rs +++ b/vm/src/obj/objgenerator.rs @@ -3,7 +3,7 @@ */ use super::objcode::PyCodeRef; -use super::objcoroinner::Coro; +use super::objcoroinner::{Coro, Variant}; use super::objtype::PyClassRef; use crate::frame::FrameRef; use crate::function::OptionalArg; @@ -32,7 +32,7 @@ impl PyGenerator { pub fn new(frame: FrameRef, vm: &VirtualMachine) -> PyGeneratorRef { PyGenerator { - inner: Coro::new(frame), + inner: Coro::new(frame, Variant::Gen), } .into_ref(vm) } From 08593cd9f3b5ceec8966626d6813b21fcb747ed3 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sat, 7 Mar 2020 14:54:18 -0600 Subject: [PATCH 08/10] Do some more refactoring, but better --- vm/src/obj/objasyncgenerator.rs | 76 ++++++++++++++++++++------------- vm/src/obj/objcoroinner.rs | 58 +++++++++++++++---------- vm/src/obj/objcoroutine.rs | 6 +++ vm/src/obj/objgenerator.rs | 6 +++ 4 files changed, 94 insertions(+), 52 deletions(-) diff --git a/vm/src/obj/objasyncgenerator.rs b/vm/src/obj/objasyncgenerator.rs index 40bc4f144..9a552aca2 100644 --- a/vm/src/obj/objasyncgenerator.rs +++ b/vm/src/obj/objasyncgenerator.rs @@ -1,6 +1,7 @@ use super::objcode::PyCodeRef; use super::objcoroinner::{Coro, Variant}; use super::objtype::{self, PyClassRef}; +use crate::exceptions::PyBaseExceptionRef; use crate::frame::FrameRef; use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; @@ -36,6 +37,12 @@ impl PyAsyncGen { .into_ref(vm) } + // TODO: fix function names situation + #[pyproperty(magic)] + fn name(&self, vm: &VirtualMachine) -> PyObjectRef { + vm.get_none() + } + #[pymethod(name = "__aiter__")] fn aiter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { zelf @@ -297,7 +304,7 @@ impl PyAsyncGenAThrow { } if !vm.is_none(&val) { return Err(vm.new_runtime_error( - "can't send non-None value to a just-started coroutine".to_owned(), + "can't send non-None value to a just-started async generator".to_owned(), )); } self.state.set(AwaitableState::Iter); @@ -306,18 +313,26 @@ impl PyAsyncGenAThrow { let (ty, val, tb) = self.value.clone(); let ret = self.ag.inner.throw(ty, val, tb, vm); let ret = if self.aclose { - self.check_no_ignore(&ret, vm)?; - ret + if self.ignored_close(&ret) { + Err(self.yield_close(vm)) + } else { + ret + } } else { PyAsyncGenWrappedValue::unbox(&self.ag, ret, vm) }; - self.check_error(ret, vm) + ret.map_err(|e| self.check_error(e, vm)) } AwaitableState::Iter => { let ret = self.ag.inner.send(val, vm); if self.aclose { - self.check_no_ignore(&ret, vm)?; - self.check_error(ret, vm) + match ret { + Ok(v) if v.payload_is::() => { + Err(self.yield_close(vm)) + } + Ok(v) => Ok(v), + Err(e) => Err(self.check_error(e, vm)), + } } else { PyAsyncGenWrappedValue::unbox(&self.ag, ret, vm) } @@ -339,12 +354,16 @@ impl PyAsyncGenAThrow { exc_tb.unwrap_or_else(|| vm.get_none()), vm, ); - if self.aclose { - self.check_no_ignore(&ret, vm)?; - self.check_error(ret, vm) + let res = if self.aclose { + if self.ignored_close(&ret) { + Err(self.yield_close(vm)) + } else { + ret + } } else { PyAsyncGenWrappedValue::unbox(&self.ag, ret, vm) - } + }; + res.map_err(|e| self.check_error(e, vm)) } #[pymethod] @@ -352,27 +371,26 @@ impl PyAsyncGenAThrow { self.state.set(AwaitableState::Closed); } - fn check_no_ignore(&self, res: &PyResult, vm: &VirtualMachine) -> PyResult<()> { - if let Ok(ref v) = res { - if v.payload_is::() { - return Err( - vm.new_runtime_error("async generator ignored GeneratorExit".to_owned()) - ); - } - } - Ok(()) + fn ignored_close(&self, res: &PyResult) -> bool { + res.as_ref() + .map_or(false, |v| v.payload_is::()) } - fn check_error(&self, res: PyResult, vm: &VirtualMachine) -> PyResult { - if self.aclose { - if let Err(ref e) = res { - if objtype::isinstance(&e, &vm.ctx.exceptions.stop_async_iteration) - || objtype::isinstance(&e, &vm.ctx.exceptions.generator_exit) - { - return Err(vm.new_exception_empty(vm.ctx.exceptions.stop_iteration.clone())); - } - } + fn yield_close(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + self.ag.running_async.set(false); + self.state.set(AwaitableState::Closed); + vm.new_runtime_error("async generator ignored GeneratorExit".to_owned()) + } + fn check_error(&self, exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyBaseExceptionRef { + self.ag.running_async.set(false); + self.state.set(AwaitableState::Closed); + if self.aclose + && (objtype::isinstance(&exc, &vm.ctx.exceptions.stop_async_iteration) + || objtype::isinstance(&exc, &vm.ctx.exceptions.generator_exit)) + { + vm.new_exception_empty(vm.ctx.exceptions.stop_iteration.clone()) + } else { + exc } - res } } diff --git a/vm/src/obj/objcoroinner.rs b/vm/src/obj/objcoroinner.rs index 51ab8f12d..7f63da5b4 100644 --- a/vm/src/obj/objcoroinner.rs +++ b/vm/src/obj/objcoroinner.rs @@ -1,4 +1,4 @@ -use super::{objiter, objtype}; +use super::objtype; use crate::exceptions::{self, PyBaseExceptionRef}; use crate::frame::{ExecutionResult, FrameRef}; use crate::pyobject::{PyObjectRef, PyResult}; @@ -71,38 +71,47 @@ impl Coro { ); self.running.set(false); self.started.set(true); - if let Err(ref e) = result { - if objtype::isinstance(e, &vm.ctx.exceptions.stop_iteration) { - let err = - vm.new_runtime_error(format!("{} raised StopIteration", self.variant.name())); - err.set_cause(Some(e.clone())); - return Err(err); - } - if self.variant == Variant::AsyncGen - && objtype::isinstance(e, &vm.ctx.exceptions.stop_async_iteration) - { - let err = - vm.new_runtime_error("async generator raise StopAsyncIteration".to_owned()); - err.set_cause(Some(e.clone())); - return Err(err); - } - } result } pub fn send(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { if self.closed.get() { - return Err(objiter::new_stop_iteration(vm)); + let cls = if let Variant::AsyncGen = self.variant { + vm.ctx.exceptions.stop_async_iteration.clone() + } else { + vm.ctx.exceptions.stop_iteration.clone() + }; + return Err(vm.new_exception_empty(cls)); } if !self.started.get() && !vm.is_none(&value) { - return Err(vm.new_type_error( - "can't send non-None value to a just-started coroutine".to_string(), - )); + return Err(vm.new_type_error(format!( + "can't send non-None value to a just-started {}", + self.variant.name() + ))); } self.frame.push_value(value.clone()); let result = self.run_with_context(|| vm.run_frame(self.frame.clone()), vm); self.maybe_close(&result); - self.variant.exec_result(result?, vm) + match result { + Ok(exec_res) => self.variant.exec_result(exec_res, vm), + Err(e) => { + if objtype::isinstance(&e, &vm.ctx.exceptions.stop_iteration) { + let err = vm + .new_runtime_error(format!("{} raised StopIteration", self.variant.name())); + err.set_cause(Some(e)); + Err(err) + } else if self.variant == Variant::AsyncGen + && objtype::isinstance(&e, &vm.ctx.exceptions.stop_async_iteration) + { + let err = vm + .new_runtime_error("async generator raised StopAsyncIteration".to_owned()); + err.set_cause(Some(e)); + Err(err) + } else { + Err(e) + } + } + } } pub fn throw( &self, @@ -142,7 +151,7 @@ impl Coro { self.closed.set(true); match result { Ok(ExecutionResult::Yield(_)) => { - Err(vm.new_runtime_error("generator ignored GeneratorExit".to_owned())) + Err(vm.new_runtime_error(format!("{} ignored GeneratorExit", self.variant.name()))) } Err(e) if !is_gen_exit(&e, vm) => Err(e), _ => Ok(()), @@ -161,6 +170,9 @@ impl Coro { pub fn frame(&self) -> FrameRef { self.frame.clone() } + pub fn name(&self) -> String { + self.frame.code.obj_name.clone() + } } pub fn is_gen_exit(exc: &PyBaseExceptionRef, vm: &VirtualMachine) -> bool { diff --git a/vm/src/obj/objcoroutine.rs b/vm/src/obj/objcoroutine.rs index 0b2397878..d1faacc2b 100644 --- a/vm/src/obj/objcoroutine.rs +++ b/vm/src/obj/objcoroutine.rs @@ -34,6 +34,12 @@ impl PyCoroutine { .into_ref(vm) } + // TODO: fix function names situation + #[pyproperty(magic)] + fn name(&self, vm: &VirtualMachine) -> PyObjectRef { + vm.get_none() + } + #[pymethod] fn send(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner.send(value, vm) diff --git a/vm/src/obj/objgenerator.rs b/vm/src/obj/objgenerator.rs index 1a2e7d62d..e91e24406 100644 --- a/vm/src/obj/objgenerator.rs +++ b/vm/src/obj/objgenerator.rs @@ -37,6 +37,12 @@ impl PyGenerator { .into_ref(vm) } + // TODO: fix function names situation + #[pyproperty(magic)] + fn name(&self, vm: &VirtualMachine) -> PyObjectRef { + vm.get_none() + } + #[pymethod(name = "__iter__")] fn iter(zelf: PyGeneratorRef) -> PyGeneratorRef { zelf From 888ec28fbf33466a05ec93487f66afb95cd18b33 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sat, 7 Mar 2020 14:54:31 -0600 Subject: [PATCH 09/10] Mark unsupported tests --- Lib/test/test_asyncgen.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 102089816..ab3743d08 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -179,6 +179,7 @@ class AsyncGenTest(unittest.TestCase): else: self.fail('StopAsyncIteration was not raised') + @unittest.skip("TODO: RUSTPYTHON") def test_async_gen_exception_03(self): async def gen(): await awaitable() @@ -342,6 +343,8 @@ class AsyncGenTest(unittest.TestCase): self.compare_generators(sync_gen_wrapper(), async_gen_wrapper()) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_async_gen_api_01(self): async def gen(): yield 123 @@ -644,6 +647,8 @@ class AsyncGenAsyncioTest(unittest.TestCase): fut.cancel() self.loop.run_until_complete(asyncio.sleep(0.01)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_async_gen_asyncio_gc_aclose_09(self): DONE = 0 @@ -738,6 +743,8 @@ class AsyncGenAsyncioTest(unittest.TestCase): self.loop.run_until_complete(run()) self.assertEqual(DONE, 10) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_async_gen_asyncio_aclose_12(self): DONE = 0 @@ -839,6 +846,8 @@ class AsyncGenAsyncioTest(unittest.TestCase): self.loop.run_until_complete(run()) self.assertEqual(DONE, 1) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_async_gen_asyncio_asend_03(self): DONE = 0 @@ -926,6 +935,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): self.loop.run_until_complete(run()) self.assertEqual(DONE, 1) + @unittest.skip("TODO: RUSTPYTHON") def test_async_gen_asyncio_athrow_02(self): DONE = 0 @@ -1044,6 +1054,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): self.loop.run_until_complete(run()) + @unittest.skip("TODO: RUSTPYTHON") def test_async_gen_asyncio_shutdown_01(self): finalized = 0 From 78f8f1400307028dc06adcc4a8b429bfd7d5c59f Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Fri, 13 Mar 2020 08:16:27 -0500 Subject: [PATCH 10/10] Apply review comments --- Cargo.lock | 39 ++++++++++++++++++++++++--------- compiler/src/compile.rs | 4 +--- vm/src/obj/objasyncgenerator.rs | 4 ++-- vm/src/obj/objcoroinner.rs | 15 +++++++------ 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb385df35..b2749b3cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -799,8 +799,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.67" -source = "git+https://github.com/rust-lang/libc#64b919f1d4cf601cf016350673564e4b364b13e5" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea0c0405123bba743ee3f91f49b1c7cfb684eef0da0a50110f758ccf24cdff0" [[package]] name = "libz-sys" @@ -890,10 +891,24 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" +[[package]] +name = "nix" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c722bee1037d430d0f8e687bbdbf222f27cc6e4e68d5caf630857bb2b6dbdce" +dependencies = [ + "bitflags", + "cc", + "cfg-if", + "libc", + "void", +] + [[package]] name = "nix" version = "0.17.0" -source = "git+https://github.com/coolreader18/nix?branch=add-redox-support#318603039bf1f7e1e56b4c1ba96fbab42d9a9aca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363" dependencies = [ "bitflags", "cc", @@ -1515,7 +1530,7 @@ dependencies = [ "maplit", "md-5", "mt19937", - "nix", + "nix 0.17.0", "num-bigint", "num-complex", "num-integer", @@ -1582,14 +1597,15 @@ dependencies = [ [[package]] name = "rustyline" version = "6.0.0" -source = "git+https://github.com/kkawakam/rustyline#91ea17d8143c0bcf8326b656428a3d959627308a" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de64be8eecbe428b6924f1d8430369a01719fbb182c26fa431ddbb0a95f5315d" dependencies = [ "cfg-if", "dirs 2.0.1", "libc", "log", "memchr", - "nix", + "nix 0.14.1", "unicode-segmentation", "unicode-width", "utf8parse", @@ -1712,7 +1728,8 @@ checksum = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" [[package]] name = "socket2" version = "0.3.11" -source = "git+https://github.com/alexcrichton/socket2-rs?rev=75fe3f2fe0e3079ad9d4dfb3a19e60bea3f2f055#75fe3f2fe0e3079ad9d4dfb3a19e60bea3f2f055" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b74de517221a2cb01a53349cf54182acdc31a074727d3079068448c0676d85" dependencies = [ "cfg-if", "libc", @@ -1889,9 +1906,11 @@ dependencies = [ [[package]] name = "time" version = "0.1.42" -source = "git+https://gitlab.redox-os.org/redox-os/time.git?branch=redox-unix#726efec5f1c47ef89ab740335439ff4fda047f8f" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" dependencies = [ "libc", + "redox_syscall", "winapi", ] @@ -2253,9 +2272,9 @@ checksum = "a7a928b876ff873d4a0ac966acce72423879dd86afcf190017aa700207188078" [[package]] name = "utf8parse" -version = "0.2.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" +checksum = "8772a4ccbb4e89959023bc5b7cb8623a795caa7092d99f3aa9501b9484d4557d" [[package]] name = "vcpkg" diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index 29a876f5f..8f88965b7 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -1551,8 +1551,6 @@ impl Compiler { self.set_source_location(expression.location); use ast::ExpressionType::*; - #[allow(unused_imports)] // not unused, overrides ast::ExpressionType::None - use Option::None; match &expression.node { Call { function, @@ -1647,7 +1645,7 @@ impl Compiler { self.mark_generator(); match value { Some(expression) => self.compile_expression(expression)?, - None => self.emit(Instruction::LoadConst { + Option::None => self.emit(Instruction::LoadConst { value: bytecode::Constant::None, }), }; diff --git a/vm/src/obj/objasyncgenerator.rs b/vm/src/obj/objasyncgenerator.rs index 9a552aca2..e209040d0 100644 --- a/vm/src/obj/objasyncgenerator.rs +++ b/vm/src/obj/objasyncgenerator.rs @@ -212,7 +212,7 @@ impl PyAsyncGenASend { let res = self.ag.inner.send(val, vm); let res = PyAsyncGenWrappedValue::unbox(&self.ag, res, vm); if res.is_err() { - self.state.set(AwaitableState::Closed); + self.close(); } res } @@ -239,7 +239,7 @@ impl PyAsyncGenASend { ); let res = PyAsyncGenWrappedValue::unbox(&self.ag, res, vm); if res.is_err() { - self.state.set(AwaitableState::Closed); + self.close(); } res } diff --git a/vm/src/obj/objcoroinner.rs b/vm/src/obj/objcoroinner.rs index 7f63da5b4..97cd810e8 100644 --- a/vm/src/obj/objcoroinner.rs +++ b/vm/src/obj/objcoroinner.rs @@ -1,4 +1,4 @@ -use super::objtype; +use super::objtype::{self, PyClassRef}; use crate::exceptions::{self, PyBaseExceptionRef}; use crate::frame::{ExecutionResult, FrameRef}; use crate::pyobject::{PyObjectRef, PyResult}; @@ -23,6 +23,12 @@ impl Variant { Self::AsyncGen => "async generator", } } + fn stop_iteration(self, vm: &VirtualMachine) -> PyClassRef { + match self { + Self::AsyncGen => vm.ctx.exceptions.stop_async_iteration.clone(), + _ => vm.ctx.exceptions.stop_iteration.clone(), + } + } } #[derive(Debug)] @@ -76,12 +82,7 @@ impl Coro { pub fn send(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { if self.closed.get() { - let cls = if let Variant::AsyncGen = self.variant { - vm.ctx.exceptions.stop_async_iteration.clone() - } else { - vm.ctx.exceptions.stop_iteration.clone() - }; - return Err(vm.new_exception_empty(cls)); + return Err(vm.new_exception_empty(self.variant.stop_iteration(vm))); } if !self.started.get() && !vm.is_none(&value) { return Err(vm.new_type_error(format!(