From d382009a5fa52d2b3d2a8f852a4a7df01b94148c Mon Sep 17 00:00:00 2001 From: Yossi Konstantinovsky Date: Thu, 15 Jul 2021 14:15:09 +0300 Subject: [PATCH] add initial pyexpat implementation --- Cargo.lock | 7 ++ Lib/plistlib.py | 3 +- vm/Cargo.toml | 1 + vm/src/stdlib/mod.rs | 2 + vm/src/stdlib/pyexpat.rs | 205 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 216 insertions(+), 2 deletions(-) create mode 100644 vm/src/stdlib/pyexpat.rs diff --git a/Cargo.lock b/Cargo.lock index 26a0c90e3..75980386c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2112,6 +2112,7 @@ dependencies = [ "widestring", "winapi", "winreg", + "xml-rs", ] [[package]] @@ -2960,6 +2961,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "xml-rs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 282d7af6b..3f8263b92 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -62,8 +62,7 @@ import os import re import struct from warnings import warn -# XXX RUSTPYTHON TODO: pyexpat -# from xml.parsers.expat import ParserCreate +from xml.parsers.expat import ParserCreate PlistFormat = enum.Enum('PlistFormat', 'FMT_XML FMT_BINARY', module=__name__) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 1e4983e24..ee8359595 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -34,6 +34,7 @@ sha-1 = "0.9.4" sha2 = "0.9.3" sha3 = "0.9.1" blake2 = "0.9.1" +xml-rs = "0.8" num-complex = { version = "0.4.0", features = ["serde"] } num-bigint = { version = "0.4.0", features = ["serde"] } diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 46a70298d..8f4cb7d92 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -26,6 +26,7 @@ mod marshal; mod math; mod operator; mod platform; +mod pyexpat; pub(crate) mod pystruct; mod random; // TODO: maybe make this an extension module, if we ever get those @@ -119,6 +120,7 @@ pub fn get_module_inits() -> StdlibMap { "marshal" => marshal::make_module, "math" => math::make_module, "_operator" => operator::make_module, + "pyexpat" => pyexpat::make_module, "_platform" => platform::make_module, "_random" => random::make_module, "_sre" => sre::make_module, diff --git a/vm/src/stdlib/pyexpat.rs b/vm/src/stdlib/pyexpat.rs new file mode 100644 index 000000000..3f24ad103 --- /dev/null +++ b/vm/src/stdlib/pyexpat.rs @@ -0,0 +1,205 @@ +/* Pyexpat builtin module +* +* +*/ + +use crate::vm::VirtualMachine; +use crate::PyObjectRef; + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let module = _pyexpat::make_module(vm); + + extend_module!(vm, module, { + "errors" => _errors::make_module(vm), + "model" => _model::make_module(vm) + }); + + module +} + +#[pymodule(name = "pyexpat")] +mod _pyexpat { + use crate::builtins::{PyStr, PyStrRef, PyTypeRef}; + use crate::byteslike::PyBytesLike; + use crate::function::{IntoFuncArgs, OptionalArg}; + use crate::pyobject::StaticType; + use crate::{ + IntoPyObject, IntoPyRef, ItemProtocol, PyObjectRef, PyRef, PyResult, PyValue, + TryFromObject, VirtualMachine, + }; + + use rustpython_common::lock::PyRwLock; + use std::io::Cursor; + use xml::reader::XmlEvent; + type MutableObject = PyRwLock; + + #[pyattr] + #[pyclass(name = "xmlparser", module = false)] + #[derive(Debug)] + struct PyExpatLikeXmlParser { + start_element: MutableObject, + end_element: MutableObject, + character_data: MutableObject, + entity_decl: MutableObject, + } + type PyExpatLikeXmlParserRef = PyRef; + + impl PyValue for PyExpatLikeXmlParser { + fn class(_vm: &VirtualMachine) -> &PyTypeRef { + Self::static_type() + } + } + + #[inline] + fn invoke_handler(vm: &VirtualMachine, handler: &MutableObject, args: T) + where + T: IntoFuncArgs, + { + vm.invoke(&handler.read().clone(), args).ok(); + } + + #[pyimpl] + impl PyExpatLikeXmlParser { + fn new(vm: &VirtualMachine) -> PyResult { + Ok(PyExpatLikeXmlParser { + start_element: MutableObject::new(vm.ctx.none()), + end_element: MutableObject::new(vm.ctx.none()), + character_data: MutableObject::new(vm.ctx.none()), + entity_decl: MutableObject::new(vm.ctx.none()), + } + .into_ref(vm)) + } + + #[pyproperty(name = "StartElementHandler")] + fn start_element_handler(&self) -> PyObjectRef { + self.start_element.read().clone() + } + + #[pyproperty(setter, name = "StartElementHandler")] + fn set_start_element_handler(&self, func: PyObjectRef) { + let mut handler = self.start_element.write(); + *handler = func; + } + + #[pyproperty(name = "EndElementHandler")] + fn end_element_handler(&self) -> PyObjectRef { + self.end_element.read().clone() + } + + #[pyproperty(setter, name = "EndElementHandler")] + fn set_end_element_handler(&self, func: PyObjectRef) { + let mut handler = self.end_element.write(); + *handler = func; + } + + #[pyproperty(name = "CharacterDataHandler")] + fn character_data_handler(&self) -> PyObjectRef { + self.character_data.read().clone() + } + + #[pyproperty(setter, name = "CharacterDataHandler")] + fn set_character_data_handler(&self, func: PyObjectRef) { + let mut handler = self.character_data.write(); + *handler = func; + } + + #[pyproperty(name = "EntityDeclHandler")] + fn entity_decl(&self) -> PyObjectRef { + self.entity_decl.read().clone() + } + + #[pyproperty(setter, name = "EntityDeclHandler")] + fn set_decl_handler(&self, func: PyObjectRef) { + let mut handler = self.entity_decl.write(); + *handler = func; + } + + fn create_config(&self) -> xml::ParserConfig { + xml::ParserConfig::new() + .cdata_to_characters(true) + .coalesce_characters(false) + .whitespace_to_characters(true) + } + + fn do_parse(&self, vm: &VirtualMachine, parser: xml::EventReader) + where + T: std::io::Read, + { + for e in parser { + match e { + Ok(XmlEvent::StartElement { + name, attributes, .. + }) => { + let dict = vm.ctx.new_dict(); + for attribute in attributes { + dict.set_item( + attribute.name.local_name.as_str(), + vm.ctx.new_str(attribute.value), + vm, + ) + .unwrap(); + } + + let name_str = PyStr::from(name.local_name).into_ref(vm); + invoke_handler(vm, &self.start_element, (name_str, dict.into_object())); + } + Ok(XmlEvent::EndElement { name, .. }) => { + let name_str = PyStr::from(name.local_name).into_ref(vm); + invoke_handler(vm, &self.end_element, (name_str,)); + } + Ok(XmlEvent::Characters(chars)) => { + let str = PyStr::from(chars).into_ref(vm); + invoke_handler(vm, &self.character_data, (str,)); + } + _ => {} + } + } + } + + #[pymethod(name = "Parse")] + fn parse(&self, data: PyStrRef, _isfinal: OptionalArg, vm: &VirtualMachine) { + let reader = Cursor::>::new(data.as_str().as_bytes().to_vec()); + let parser = self.create_config().create_reader(reader); + self.do_parse(vm, parser); + } + + #[pymethod(name = "ParseFile")] + fn parse_file(&self, file: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // todo: read chunks at a time + let read_res = vm.call_method(&file, "read", ())?; + let bytes_like = PyBytesLike::try_from_object(vm, read_res)?; + let buf = bytes_like.borrow_buf().to_vec(); + let reader = Cursor::new(buf); + let parser = self.create_config().create_reader(reader); + self.do_parse(vm, parser); + + // todo: return value + Ok(()) + } + } + + #[derive(FromArgs)] + #[allow(dead_code)] + struct ParserCreateArgs { + #[pyarg(any, optional)] + encoding: OptionalArg, + #[pyarg(any, optional)] + namespace_separator: OptionalArg, + #[pyarg(any, optional)] + intern: OptionalArg, + } + + #[pyfunction(name = "ParserCreate")] + fn parser_create( + _args: ParserCreateArgs, + vm: &VirtualMachine, + ) -> PyResult { + PyExpatLikeXmlParser::new(vm) + } +} + +#[pymodule(name = "model")] +mod _model {} + +#[pymodule(name = "errors")] +mod _errors {}