From c54b4bb833e89d858bcc403dde22b00fddb1dde2 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sat, 5 Sep 2020 16:10:39 +1200 Subject: [PATCH] Split py_compile_bytecode into py_compile and py_freeze. --- Cargo.lock | 12 +++- derive/Cargo.toml | 1 + derive/src/compile_bytecode.rs | 100 +++++++++++++++++++++++++-------- derive/src/lib.rs | 9 ++- examples/freeze/main.rs | 2 +- examples/mini_repl.rs | 12 +--- vm/pylib-crate/src/lib.rs | 2 +- vm/src/frozen.rs | 2 +- wasm/lib/src/browser_module.rs | 7 +-- 9 files changed, 104 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a207d4c8..374231ad8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -313,7 +313,7 @@ dependencies = [ "atty", "bitflags", "strsim 0.8.0", - "textwrap", + "textwrap 0.11.0", "unicode-width", "vec_map", ] @@ -1649,6 +1649,7 @@ dependencies = [ "rustpython-compiler", "syn", "syn-ext", + "textwrap 0.12.1", ] [[package]] @@ -2062,6 +2063,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "textwrap" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789" +dependencies = [ + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.20" diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 89fd2c03e..61c019918 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -19,3 +19,4 @@ rustpython-compiler = { path = "../compiler", version = "0.1.1" } rustpython-bytecode = { path = "../bytecode", version = "0.1.1" } maplit = "1.0" once_cell = "1.3.1" +textwrap = "0.12.1" diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs index cec3cb605..3b3f2d0c5 100644 --- a/derive/src/compile_bytecode.rs +++ b/derive/src/compile_bytecode.rs @@ -1,6 +1,6 @@ //! Parsing and processing for this form: //! ```ignore -//! py_compile_input!( +//! py_compile!( //! // either: //! source = "python_source_code", //! // or @@ -24,7 +24,8 @@ use std::env; use std::fs; use std::path::{Path, PathBuf}; use syn::parse::{Parse, ParseStream, Result as ParseResult}; -use syn::{self, parse2, Lit, LitByteStr, LitStr, Meta, Token}; +use syn::spanned::Spanned; +use syn::{self, parse2, Lit, LitByteStr, LitStr, Macro, Meta, MetaNameValue, Token}; static CARGO_MANIFEST_DIR: Lazy = Lazy::new(|| { PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present")) @@ -62,7 +63,25 @@ impl CompilationSource { mode: compile::Mode, module_name: String, ) -> Result, Diagnostic> { - Ok(match &self.kind { + match &self.kind { + CompilationSourceKind::Dir(rel_path) => { + self.compile_dir(&CARGO_MANIFEST_DIR.join(rel_path), String::new(), mode) + } + _ => Ok(hashmap! { + module_name.clone() => FrozenModule { + code: self.compile_single(mode, module_name)?, + package: false, + }, + }), + } + } + + fn compile_single( + &self, + mode: compile::Mode, + module_name: String, + ) -> Result { + match &self.kind { CompilationSourceKind::File(rel_path) => { let path = CARGO_MANIFEST_DIR.join(rel_path); let source = fs::read_to_string(&path).map_err(|err| { @@ -71,25 +90,17 @@ impl CompilationSource { format!("Error reading file {:?}: {}", path, err), ) })?; - hashmap! { - module_name.clone() => FrozenModule { - code: self.compile_string(&source, mode, module_name, || rel_path.display())?, - package: false, - }, - } + self.compile_string(&source, mode, module_name, || rel_path.display()) } CompilationSourceKind::SourceCode(code) => { - hashmap! { - module_name.clone() => FrozenModule { - code: self.compile_string(code, mode, module_name, || "string literal")?, - package: false, - }, - } + self.compile_string(&textwrap::dedent(code), mode, module_name, || { + "string literal" + }) } - CompilationSourceKind::Dir(rel_path) => { - self.compile_dir(&CARGO_MANIFEST_DIR.join(rel_path), String::new(), mode)? + CompilationSourceKind::Dir(_) => { + unreachable!("Can't use compile_single with directory source") } - }) + } } fn compile_dir( @@ -157,7 +168,7 @@ struct PyCompileInput { } impl PyCompileInput { - fn parse(&self) -> Result { + fn parse(&self, allow_dir: bool) -> Result { let mut module_name = None; let mut mode = None; let mut source: Option = None; @@ -214,6 +225,10 @@ impl PyCompileInput { span: extract_spans(&name_value).unwrap(), }); } else if ident == "dir" { + if !allow_dir { + bail_span!(ident, "py_compile doesn't accept dir") + } + assert_source_empty(&source)?; let path = match &name_value.lit { Lit::Str(s) => PathBuf::from(s.value()), @@ -236,7 +251,7 @@ impl PyCompileInput { let source = source.ok_or_else(|| { Diagnostic::span_error( self.span, - "Must have either file or source in py_compile_bytecode!()", + "Must have either file or source in py_compile!()/py_freeze!()", ) })?; @@ -249,11 +264,32 @@ impl PyCompileInput { } } +fn parse_meta(input: ParseStream) -> ParseResult { + let path = input.call(syn::Path::parse_mod_style)?; + let eq_token: Token![=] = input.parse()?; + let span = input.span(); + if input.peek(LitStr) { + Ok(Meta::NameValue(MetaNameValue { + path, + eq_token, + lit: Lit::Str(input.parse()?), + })) + } else if let Ok(mac) = input.parse::() { + Ok(Meta::NameValue(MetaNameValue { + path, + eq_token, + lit: Lit::Str(LitStr::new(&mac.tokens.to_string(), mac.span())), + })) + } else { + Err(syn::Error::new(span, "Expected string or stringify macro")) + } +} + impl Parse for PyCompileInput { fn parse(input: ParseStream) -> ParseResult { let span = input.cursor().span(); let metas = input - .parse_terminated::(Meta::parse)? + .parse_terminated::(parse_meta)? .into_iter() .collect(); Ok(PyCompileInput { span, metas }) @@ -267,9 +303,27 @@ struct PyCompileArgs { crate_name: syn::Ident, } -pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result { +pub fn impl_py_compile(input: TokenStream2) -> Result { let input: PyCompileInput = parse2(input)?; - let args = input.parse()?; + let args = input.parse(false)?; + + let crate_name = args.crate_name; + let code = args.source.compile_single(args.mode, args.module_name)?; + + let bytes = code.to_bytes(); + let bytes = LitByteStr::new(&bytes, Span::call_site()); + + let output = quote! { + ::#crate_name::bytecode::CodeObject::from_bytes(#bytes) + .expect("Deserializing CodeObject failed") + }; + + Ok(output) +} + +pub fn impl_py_freeze(input: TokenStream2) -> Result { + let input: PyCompileInput = parse2(input)?; + let args = input.parse(true)?; let crate_name = args.crate_name; let code_map = args.source.compile(args.mode, args.module_name)?; diff --git a/derive/src/lib.rs b/derive/src/lib.rs index c09f5f29f..e3cc1c7f5 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -62,6 +62,11 @@ pub fn pystruct_sequence(input: TokenStream) -> TokenStream { } #[proc_macro] -pub fn py_compile_bytecode(input: TokenStream) -> TokenStream { - result_to_tokens(compile_bytecode::impl_py_compile_bytecode(input.into())) +pub fn py_compile(input: TokenStream) -> TokenStream { + result_to_tokens(compile_bytecode::impl_py_compile(input.into())) +} + +#[proc_macro] +pub fn py_freeze(input: TokenStream) -> TokenStream { + result_to_tokens(compile_bytecode::impl_py_freeze(input.into())) } diff --git a/examples/freeze/main.rs b/examples/freeze/main.rs index b04d30e67..0923c52b2 100644 --- a/examples/freeze/main.rs +++ b/examples/freeze/main.rs @@ -12,7 +12,7 @@ fn run(vm: &vm::VirtualMachine) -> vm::pyobject::PyResult<()> { // the file parameter is relevant to the directory where the crate's Cargo.toml is located, see $CARGO_MANIFEST_DIR: // https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates let modules: HashMap = - vm::py_compile_bytecode!(file = "examples/freeze/freeze.py"); + vm::py_freeze!(file = "examples/freeze/freeze.py"); let res = vm.run_code_obj( vm.ctx diff --git a/examples/mini_repl.rs b/examples/mini_repl.rs index 8565496ab..d1cb00c15 100644 --- a/examples/mini_repl.rs +++ b/examples/mini_repl.rs @@ -10,7 +10,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; // this needs to be in scope in order to insert things into scope.globals use vm::pyobject::ItemProtocol; -// This has to be a macro because it uses the py_compile_bytecode macro, +// This has to be a macro because it uses the py_compile macro, // which compiles python source to optimized bytecode at compile time, so that // the program you're embedding this into doesn't take longer to start up. macro_rules! add_python_function { @@ -19,15 +19,7 @@ macro_rules! add_python_function { // (a PyRef is a special reference that points to something in the VirtualMachine) use vm::pyobject::PyValue; - // you can safely assume that only one module will be created when passing a source literal - // to py_compile_bytecode. However, it is also possible to pass directories, which may - // return more modules. - let (_, vm::bytecode::FrozenModule { code, .. }): (String, _) = - vm::py_compile_bytecode!(source = $src) - .into_iter() - .collect::>() - .pop() - .expect("No modules found in the provided source!"); + let code = vm::py_compile!(source = $src); // takes the first constant in the file that's a function let def = code diff --git a/vm/pylib-crate/src/lib.rs b/vm/pylib-crate/src/lib.rs index 98059df0b..9c90420f6 100644 --- a/vm/pylib-crate/src/lib.rs +++ b/vm/pylib-crate/src/lib.rs @@ -13,5 +13,5 @@ use { }; #[cfg(feature = "compiled-bytecode")] pub fn frozen_stdlib() -> HashMap { - rustpython_derive::py_compile_bytecode!(dir = "Lib", crate_name = "rustpython_pylib") + rustpython_derive::py_freeze!(dir = "Lib", crate_name = "rustpython_pylib") } diff --git a/vm/src/frozen.rs b/vm/src/frozen.rs index 379e90b8e..937a1ddc4 100644 --- a/vm/src/frozen.rs +++ b/vm/src/frozen.rs @@ -6,7 +6,7 @@ pub fn get_module_inits() -> HashMap { macro_rules! ext_modules { ($($t:tt)*) => { - modules.extend(py_compile_bytecode!($($t)*)); + modules.extend(py_freeze!($($t)*)); }; } diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index b260e1b0f..484b042b0 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -382,8 +382,7 @@ pub fn setup_browser_module(vm: &mut VirtualMachine) { state .stdlib_inits .insert("_browser".to_owned(), Box::new(make_module)); - state.frozen.extend(py_compile_bytecode!( - file = "src/browser.py", - module_name = "browser", - )); + state + .frozen + .extend(py_freeze!(file = "src/browser.py", module_name = "browser",)); }