rework pylib to generate all frozen libs

Cargo.toml

@@ -19,7 +19,7 @@ members = [
 ]
 
 [features]
-default = ["threading", "pylib", "stdlib", "zlib", "importlib", "encodings"]
+default = ["threading", "stdlib", "zlib", "importlib", "encodings"]
 importlib = ["rustpython-vm/importlib"]
 encodings = ["rustpython-vm/encodings"]
 stdlib = ["rustpython-stdlib"]
@@ -27,7 +27,6 @@ flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
 freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
 jit = ["rustpython-vm/jit"]
 threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
-pylib = ["rustpython-vm/pylib"]
 zlib = ["stdlib", "rustpython-stdlib/zlib"]
 bz2 = ["stdlib", "rustpython-stdlib/bz2"]
 ssl = ["rustpython-stdlib/ssl"]

stdlib/Cargo.toml

@@ -81,8 +81,7 @@ page_size = "0.4"
 termios = "0.3.3"
 
 [features]
-default = ["compile-parse", "pylib"]
-pylib = ["rustpython-vm/pylib"]
+default = ["compile-parse"]
 compile-parse = ["rustpython-vm/compile-parse", "rustpython-parser"]
 threading = ["rustpython-common/threading", "rustpython-vm/threading"]
 parser = ["rustpython-vm/parser"]

vm/Cargo.toml

@@ -9,13 +9,12 @@ edition = "2021"
 include = ["src/**/*.rs", "Cargo.toml", "build.rs", "Lib/**/*.py"]
 
 [features]
-default = ["compile-parse", "pylib"]
+default = ["compile-parse"]
 importlib = []
 encodings = ["importlib"]
 vm-tracing-logging = []
 flame-it = ["flame", "flamer"]
-pylib = ["rustpython-pylib"]
-freeze-stdlib = ["rustpython-pylib", "rustpython-pylib/compiled-bytecode"]
+freeze-stdlib = ["rustpython-pylib/stdlib"]
 jit = ["rustpython-jit"]
 threading = ["rustpython-common/threading"]
 compile-parse = ["parser", "compiler"]
@@ -37,7 +36,7 @@ rustpython-compiler = { path = "../compiler/porcelain", optional = true, version
 rustpython-compiler-core = { path = "../compiler", optional = true, version = "0.1.2" }
 rustpython-bytecode = { path = "../bytecode", version = "0.1.2" }
 rustpython-jit = { path = "../jit", optional = true, version = "0.1.2" }
-rustpython-pylib = { path = "pylib-crate", optional = true, version = "0.1.0" }
+rustpython-pylib = { path = "pylib-crate", version = "0.1.0" }
 rand = "0.8.5"
 getrandom = { version = "0.2.6", features = ["js"] }
 log = "0.4.16"

vm/pylib-crate/Cargo.toml

@@ -9,11 +9,11 @@ edition = "2021"
 include = ["Cargo.toml", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"]
 
 [features]
-compiled-bytecode = ["rustpython-derive", "rustpython-bytecode"]
+stdlib = []
 
 [dependencies]
-rustpython-derive = { version = "0.1.2", path = "../../derive", optional = true }
-rustpython-bytecode = { version = "0.1.2", path = "../../bytecode", optional = true }
+rustpython-derive = { version = "0.1.2", path = "../../derive" }
+rustpython-bytecode = { version = "0.1.2", path = "../../bytecode" }
 
 [build-dependencies]
 glob = "0.3"

vm/pylib-crate/Lib

@@ -1 +0,0 @@
-../../Lib/

vm/pylib-crate/build.rs

@@ -1,9 +1,23 @@
 fn main() {
-    #[cfg(feature = "compiled-bytecode")]
-    for entry in glob::glob("../../Lib/**/*")
-        .expect("Lib/ exists?")
-        .flatten()
-    {
+    rerun_if_changed("../Lib/python_builtins/*");
+
+    #[cfg(not(feature = "stdlib"))]
+    rerun_if_changed("../Lib/core_modules/*");
+
+    #[cfg(feature = "stdlib")]
+    rerun_if_changed("../../Lib/**/*");
+
+    if cfg!(windows) {
+        if let Ok(real_path) = std::fs::read_to_string("Lib") {
+            println!("rustc-env:win_lib_path={:?}", real_path);
+        }
+    }
+}
+
+fn rerun_if_changed(pattern: &str) {
+    let glob =
+        glob::glob(pattern).unwrap_or_else(|e| panic!("failed to glob {:?}: {}", pattern, e));
+    for entry in glob.flatten() {
         if entry.is_dir() {
             continue;
         }
@@ -13,9 +27,4 @@ fn main() {
         }
         println!("cargo:rerun-if-changed={}", display);
     }
-    if cfg!(windows) {
-        if let Ok(real_path) = std::fs::read_to_string("Lib") {
-            println!("rustc-env:win_lib_path={:?}", real_path);
-        }
-    }
 }

vm/pylib-crate/src/lib.rs

@@ -6,12 +6,27 @@
 // so build.rs sets this env var
 pub const LIB_PATH: &str = match option_env!("win_lib_path") {
     Some(s) => s,
-    None => concat!(env!("CARGO_MANIFEST_DIR"), "/Lib"),
+    None => concat!(env!("CARGO_MANIFEST_DIR"), "/../../Lib"),
 };
 
-#[cfg(feature = "compiled-bytecode")]
 use rustpython_bytecode::FrozenModule;
-#[cfg(feature = "compiled-bytecode")]
+
+pub fn frozen_builtins() -> impl Iterator<Item = (String, FrozenModule)> {
+    rustpython_derive::py_freeze!(
+        dir = "../Lib/python_builtins",
+        crate_name = "rustpython_bytecode"
+    )
+}
+
+#[cfg(not(feature = "stdlib"))]
+pub fn frozen_core() -> impl Iterator<Item = (String, FrozenModule)> {
+    rustpython_derive::py_freeze!(
+        dir = "../Lib/core_modules",
+        crate_name = "rustpython_bytecode"
+    )
+}
+
+#[cfg(feature = "stdlib")]
 pub fn frozen_stdlib() -> impl Iterator<Item = (String, FrozenModule)> {
-    rustpython_derive::py_freeze!(dir = "Lib", crate_name = "rustpython_bytecode")
+    rustpython_derive::py_freeze!(dir = "../../Lib", crate_name = "rustpython_bytecode")
 }

vm/src/frozen.rs

@@ -20,12 +20,13 @@ pub fn get_module_inits() -> impl Iterator<Item = (String, FrozenModule)> {
     // Python modules that the vm calls into, but are not actually part of the stdlib. They could
     // in theory be implemented in Rust, but are easiest to do in Python for one reason or another.
     // Includes _importlib_bootstrap and _importlib_bootstrap_external
-    ext_modules!(iter, dir = "Lib/python_builtins/");
+    ext_modules!(iter, (rustpython_pylib::frozen_builtins()));
 
     #[cfg(not(feature = "freeze-stdlib"))]
     // core stdlib Python modules that the vm calls into, but are still used in Python
     // application code, e.g. copyreg
-    ext_modules!(iter, dir = "Lib/core_modules/");
+    ext_modules!(iter, (rustpython_pylib::frozen_core()));
+
     // if we're on freeze-stdlib, the core stdlib modules will be included anyway
     #[cfg(feature = "freeze-stdlib")]
     ext_modules!(iter, (rustpython_pylib::frozen_stdlib()));

vm/src/vm/setting.rs

@@ -89,10 +89,7 @@ impl Default for Settings {
             dev_mode: false,
             warn_default_encoding: false,
             warnopts: vec![],
-            path_list: vec![
-                #[cfg(all(feature = "pylib", not(feature = "freeze-stdlib")))]
-                rustpython_pylib::LIB_PATH.to_owned(),
-            ],
+            path_list: vec![rustpython_pylib::LIB_PATH.to_owned()],
             argv: vec![],
             hash_seed: None,
             stdio_unbuffered: false,