From 936826bd44671a250fddb45e665a382a5b452963 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Wed, 28 Apr 2021 09:22:17 -0500 Subject: [PATCH 1/4] Replace symlinks-to-hardlinks.ps1 hack with a nicer hack --- .github/workflows/ci.yaml | 8 +-- README.md | 2 - derive/src/compile_bytecode.rs | 97 ++++++++++++++++++++++++++----- scripts/symlinks-to-hardlinks.ps1 | 31 ---------- vm/src/frozen.rs | 1 - 5 files changed, 83 insertions(+), 56 deletions(-) delete mode 100644 scripts/symlinks-to-hardlinks.ps1 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6b5ce0a75..a7bea9f74 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,9 +29,7 @@ jobs: - uses: actions/checkout@master - uses: actions-rs/toolchain@v1 - name: Set up the Windows environment - run: | - choco install llvm - powershell.exe scripts/symlinks-to-hardlinks.ps1 + run: choco install llvm if: runner.os == 'Windows' - name: Set up the Mac environment run: brew install autoconf automake libtool @@ -62,9 +60,7 @@ jobs: with: python-version: 3.8 - name: Set up the Windows environment - run: | - choco install llvm - powershell.exe scripts/symlinks-to-hardlinks.ps1 + run: choco install llvm if: runner.os == 'Windows' - name: Set up the Mac environment run: brew install autoconf automake libtool diff --git a/README.md b/README.md index e632bd675..490334fd3 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,6 @@ To build RustPython locally, do the following: $ git clone https://github.com/RustPython/RustPython $ cd RustPython - # if you're on windows: - $ powershell scripts\symlinks-to-hardlinks.ps1 # --release is needed (at least on windows) to prevent stack overflow $ cargo run --release demo.py Hello, RustPython! diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs index 43d84c770..666ac754c 100644 --- a/derive/src/compile_bytecode.rs +++ b/derive/src/compile_bytecode.rs @@ -31,6 +31,47 @@ static CARGO_MANIFEST_DIR: Lazy = Lazy::new(|| { PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present")) }); +fn is_git_symlink(path: &Path) -> bool { + use std::io::{BufRead, BufReader}; + use std::process::{Command, Stdio}; + let res = path.parent().and_then(|dir| { + let child = Command::new("git") + .arg("-C") + .arg(dir) + .arg("ls-files") + .arg("-s") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .ok()?; + let git_root = Command::new("git") + .arg("-C") + .arg(dir) + .arg("rev-parse") + .arg("--show-toplevel") + .output() + .ok()? + .stdout; + let git_root = Path::new(std::str::from_utf8(&git_root).unwrap().trim()); + let stdout = BufReader::new(child.stdout.unwrap()); + let lines = stdout + .lines() + .filter_map(|line| line.ok()) + .filter(|line| line.starts_with("120000")); + let path = path.canonicalize().ok()?; + for line in lines { + let link_path = line.splitn(2, '\t').nth(1).unwrap(); + let link_path = git_root.join(link_path).canonicalize().ok()?; + if link_path == path { + return Some(true); + } + } + Some(false) + }); + res.unwrap_or(false) +} + enum CompilationSourceKind { /// Source is a File (Path) File(PathBuf), @@ -115,9 +156,20 @@ impl CompilationSource { mode: compile::Mode, ) -> Result, Diagnostic> { let mut code_map = HashMap::new(); - let paths = fs::read_dir(&path).map_err(|err| { - Diagnostic::spans_error(self.span, format!("Error listing dir {:?}: {}", path, err)) - })?; + let paths = fs::read_dir(path) + .or_else(|e| { + // handle git symlinks - git for windows will make it just a text file containing + // the target + if cfg!(windows) && is_git_symlink(path) { + if let Ok(real_path) = fs::read_to_string(path) { + return fs::read_dir(real_path.trim()); + } + } + Err(e) + }) + .map_err(|err| { + Diagnostic::spans_error(self.span, format!("Error listing dir {:?}: {}", path, err)) + })?; for path in paths { let path = path.map_err(|err| { Diagnostic::spans_error(self.span, format!("Failed to list file: {}", err)) @@ -133,12 +185,6 @@ impl CompilationSource { mode, )?); } else if file_name.ends_with(".py") { - let source = fs::read_to_string(&path).map_err(|err| { - Diagnostic::spans_error( - self.span, - format!("Error reading file {:?}: {}", path, err), - ) - })?; let stem = path.file_stem().unwrap().to_str().unwrap(); let is_init = stem == "__init__"; let module_name = if is_init { @@ -148,15 +194,34 @@ impl CompilationSource { } else { format!("{}.{}", parent, stem) }; + + let compile_path = |src_path: &Path| { + let source = fs::read_to_string(src_path).map_err(|err| { + Diagnostic::spans_error( + self.span, + format!("Error reading file {:?}: {}", path, err), + ) + })?; + self.compile_string(&source, mode, module_name.clone(), || { + path.strip_prefix(&*CARGO_MANIFEST_DIR) + .ok() + .unwrap_or(&path) + .display() + }) + }; + let code = compile_path(&path).or_else(|e| { + if cfg!(windows) && is_git_symlink(&path) { + if let Ok(real_path) = fs::read_to_string(&path) { + return compile_path(real_path.trim().as_ref()); + } + } + Err(e) + })?; + code_map.insert( - module_name.clone(), + module_name, FrozenModule { - code: self.compile_string(&source, mode, module_name, || { - path.strip_prefix(&*CARGO_MANIFEST_DIR) - .ok() - .unwrap_or(&path) - .display() - })?, + code, package: is_init, }, ); diff --git a/scripts/symlinks-to-hardlinks.ps1 b/scripts/symlinks-to-hardlinks.ps1 deleted file mode 100644 index 9ec79de4e..000000000 --- a/scripts/symlinks-to-hardlinks.ps1 +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env powershell - -Set-Location -Path $(Split-Path -Parent $PSScriptRoot) - -git ls-files -s | Select-String '^120000' | ConvertFrom-String -PropertyNames Get-Content,Hash,_,Path | ForEach-Object { - $symlink = $_.Path - - git checkout --quiet -- $symlink - if (Test-Path $symlink -PathType Leaf) { - $parent = (Get-Item $symlink).Directory - } else { - $parent = (Get-Item $symlink).Parent - } - $child = (Get-Content $symlink) - - $src = (Join-Path -Path $parent -ChildPath $child) - - if (Test-Path $src -PathType Leaf) { - Remove-Item $symlink - New-Item -ItemType HardLink -Name $symlink -Value $src - } elseif (Test-Path $src -PathType Container) { - Remove-Item $symlink - New-Item -ItemType Junction -Name $symlink -Value $src - } else { - Write-Error 'error: git-rm-symlink: Not a valid source\n' - Write-Error '$symlink =/=> $src...' - return - } - - git update-index --assume-unchanged $symlink -} diff --git a/vm/src/frozen.rs b/vm/src/frozen.rs index ec94b7ecd..160ae22ab 100644 --- a/vm/src/frozen.rs +++ b/vm/src/frozen.rs @@ -38,7 +38,6 @@ pub fn get_module_inits() -> impl Iterator Date: Thu, 29 Apr 2021 18:21:02 -0500 Subject: [PATCH 2/4] Fix thanks to @jhoobergs Co-authored-by: Jesse Hoobergs --- derive/src/compile_bytecode.rs | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs index 666ac754c..29c9497d7 100644 --- a/derive/src/compile_bytecode.rs +++ b/derive/src/compile_bytecode.rs @@ -35,16 +35,6 @@ fn is_git_symlink(path: &Path) -> bool { use std::io::{BufRead, BufReader}; use std::process::{Command, Stdio}; let res = path.parent().and_then(|dir| { - let child = Command::new("git") - .arg("-C") - .arg(dir) - .arg("ls-files") - .arg("-s") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .ok()?; let git_root = Command::new("git") .arg("-C") .arg(dir) @@ -54,6 +44,16 @@ fn is_git_symlink(path: &Path) -> bool { .ok()? .stdout; let git_root = Path::new(std::str::from_utf8(&git_root).unwrap().trim()); + let child = Command::new("git") + .arg("-C") + .arg(git_root) + .arg("ls-files") + .arg("-s") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .ok()?; let stdout = BufReader::new(child.stdout.unwrap()); let lines = stdout .lines() @@ -161,7 +161,7 @@ impl CompilationSource { // handle git symlinks - git for windows will make it just a text file containing // the target if cfg!(windows) && is_git_symlink(path) { - if let Ok(real_path) = fs::read_to_string(path) { + if let Ok(real_path) = fs::read_to_string(path.canonicalize().unwrap()) { return fs::read_dir(real_path.trim()); } } @@ -211,8 +211,9 @@ impl CompilationSource { }; let code = compile_path(&path).or_else(|e| { if cfg!(windows) && is_git_symlink(&path) { - if let Ok(real_path) = fs::read_to_string(&path) { - return compile_path(real_path.trim().as_ref()); + if let Ok(real_path) = fs::read_to_string(path.canonicalize().unwrap()) { + let joined = path.parent().unwrap().join(real_path.trim()); + return compile_path(&joined); } } Err(e) From 353183dc794c49bb23371f945cd15991307df228 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sat, 1 May 2021 13:35:03 -0500 Subject: [PATCH 3/4] Don't check git for symlinks Co-authored-by: Jesse Hoobergs --- derive/src/compile_bytecode.rs | 53 +++++----------------------------- 1 file changed, 7 insertions(+), 46 deletions(-) diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs index 29c9497d7..e6f3b7c7a 100644 --- a/derive/src/compile_bytecode.rs +++ b/derive/src/compile_bytecode.rs @@ -31,47 +31,6 @@ static CARGO_MANIFEST_DIR: Lazy = Lazy::new(|| { PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present")) }); -fn is_git_symlink(path: &Path) -> bool { - use std::io::{BufRead, BufReader}; - use std::process::{Command, Stdio}; - let res = path.parent().and_then(|dir| { - let git_root = Command::new("git") - .arg("-C") - .arg(dir) - .arg("rev-parse") - .arg("--show-toplevel") - .output() - .ok()? - .stdout; - let git_root = Path::new(std::str::from_utf8(&git_root).unwrap().trim()); - let child = Command::new("git") - .arg("-C") - .arg(git_root) - .arg("ls-files") - .arg("-s") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .ok()?; - let stdout = BufReader::new(child.stdout.unwrap()); - let lines = stdout - .lines() - .filter_map(|line| line.ok()) - .filter(|line| line.starts_with("120000")); - let path = path.canonicalize().ok()?; - for line in lines { - let link_path = line.splitn(2, '\t').nth(1).unwrap(); - let link_path = git_root.join(link_path).canonicalize().ok()?; - if link_path == path { - return Some(true); - } - } - Some(false) - }); - res.unwrap_or(false) -} - enum CompilationSourceKind { /// Source is a File (Path) File(PathBuf), @@ -158,9 +117,7 @@ impl CompilationSource { let mut code_map = HashMap::new(); let paths = fs::read_dir(path) .or_else(|e| { - // handle git symlinks - git for windows will make it just a text file containing - // the target - if cfg!(windows) && is_git_symlink(path) { + if cfg!(windows) { if let Ok(real_path) = fs::read_to_string(path.canonicalize().unwrap()) { return fs::read_dir(real_path.trim()); } @@ -210,10 +167,14 @@ impl CompilationSource { }) }; let code = compile_path(&path).or_else(|e| { - if cfg!(windows) && is_git_symlink(&path) { + if cfg!(windows) { if let Ok(real_path) = fs::read_to_string(path.canonicalize().unwrap()) { let joined = path.parent().unwrap().join(real_path.trim()); - return compile_path(&joined); + if joined.exists() { + return compile_path(&joined); + } else { + return Err(e); + } } } Err(e) From 9a83e15918513c4393f73c3f225f4c9219edea71 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sat, 1 May 2021 23:20:10 -0500 Subject: [PATCH 4/4] Fix cfg for windows-gnu target --- vm/src/stdlib/os.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 9589ca7a8..e89793900 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -2764,6 +2764,7 @@ pub(crate) use posix::raw_set_inheritable; #[pymodule] mod nt { use super::*; + #[cfg(target_env = "msvc")] use crate::builtins::list::PyListRef; #[cfg(target_env = "msvc")] use winapi::vc::vcruntime::intptr_t; @@ -3120,7 +3121,7 @@ mod nt { } #[cfg(windows)] use nt as platform; -#[cfg(windows)] +#[cfg(all(windows, target_env = "msvc"))] pub use nt::{_set_thread_local_invalid_parameter_handler, silent_iph_handler}; #[cfg(not(any(unix, windows)))]