diff --git a/crates/ty/tests/cli/python_environment.rs b/crates/ty/tests/cli/python_environment.rs index e7966e5d83..4526e099b6 100644 --- a/crates/ty/tests/cli/python_environment.rs +++ b/crates/ty/tests/cli/python_environment.rs @@ -462,7 +462,7 @@ import colorama info: Searched in the following paths during module resolution: info: 1. /project (first-party code) info: 2. vendored://stdlib (stdlib typeshed stubs vendored by ty) - info: 3. /opt/homebrew/Cellar/python@3.13/3.13.5/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (site-packages) + info: 3. /opt/homebrew/Cellar/python@3.13/3.13.5/lib/python3.13/site-packages (site-packages) info: make sure your Python environment is properly configured: https://docs.astral.sh/ty/modules/#python-environment error[unresolved-import]: Cannot resolve imported module `colorama` @@ -473,7 +473,7 @@ import colorama info: Searched in the following paths during module resolution: info: 1. /project (first-party code) info: 2. vendored://stdlib (stdlib typeshed stubs vendored by ty) - info: 3. /opt/homebrew/Cellar/python@3.13/3.13.5/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (site-packages) + info: 3. /opt/homebrew/Cellar/python@3.13/3.13.5/lib/python3.13/site-packages (site-packages) info: make sure your Python environment is properly configured: https://docs.astral.sh/ty/modules/#python-environment Found 2 diagnostics @@ -495,7 +495,7 @@ import colorama info: Searched in the following paths during module resolution: info: 1. /project (first-party code) info: 2. vendored://stdlib (stdlib typeshed stubs vendored by ty) - info: 3. /opt/homebrew/Cellar/python@3.13/3.13.5/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (site-packages) + info: 3. /opt/homebrew/Cellar/python@3.13/3.13.5/lib/python3.13/site-packages (site-packages) info: make sure your Python environment is properly configured: https://docs.astral.sh/ty/modules/#python-environment error[unresolved-import]: Cannot resolve imported module `colorama` @@ -506,7 +506,7 @@ import colorama info: Searched in the following paths during module resolution: info: 1. /project (first-party code) info: 2. vendored://stdlib (stdlib typeshed stubs vendored by ty) - info: 3. /opt/homebrew/Cellar/python@3.13/3.13.5/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (site-packages) + info: 3. /opt/homebrew/Cellar/python@3.13/3.13.5/lib/python3.13/site-packages (site-packages) info: make sure your Python environment is properly configured: https://docs.astral.sh/ty/modules/#python-environment Found 2 diagnostics diff --git a/crates/ty/tests/file_watching.rs b/crates/ty/tests/file_watching.rs index 4381f74ff9..a7caee48f0 100644 --- a/crates/ty/tests/file_watching.rs +++ b/crates/ty/tests/file_watching.rs @@ -3,6 +3,10 @@ use std::io::Write; use std::time::{Duration, Instant}; use anyhow::{Context, anyhow}; +use insta::assert_snapshot; +use ruff_db::diagnostic::{ + Diagnostic, DiagnosticFormat, DisplayDiagnosticConfig, DisplayDiagnostics, +}; use ruff_db::files::{File, FileError, system_path_to_file}; use ruff_db::source::source_text; use ruff_db::system::{ @@ -17,9 +21,12 @@ use ty_project::metadata::options::{EnvironmentOptions, Options, SrcOptions}; use ty_project::metadata::pyproject::{PyProject, Tool}; use ty_project::metadata::python_version::SupportedPythonVersion; use ty_project::metadata::value::{RelativeGlobPattern, RelativePathBuf}; +#[cfg(unix)] +use ty_project::watch::watch_paths; use ty_project::watch::{ChangeEvent, ProjectWatcher, directory_watcher}; use ty_project::{ChangeResult, Db, ProjectDatabase, ProjectMetadata, UseUv}; use ty_python_core::platform::PythonPlatform; +use ty_static::EnvVars; struct TestCase { db: ProjectDatabase, @@ -52,6 +59,14 @@ impl TestCase { &self.root_dir } + fn write_virtual_environment( + &self, + path: impl AsRef, + python_version: PythonVersion, + ) -> anyhow::Result<()> { + write_virtual_environment(self.root_path(), path, python_version) + } + fn db(&self) -> &ProjectDatabase { &self.db } @@ -262,6 +277,15 @@ impl TestCase { names.sort(); names } + + fn render_diagnostics(&self, diagnostics: &[Diagnostic]) -> String { + DisplayDiagnostics::new( + self.db(), + &DisplayDiagnosticConfig::new("ty").format(DiagnosticFormat::Concise), + diagnostics, + ) + .to_string() + } } trait MatchEvent { @@ -293,6 +317,7 @@ struct SetupContext<'a> { override_options: Option, fallback_options: Option, included_paths: Option>, + allow_invalid_settings: bool, } impl<'a> SetupContext<'a> { @@ -323,7 +348,7 @@ impl<'a> SetupContext<'a> { ) -> anyhow::Result<()> { let relative_path = relative_path.as_ref(); let absolute_path = self.join_project_path(relative_path); - Self::write_file_impl(absolute_path, &dedent(content)) + write_file(absolute_path, &dedent(content)) } fn write_file( @@ -333,29 +358,25 @@ impl<'a> SetupContext<'a> { ) -> anyhow::Result<()> { let relative_path = relative_path.as_ref(); let absolute_path = self.join_root_path(relative_path); - Self::write_file_impl(absolute_path, content) + write_file(absolute_path, content) } - fn write_file_impl(path: impl AsRef, content: &str) -> anyhow::Result<()> { - let path = path.as_ref(); - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .with_context(|| format!("Failed to create parent directory for file `{path}`"))?; - } - - let mut file = std::fs::File::create(path.as_std_path()) - .with_context(|| format!("Failed to open file `{path}`"))?; - file.write_all(content.as_bytes()) - .with_context(|| format!("Failed to write to file `{path}`"))?; - file.sync_data()?; - - Ok(()) + fn write_virtual_environment( + &self, + path: impl AsRef, + python_version: PythonVersion, + ) -> anyhow::Result<()> { + write_virtual_environment(self.root_path(), path, python_version) } fn set_options(&mut self, options: Options) { self.options = Some(options); } + fn allow_invalid_settings(&mut self) { + self.allow_invalid_settings = true; + } + fn set_config_file_override(&mut self, path: impl AsRef) { self.config_file_override = Some(self.join_project_path(path)); } @@ -445,6 +466,7 @@ where override_options: None, fallback_options: None, included_paths: None, + allow_invalid_settings: false, }; setup_files @@ -457,6 +479,7 @@ where override_options, fallback_options, included_paths, + allow_invalid_settings, .. } = setup_context; @@ -500,7 +523,11 @@ where } } - let mut db = ProjectDatabase::fallible(project, system)?; + let mut db = if allow_invalid_settings { + ProjectDatabase::use_defaults(project, system) + } else { + ProjectDatabase::fallible(project, system)? + }; if let Some(included_paths) = included_paths { db.project().set_included_paths(&mut db, included_paths); @@ -544,6 +571,59 @@ where Ok(test_case) } +/// Creates or updates a mock virtual environment, resolving relative paths against the test root. +fn write_virtual_environment( + root: &SystemPath, + path: impl AsRef, + python_version: PythonVersion, +) -> anyhow::Result<()> { + let environment = root.join(path); + let python_home = root.join("base/bin"); + write_file(python_home.join("python"), "")?; + write_file( + environment.join(if cfg!(windows) { + "Scripts/python.exe" + } else { + "bin/python" + }), + "", + )?; + let site_packages = if cfg!(windows) { + environment.join("Lib/site-packages") + } else { + environment.join(format!("lib/python{python_version}/site-packages")) + }; + std::fs::create_dir_all(site_packages)?; + let pyvenv_cfg = environment.join("pyvenv.cfg"); + let pyvenv_cfg_contents = format!( + r" + home = {python_home} + version = {python_version} + " + ); + if pyvenv_cfg.as_std_path().try_exists()? { + update_file(pyvenv_cfg, &pyvenv_cfg_contents) + } else { + write_file(pyvenv_cfg, &dedent(&pyvenv_cfg_contents)) + } +} + +fn write_file(path: impl AsRef, content: &str) -> anyhow::Result<()> { + let path = path.as_ref(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("Failed to create parent directory for file `{path}`"))?; + } + + let mut file = std::fs::File::create(path.as_std_path()) + .with_context(|| format!("Failed to open file `{path}`"))?; + file.write_all(content.as_bytes()) + .with_context(|| format!("Failed to write to file `{path}`"))?; + file.sync_data()?; + + Ok(()) +} + /// Dedents and updates a file's content, ensuring that its last modified time changes. fn update_file(path: impl AsRef, content: &str) -> anyhow::Result<()> { let path = path.as_ref().as_std_path(); @@ -1029,27 +1109,33 @@ fn script_exclusion_tracks_file_creation_and_metadata_edits() -> anyhow::Result< # /// missing "; - assert!(case.db().check().is_empty()); + assert_eq!(case.db().check().as_slice(), &[]); std::fs::write(path.as_std_path(), dedent(script).as_ref())?; let changes = case.take_watch_changes(event_for_file("script.py")); let changes = case.apply_changes(&changes); assert!(changes.scripts_to_synchronize(case.db()).is_empty()); let file = case.system_file(&path)?; - assert!(case.db().check().is_empty()); - assert!(case.db().check_file(file).is_empty()); + assert_eq!(case.db().check().as_slice(), &[]); + assert_eq!(case.db().check_file(file).as_slice(), &[]); update_file(&path, "missing\n")?; let changes = case.take_watch_changes(event_for_file("script.py")); case.apply_changes(&changes); - assert_eq!(case.db().check().len(), 1); - assert_eq!(case.db().check_file(file).len(), 1); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"script.py:1:1: error[unresolved-reference] Name `missing` used when not defined" + ); + assert_snapshot!( + case.render_diagnostics(&case.db().check_file(file)), + @"script.py:1:1: error[unresolved-reference] Name `missing` used when not defined" + ); update_file(&path, script)?; let changes = case.take_watch_changes(event_for_file("script.py")); case.apply_changes(&changes); - assert!(case.db().check().is_empty()); - assert!(case.db().check_file(file).is_empty()); + assert_eq!(case.db().check().as_slice(), &[]); + assert_eq!(case.db().check_file(file).as_slice(), &[]); Ok(()) } @@ -1071,8 +1157,14 @@ fn explicitly_included_file_remains_checked_when_becoming_a_script() -> anyhow:: case.db .project() .set_included_paths(&mut case.db, vec![path.clone()]); - assert_eq!(case.db().check().len(), 1); - assert_eq!(case.db().check_file(file).len(), 1); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"script.py:1:1: error[unresolved-reference] Name `missing` used when not defined" + ); + assert_snapshot!( + case.render_diagnostics(&case.db().check_file(file)), + @"script.py:1:1: error[unresolved-reference] Name `missing` used when not defined" + ); update_file( &path, @@ -1085,8 +1177,14 @@ fn explicitly_included_file_remains_checked_when_becoming_a_script() -> anyhow:: )?; let changes = case.take_watch_changes(event_for_file("script.py")); case.apply_changes(&changes); - assert_eq!(case.db().check().len(), 1); - assert_eq!(case.db().check_file(file).len(), 1); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"script.py:5:1: error[unresolved-reference] Name `missing` used when not defined" + ); + assert_snapshot!( + case.render_diagnostics(&case.db().check_file(file)), + @"script.py:5:1: error[unresolved-reference] Name `missing` used when not defined" + ); Ok(()) } @@ -1526,15 +1624,16 @@ fn script_dependency_changes() -> anyhow::Result<()> { ) })?; let dependency = case.root_path().join("dependencies/dependency.py"); - assert!(case.db().check().is_empty()); + assert_eq!(case.db().check().as_slice(), &[]); update_file(&dependency, "value = 'wrong'")?; let changes = case.stop_watch(event_for_file("dependency.py")); case.apply_changes(&changes); - let diagnostics = case.db().check(); - assert_eq!(diagnostics.len(), 1); - assert_eq!(diagnostics[0].id().as_str(), "invalid-assignment"); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @r#"script.py:8:15: error[invalid-assignment] Object of type `Literal["wrong"]` is not assignable to `int`"# + ); Ok(()) } @@ -1638,7 +1737,7 @@ fn nested_search_path_picks_up_missed_dependency_edit() -> anyhow::Result<()> { let dependency = case.root_path().join("dependencies/pkg/dependency.py"); // The first check caches the dependency through the parent search path. - assert!(case.db().check().is_empty()); + assert_eq!(case.db().check().as_slice(), &[]); // Stop watching the parent before editing the dependency, so ty misses that file event. update_file(case.project_path("pyproject.toml"), "[tool.ty]\n")?; @@ -1670,9 +1769,10 @@ fn nested_search_path_picks_up_missed_dependency_edit() -> anyhow::Result<()> { let changes = case.take_watch_changes(event_for_file("main.py")); case.apply_changes(&changes); - let diagnostics = case.db().check(); - assert_eq!(diagnostics.len(), 1); - assert_eq!(diagnostics[0].id().as_str(), "invalid-assignment"); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @r#"main.py:3:15: error[invalid-assignment] Object of type `Literal["wrong"]` is not assignable to `int`"# + ); Ok(()) } @@ -1715,7 +1815,7 @@ version = 3.12 })?; // The initial import resolves through `old`, as named by `dependency.pth`. - assert!(case.db().check().is_empty()); + assert_eq!(case.db().check().as_slice(), &[]); // Stop watching the virtual environment before changing its `.pth` file. update_file(case.project_path("pyproject.toml"), "[tool.ty]\n")?; @@ -1751,15 +1851,17 @@ version = 3.12 #[test] fn change_python_version_and_platform() -> anyhow::Result<()> { let mut case = setup(|context: &mut SetupContext| { - // `sys.last_exc` is a Python 3.12 only feature // `os.getegid()` is Unix only context.write_project_file( "bar.py", - r#" -import sys -import os -print(sys.last_exc, os.getegid()) -"#, + r" + import sys + import os + from typing_extensions import reveal_type + + reveal_type(sys.version_info[:2]) + os.getegid() + ", )?; context.set_options(Options { environment: Some(EnvironmentOptions { @@ -1775,17 +1877,12 @@ print(sys.last_exc, os.getegid()) Ok(()) })?; - let diagnostics = case.db.check(); - - assert_eq!(diagnostics.len(), 2); - assert_eq!( - diagnostics[0].headline_message(), - "Module `sys` has no member `last_exc`" - ); - assert_eq!( - diagnostics[1].headline_message(), - "Module `os` has no member `getegid`" - ); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @" + bar.py:6:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[11]]` + bar.py:7:1: error[unresolved-attribute] Module `os` has no member `getegid` + "); // Change the python version case.update_options(Options { @@ -1800,12 +1897,290 @@ print(sys.last_exc, os.getegid()) }) .expect("Search path settings to be valid"); - let diagnostics = case.db.check(); - assert!(diagnostics.is_empty()); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"bar.py:6:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[12]]`" + ); Ok(()) } +#[test] +fn recreating_default_environment_updates_inferred_python_version() -> anyhow::Result<()> { + let mut case = setup(|context: &mut SetupContext| { + context + .write_virtual_environment(context.join_project_path(".venv"), PythonVersion::PY311)?; + context.write_project_file( + "main.py", + r" + import sys + from typing_extensions import reveal_type + + reveal_type(sys.version_info[:2]) + ", + ) + })?; + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:5:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[11]]`" + ); + + let environment = case.project_path(".venv"); + + // Without `.venv`, ty uses its default Python version. + std::fs::remove_dir_all(&environment)?; + let changes = case.take_watch_changes(event_for_file(".venv")); + case.apply_changes(&changes); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:5:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[14]]`" + ); + + case.write_virtual_environment(&environment, PythonVersion::PY311)?; + let changes = case.stop_watch(event_for_file(".venv")); + case.apply_changes(&changes); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:5:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[11]]`" + ); + Ok(()) +} + +#[test] +fn repairing_default_environment_updates_inferred_python_version() -> anyhow::Result<()> { + let mut case = setup(|context: &mut SetupContext| { + context + .write_virtual_environment(context.join_project_path(".venv"), PythonVersion::PY311)?; + // Without `home`, the environment cannot be selected at startup. + context.write_project_file( + ".venv/pyvenv.cfg", + r" + version = 3.11 + ", + )?; + context.write_project_file( + "main.py", + r" + import sys + from typing_extensions import reveal_type + + reveal_type(sys.version_info[:2]) + ", + ) + })?; + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:5:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[14]]`" + ); + + // Write a valid `pyvenv.cfg`. + case.write_virtual_environment(case.project_path(".venv"), PythonVersion::PY311)?; + let changes = case.stop_watch(event_for_file("pyvenv.cfg")); + case.apply_changes(&changes); + + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:5:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[11]]`" + ); + Ok(()) +} + +#[test] +fn creating_site_packages_after_metadata_updates_inferred_python_version() -> anyhow::Result<()> { + let library = if cfg!(windows) { "Lib" } else { "lib" }; + let mut case = setup(|context: &mut SetupContext| { + context.write_file("base/bin/python", "")?; + std::fs::create_dir(context.join_project_path(".venv"))?; + context.write_project_file( + "main.py", + r" + import sys + from typing_extensions import reveal_type + + reveal_type(sys.version_info[:2]) + ", + ) + })?; + + // uv writes pyvenv.cfg before creating site-packages. Process these in separate batches. + let python_home = case.root_path().join("base/bin"); + write_file( + case.project_path(".venv/pyvenv.cfg"), + &dedent(&format!( + r" + home = {python_home} + version = 3.11 + " + )), + )?; + let changes = case.take_watch_changes(event_for_file("pyvenv.cfg")); + case.apply_changes(&changes); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:5:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[14]]`" + ); + + let library = case.project_path(".venv").join(library); + let site_packages = library.join(if cfg!(windows) { + "site-packages" + } else { + "python3.11/site-packages" + }); + std::fs::create_dir_all(site_packages)?; + // Watchers may report the library directory, its descendants, or both. + let changes = case.stop_watch(|change: &ChangeEvent| { + matches!(change, ChangeEvent::Created { path, .. } if path.starts_with(&library)) + }); + case.apply_changes(&changes); + + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:5:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[11]]`" + ); + Ok(()) +} + +#[test] +fn moving_configured_interpreter_parent_into_project_updates_inferred_python_version() +-> anyhow::Result<()> { + let python = if cfg!(windows) { + "envs/env/Scripts/python.exe" + } else { + "envs/env/bin/python" + }; + let mut case = setup(|context: &mut SetupContext| { + context.allow_invalid_settings(); + context.write_virtual_environment("unwatched/env", PythonVersion::PY311)?; + context.write_project_file( + "pyproject.toml", + &format!( + r#" + [tool.ty.environment] + python = "./{python}" + "# + ), + )?; + context.write_project_file( + "main.py", + r" + import sys + from typing_extensions import reveal_type + + reveal_type(sys.version_info[:2]) + ", + ) + })?; + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:5:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[14]]`" + ); + + // Moving the parent can produce a single directory event for the whole environment. + std::fs::rename( + case.root_path().join("unwatched"), + case.project_path("envs"), + )?; + let changes = case.stop_watch(event_for_file("envs")); + case.apply_changes(&changes); + + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:5:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[11]]`" + ); + Ok(()) +} + +#[test] +fn repairing_activated_environment_updates_inferred_python_version() -> anyhow::Result<()> { + let mut case = setup_with_system( + |context: &mut SetupContext| { + context.allow_invalid_settings(); + context.write_virtual_environment( + context.join_project_path("active-env"), + PythonVersion::PY311, + )?; + context.write_project_file( + "active-env/pyvenv.cfg", + r" + version = 3.11 + ", + )?; + context.write_project_file( + "main.py", + r" + import sys + from typing_extensions import reveal_type + + reveal_type(sys.version_info[:2]) + ", + ) + }, + |system| { + system.set_env_var( + EnvVars::VIRTUAL_ENV, + system.current_directory().join("active-env").as_str(), + ); + }, + )?; + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:5:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[14]]`" + ); + + case.write_virtual_environment(case.project_path("active-env"), PythonVersion::PY311)?; + let changes = case.stop_watch(event_for_file("pyvenv.cfg")); + case.apply_changes(&changes); + + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:5:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[11]]`" + ); + Ok(()) +} + +#[test] +fn external_interpreter_changes_update_inferred_python_version() -> anyhow::Result<()> { + let python = if cfg!(windows) { + "environment/Scripts/python.exe" + } else { + "environment/bin/python" + }; + let mut case = setup(|context: &mut SetupContext| { + context.write_virtual_environment("environment", PythonVersion::PY311)?; + context.write_project_file( + "main.py", + r" + import sys + from typing_extensions import reveal_type + + reveal_type(sys.version_info[:2]) + ", + )?; + context.write_project_file( + "pyproject.toml", + &format!( + r#" + [tool.ty.environment] + python = "../{python}" + "# + ), + ) + })?; + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:5:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[11]]`" + ); + + case.write_virtual_environment("environment", PythonVersion::PY312)?; + let events = case.stop_watch(event_for_file("pyvenv.cfg")); + case.apply_changes(&events); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:5:13: info[revealed-type] Revealed type: `tuple[Literal[3], Literal[12]]`" + ); + Ok(()) +} + #[cfg_attr(windows, ignore = "site-packages layout inference is Unix-only")] #[test] fn reloading_options_updates_inferred_python_version_diagnostics_when_metadata_is_unchanged() @@ -1833,15 +2208,9 @@ fn reloading_options_updates_inferred_python_version_diagnostics_when_metadata_i Ok(()) })?; - let diagnostics = case.db.check(); - - assert_eq!(diagnostics.len(), 1); - assert_eq!( - diagnostics[0].headline_message(), - format!( - "Ignoring unsupported inferred Python version `3.{unsupported_minor}`; ty will use Python {} instead.", - PythonVersion::latest_ty() - ) + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @".venv/pyvenv.cfg: warning[unsupported-python-version] Ignoring unsupported inferred Python version `3.16`; ty will use Python 3.14 instead." ); std::fs::rename( @@ -1854,11 +2223,7 @@ fn reloading_options_updates_inferred_python_version_diagnostics_when_metadata_i // manually reloads the options instead of exercising the file-watching path for that rename. case.update_options(Options::default())?; - let diagnostics = case.db.check(); - assert!( - diagnostics.is_empty(), - "Expected no diagnostics but got: {diagnostics:#?}" - ); + assert_eq!(case.db().check().as_slice(), &[]); Ok(()) } @@ -2290,6 +2655,52 @@ mod unix { Ok(()) } + #[test] + fn symlinked_site_packages_outside_environment() -> anyhow::Result<()> { + let mut case = setup(|context: &mut SetupContext| { + context.write_virtual_environment("environment", PythonVersion::PY312)?; + let environment_site_packages = + context.join_root_path("environment/lib/python3.12/site-packages"); + let shared_site_packages = context.join_root_path("shared-site-packages"); + std::fs::rename(&environment_site_packages, &shared_site_packages)?; + std::os::unix::fs::symlink(&shared_site_packages, &environment_site_packages)?; + context.write_file("shared-site-packages/dependency.py", "value: int = 1")?; + context.write_project_file( + "ty.toml", + r#" + [environment] + python = "../environment" + "#, + )?; + context.write_project_file( + "main.py", + r" + from dependency import value + + result: int = value + ", + ) + })?; + assert_eq!(case.db().check(), []); + + let environment = case.root_path().join("environment"); + let site_packages = case.root_path().join("shared-site-packages"); + // The target is outside the environment and needs its own watch. + assert_eq!( + watch_paths(case.db(), case.db().project()).paths(), + [case.project_path(""), environment, site_packages.clone()] + ); + + update_file(site_packages.join("dependency.py"), "value: str = ''")?; + let changes = case.stop_watch(event_for_file("dependency.py")); + case.apply_changes(&changes); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"main.py:4:15: error[invalid-assignment] Object of type `str` is not assignable to `int`" + ); + Ok(()) + } + /// A module search path is a symlink. /// /// Setup: @@ -2517,12 +2928,7 @@ fn changes_to_user_configuration() -> anyhow::Result<()> { let foo = case .system_file(case.project_path("foo.py")) .expect("foo.py to exist"); - let diagnostics = case.db().check_file(foo); - - assert!( - diagnostics.is_empty(), - "Expected no diagnostics but got: {diagnostics:#?}" - ); + assert_eq!(case.db().check_file(foo).as_slice(), &[]); // Enable division-by-zero in the user configuration with warning severity update_file( @@ -2537,11 +2943,9 @@ fn changes_to_user_configuration() -> anyhow::Result<()> { case.apply_changes(&changes); - let diagnostics = case.db().check_file(foo); - - assert!( - diagnostics.len() == 1, - "Expected exactly one diagnostic but got: {diagnostics:#?}" + assert_snapshot!( + case.render_diagnostics(&case.db().check_file(foo)), + @"foo.py:1:5: warning[division-by-zero] Cannot divide object of type `Literal[10]` by zero" ); // Removing the option from the user configuration must not retain the old warning level. @@ -2550,11 +2954,7 @@ fn changes_to_user_configuration() -> anyhow::Result<()> { let changes = case.stop_watch(event_for_file("ty.toml")); case.apply_changes(&changes); - let diagnostics = case.db().check_file(foo); - assert!( - diagnostics.is_empty(), - "Expected no diagnostics but got: {diagnostics:#?}" - ); + assert_eq!(case.db().check_file(foo).as_slice(), &[]); Ok(()) } @@ -2577,7 +2977,7 @@ fn project_reload_preserves_override_options() -> anyhow::Result<()> { let foo = case .system_file(case.project_path("foo.py")) .expect("foo.py to exist"); - assert!(case.db().check_file(foo).is_empty()); + assert_eq!(case.db().check_file(foo).as_slice(), &[]); case.update_options(Options::from_toml_str( r#" @@ -2587,11 +2987,7 @@ fn project_reload_preserves_override_options() -> anyhow::Result<()> { ValueSource::Cli, )?)?; - let diagnostics = case.db().check_file(foo); - assert!( - diagnostics.is_empty(), - "Expected override options to survive reload but got: {diagnostics:#?}" - ); + assert_eq!(case.db().check_file(foo).as_slice(), &[]); Ok(()) } @@ -2614,7 +3010,10 @@ fn project_reload_preserves_fallback_options() -> anyhow::Result<()> { let foo = case .system_file(case.project_path("foo.py")) .expect("foo.py to exist"); - assert_eq!(case.db().check_file(foo).len(), 1); + assert_snapshot!( + case.render_diagnostics(&case.db().check_file(foo)), + @"foo.py:1:5: warning[division-by-zero] Cannot divide object of type `Literal[10]` by zero" + ); case.update_options(Options::from_toml_str( r#" @@ -2624,11 +3023,9 @@ fn project_reload_preserves_fallback_options() -> anyhow::Result<()> { ValueSource::Cli, )?)?; - let diagnostics = case.db().check_file(foo); - assert_eq!( - diagnostics.len(), - 1, - "Expected fallback options to survive reload but got: {diagnostics:#?}" + assert_snapshot!( + case.render_diagnostics(&case.db().check_file(foo)), + @"foo.py:1:5: warning[division-by-zero] Cannot divide object of type `Literal[10]` by zero" ); Ok(()) @@ -2665,12 +3062,7 @@ fn changes_to_config_file_override() -> anyhow::Result<()> { let foo = case .system_file(case.project_path("foo.py")) .expect("foo.py to exist"); - let diagnostics = case.db().check_file(foo); - - assert!( - diagnostics.is_empty(), - "Expected no diagnostics but got: {diagnostics:#?}" - ); + assert_eq!(case.db().check_file(foo).as_slice(), &[]); // Enable division-by-zero in the explicitly specified configuration with warning severity update_file( @@ -2685,11 +3077,9 @@ fn changes_to_config_file_override() -> anyhow::Result<()> { case.apply_changes(&changes); - let diagnostics = case.db().check_file(foo); - - assert!( - diagnostics.len() == 1, - "Expected exactly one diagnostic but got: {diagnostics:#?}" + assert_snapshot!( + case.render_diagnostics(&case.db().check_file(foo)), + @"foo.py:1:5: warning[division-by-zero] Cannot divide object of type `Literal[10]` by zero" ); Ok(()) @@ -2771,7 +3161,7 @@ fn rename_files_casing_only() -> anyhow::Result<()> { fn submodule_cache_invalidation_created() -> anyhow::Result<()> { let mut case = setup([("lib.py", ""), ("bar/__init__.py", ""), ("bar/foo.py", "")])?; - insta::assert_snapshot!( + assert_snapshot!( case.sorted_submodule_names("bar").join("\n"), @"bar.foo", ); @@ -2780,7 +3170,7 @@ fn submodule_cache_invalidation_created() -> anyhow::Result<()> { let changes = case.stop_watch(event_for_file("wazoo.py")); case.apply_changes(&changes); - insta::assert_snapshot!( + assert_snapshot!( case.sorted_submodule_names("bar").join("\n"), @" bar.foo @@ -2802,7 +3192,7 @@ fn submodule_cache_invalidation_deleted() -> anyhow::Result<()> { ("bar/wazoo.py", ""), ])?; - insta::assert_snapshot!( + assert_snapshot!( case.sorted_submodule_names("bar").join("\n"), @" bar.foo @@ -2814,7 +3204,7 @@ fn submodule_cache_invalidation_deleted() -> anyhow::Result<()> { let changes = case.stop_watch(event_for_file("wazoo.py")); case.apply_changes(&changes); - insta::assert_snapshot!( + assert_snapshot!( case.sorted_submodule_names("bar").join("\n"), @"bar.foo", ); @@ -2828,7 +3218,7 @@ fn submodule_cache_invalidation_deleted() -> anyhow::Result<()> { fn submodule_cache_invalidation_created_then_deleted() -> anyhow::Result<()> { let mut case = setup([("lib.py", ""), ("bar/__init__.py", ""), ("bar/foo.py", "")])?; - insta::assert_snapshot!( + assert_snapshot!( case.sorted_submodule_names("bar").join("\n"), @"bar.foo", ); @@ -2841,7 +3231,7 @@ fn submodule_cache_invalidation_created_then_deleted() -> anyhow::Result<()> { let changes = case.stop_watch(event_for_file("wazoo.py")); case.apply_changes(&changes); - insta::assert_snapshot!( + assert_snapshot!( case.sorted_submodule_names("bar").join("\n"), @"bar.foo", ); @@ -2856,7 +3246,7 @@ fn submodule_cache_invalidation_created_then_deleted() -> anyhow::Result<()> { fn submodule_cache_invalidation_after_pyproject_created() -> anyhow::Result<()> { let mut case = setup([("lib.py", ""), ("bar/__init__.py", ""), ("bar/foo.py", "")])?; - insta::assert_snapshot!( + assert_snapshot!( case.sorted_submodule_names("bar").join("\n"), @"bar.foo", ); @@ -2867,7 +3257,7 @@ fn submodule_cache_invalidation_after_pyproject_created() -> anyhow::Result<()> let changes = case.take_watch_changes(event_for_file("wazoo.py")); case.apply_changes(&changes); - insta::assert_snapshot!( + assert_snapshot!( case.sorted_submodule_names("bar").join("\n"), @" bar.foo @@ -2884,6 +3274,7 @@ mod uv_metadata { use std::time::Duration; use anyhow::Context; + use insta::assert_snapshot; use ruff_db::diagnostic::DiagnosticId; use ruff_db::files::File; use ruff_db::system::{OsSystem, System as _}; @@ -2914,7 +3305,10 @@ mod uv_metadata { )?; let project = case.db().project(); let program_settings = project.program_settings(case.db()).clone(); - assert_eq!(case.db().check()[0].id().as_str(), "invalid-assignment"); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @r#"main.py:1:14: error[invalid-assignment] Object of type `Literal["wrong"]` is not assignable to `int`"# + ); // uv rejects this setting, but ty must still apply its rule configuration. update_and_synchronize_project( @@ -2953,7 +3347,7 @@ mod uv_metadata { invalid-assignment = "ignore" "#, )?; - assert!(case.db().check().is_empty()); + assert_eq!(case.db().check().as_slice(), &[]); Ok(()) } @@ -3011,7 +3405,7 @@ mod uv_metadata { )], )?; - assert!(case.db().check().is_empty()); + assert_eq!(case.db().check().as_slice(), &[]); assert!(!update_and_synchronize_script( &mut case, @@ -3025,7 +3419,7 @@ mod uv_metadata { "#, )?); - assert!(case.db().check().is_empty()); + assert_eq!(case.db().check().as_slice(), &[]); Ok(()) } @@ -3046,9 +3440,10 @@ mod uv_metadata { )], )?; - let diagnostics = case.db().check(); - assert_eq!(diagnostics.len(), 1); - assert_eq!(diagnostics[0].id().as_str(), "unresolved-import"); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"script.py:6:6: error[unresolved-import] Cannot resolve imported module `attrs`" + ); let synchronized = update_and_synchronize_script( &mut case, @@ -3070,7 +3465,7 @@ mod uv_metadata { }); case.apply_changes(&changes); - assert!(case.db().check().is_empty()); + assert_eq!(case.db().check().as_slice(), &[]); Ok(()) } @@ -3082,11 +3477,9 @@ mod uv_metadata { &[("script.py", "from attrs import define\n")], )?; - let ordinary = case.db().check(); - assert!( - ordinary - .iter() - .any(|diagnostic| diagnostic.id().as_str() == "unresolved-import") + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"script.py:1:6: error[unresolved-import] Cannot resolve imported module `attrs`" ); update_and_synchronize_script( @@ -3099,8 +3492,7 @@ mod uv_metadata { from attrs import define "#, )?; - let diagnostics = case.db().check(); - assert!(diagnostics.is_empty(), "{diagnostics:?}"); + assert_eq!(case.db().check().as_slice(), &[]); Ok(()) } @@ -3140,11 +3532,7 @@ mod uv_metadata { )?; assert!(synchronized); - let corrected = case.db().check(); - assert!( - corrected.is_empty(), - "unexpected diagnostics: {corrected:?}" - ); + assert_eq!(case.db().check().as_slice(), &[]); Ok(()) } @@ -3176,16 +3564,17 @@ mod uv_metadata { .context("script environment has no site-packages")? .to_path_buf(); let pth = site_packages.join("dependency.pth"); - let diagnostics = case.db().check(); - assert_eq!(diagnostics.len(), 1); - assert_eq!(diagnostics[0].id().as_str(), "unresolved-import"); + assert_snapshot!( + case.render_diagnostics(&case.db().check()), + @"script.py:6:6: error[unresolved-import] Cannot resolve imported module `dependency`" + ); let dependencies = case.root_path().join("dependencies"); std::fs::write(pth.as_std_path(), dependencies.as_str())?; let changes = case.take_watch_changes(event_for_file("dependency.pth")); case.apply_changes(&changes); - assert!(case.db().check().is_empty()); + assert_eq!(case.db().check().as_slice(), &[]); let dependency = dependencies.join("dependency.py"); update_file(&dependency, "value = 2")?; diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 5d0f29002c..0fa922a4df 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -419,9 +419,7 @@ mod tests { use ty_module_resolver::SearchPathSettings; use ty_project::{Db as _, ProjectMetadata, SemanticDb as _}; use ty_python_core::ProgramFile; - use ty_python_core::platform::PythonPlatform; - use ty_python_core::program::{FallibleStrategy, ProgramSettings}; - use ty_python_semantic::PythonVersionWithSource; + use ty_python_core::program::FallibleStrategy; /// A way to create a simple single-file (named `main.py`) cursor test. /// @@ -674,14 +672,10 @@ mod tests { .to_search_paths(db.system(), db.vendored(), &FallibleStrategy) .expect("valid search paths"); - db.project().update_program( - &mut db, - ProgramSettings { - python_version: PythonVersionWithSource::default(), - python_platform: PythonPlatform::default(), - search_paths, - }, - ); + let project = db.project(); + let mut settings = project.program_settings(&db).clone(); + settings.search_paths = search_paths; + project.update_program(&mut db, settings); db.files() .try_add_root(&db, &project_root, FileRootKind::Project); diff --git a/crates/ty_module_resolver/src/resolve.rs b/crates/ty_module_resolver/src/resolve.rs index 5442f19c78..6f5e6acede 100644 --- a/crates/ty_module_resolver/src/resolve.rs +++ b/crates/ty_module_resolver/src/resolve.rs @@ -834,10 +834,12 @@ impl SearchPaths { let mut site_packages: Vec<_> = Vec::with_capacity(site_packages_paths.len()); for path in site_packages_paths { + // Resolve symlinks so site-packages outside the environment keeps its own file watch, + // and imports use the same path as the reported changes. + let path = canonicalize(path, system); tracing::debug!("Adding site-packages search path `{path}`"); let path = strategy.fallback_opt( - SearchPath::site_packages(system, path.clone()) - .map_err(SearchPathSettingsError::from), + SearchPath::site_packages(system, path).map_err(SearchPathSettingsError::from), |err| { tracing::debug!("Skipping invalid site-packages search-path: {err}"); }, diff --git a/crates/ty_project/src/db.rs b/crates/ty_project/src/db.rs index be3494da77..8e55f2fc7a 100644 --- a/crates/ty_project/src/db.rs +++ b/crates/ty_project/src/db.rs @@ -740,7 +740,6 @@ pub(crate) mod testing { use ruff_python_ast::PythonVersion; use ty_module_resolver::SearchPathSettings; use ty_python_core::ProgramFile; - use ty_python_core::platform::PythonPlatform; use ty_python_core::program::{FallibleStrategy, ProgramSettings}; #[cfg(feature = "testing")] use ty_python_semantic::ProgramEnvironment; @@ -803,11 +802,8 @@ pub(crate) mod testing { db.files().try_add_root(&db, &root, FileRootKind::Project); - let program_settings = ProgramSettings { - python_version: PythonVersionWithSource::default(), - python_platform: PythonPlatform::default(), - search_paths, - }; + let mut program_settings = ProgramSettings::empty(db.vendored()); + program_settings.search_paths = search_paths; let project = Project::from_metadata( &db, project, @@ -821,16 +817,13 @@ pub(crate) mod testing { #[cfg(feature = "testing")] pub fn set_python_version(&mut self, python_version: PythonVersion) { - let program = self.project().program(self); - let settings = ProgramSettings { - python_version: PythonVersionWithSource { - source: ty_python_semantic::PythonVersionSource::Default, - version: python_version, - }, - python_platform: program.python_platform(self).clone(), - search_paths: program.search_paths(self).clone(), + let project = self.project(); + let mut settings = project.program_settings(self).clone(); + settings.python_version = PythonVersionWithSource { + source: ty_python_semantic::PythonVersionSource::Default, + version: python_version, }; - self.project().update_program(self, settings); + project.update_program(self, settings); } } diff --git a/crates/ty_project/src/db/changes.rs b/crates/ty_project/src/db/changes.rs index e9cab9b6ce..a30b452b93 100644 --- a/crates/ty_project/src/db/changes.rs +++ b/crates/ty_project/src/db/changes.rs @@ -10,12 +10,12 @@ use ruff_db::files::{File, Files, system_path_to_file}; use ruff_db::system::{SystemPath, SystemPathBuf}; use rustc_hash::FxHashSet; use ty_python_core::program::FallibleStrategy; +use ty_python_semantic::PythonEnvironment; /// Represents the result of applying changes to the project database. pub struct ChangeResult { project_changed: bool, project_sync_path: Option, - custom_stdlib_changed: bool, changed_files: ChangedFiles, } @@ -32,11 +32,6 @@ impl ChangeResult { self.project_sync_path.as_deref() } - /// Returns `true` if the custom stdlib's VERSIONS file has changed. - pub fn custom_stdlib_changed(&self) -> bool { - self.custom_stdlib_changed - } - /// Returns the scripts whose environments may need synchronization after these file events. /// /// Returns no scripts if the project was unindexed when the changes were applied. @@ -99,6 +94,14 @@ impl ProjectDatabase { let project = self.project(); let project_root = project.root(self).to_path_buf(); let configuration_paths = ConfigurationPaths::from_metadata(project.metadata(self)); + let virtual_environment = project.program_settings(self).virtual_environment.clone(); + let python_path = project + .metadata(self) + .configured_python_path(self.system()) + .or_else(|| { + PythonEnvironment::virtual_environment_candidate(Some(&project_root), self.system()) + .map(|(path, _)| SystemPath::absolute(path, self.system().current_directory())) + }); let program = self.project().program(self); let custom_stdlib_versions_path = program .custom_stdlib_search_path(self) @@ -107,7 +110,6 @@ impl ProjectDatabase { let mut result = ChangeResult { project_changed: false, project_sync_path: None, - custom_stdlib_changed: false, changed_files: if project.file_set(self).is_lazy() { ChangedFiles::Unindexed } else { @@ -125,6 +127,7 @@ impl ProjectDatabase { let mut removed_paths = BTreeSet::default(); let mut reload_project = false; let mut reload_project_files = false; + let mut refresh_program_settings = false; // TODO: This should be removed once the incremental checker is ported // over to the `ignore` crate, since the `ignore` crate will respect // the settings provided in `create_walker`. ---AG @@ -138,6 +141,12 @@ impl ProjectDatabase { for change in changes { tracing::debug!("Handling file watcher change event: {:?}", change); + refresh_program_settings |= affects_python_environment( + change, + virtual_environment.as_deref(), + python_path.as_deref(), + ); + if let Some(path) = change.system_path() { if configuration_paths.is_configuration(path, &project_root) { File::sync_path(self, path); @@ -190,7 +199,7 @@ impl ProjectDatabase { } if Some(path) == custom_stdlib_versions_path.as_deref() { - result.custom_stdlib_changed = true; + refresh_program_settings = true; } } @@ -300,7 +309,7 @@ impl ProjectDatabase { .as_ref() .is_some_and(|versions_path| versions_path.starts_with(path)) { - result.custom_stdlib_changed = true; + refresh_program_settings = true; } if configuration_paths.may_contain_configuration(path, &project_root) { @@ -383,7 +392,7 @@ impl ProjectDatabase { removed_paths.clear(); } - if result.custom_stdlib_changed { + if refresh_program_settings { let metadata = project.metadata(self); let merged_options = metadata.to_merged_options(); match merged_options.to_program_settings( @@ -488,3 +497,70 @@ impl ConfigurationPaths { fn is_ignore_file(path: &SystemPath) -> bool { matches!(path.file_name(), Some(".gitignore" | ".ignore")) } + +/// `pyvenv.cfg` records the environment's Python version and whether imports include system +/// site-packages. Recreating an environment can change both without changing ty's configuration. +/// +/// `python_path` is the configured environment or interpreter path, +/// or a virtual environment path (e.g. `VIRTUAL_ENV` or the project's `.venv`). +/// +/// uv writes `pyvenv.cfg` before creating `site-packages`. If these events arrive in separate +/// batches, creating `site-packages` must retry resolution. A watcher may only report the creation +/// of the `site-packages` parent `lib` or `lib/pythonX.Y` directory, so we also handle those +/// directories (`lib64` on some Unix systems, `Lib` on Windows). We match `site-packages` by name; +/// an unrelated directory only causes an extra refresh of the same settings. +/// +/// Similar to `site-packages`, renaming a directory to `.venv` can change the inferred virtual +/// environment without an event for `pyvenv.cfg`. That's why we need to rediscover the virtual +/// environment when any ancestor path of a valid virtual environment location is created or deleted. +fn affects_python_environment( + change: &ChangeEvent, + virtual_environment: Option<&SystemPath>, + python_path: Option<&SystemPath>, +) -> bool { + let may_be_environment_root = |path: &SystemPath| { + virtual_environment == Some(path) + || python_path.is_some_and(|python_path| python_path.starts_with(path)) + }; + + match change { + ChangeEvent::Created { path, .. } + | ChangeEvent::Changed { path, .. } + | ChangeEvent::Deleted { path, .. } + if path.file_name() == Some("pyvenv.cfg") => + { + path.parent().is_some_and(may_be_environment_root) + } + // The configured path can be an interpreter file as well as an environment directory. + ChangeEvent::Created { path, .. } | ChangeEvent::Deleted { path, .. } + if python_path.is_some_and(|python_path| python_path.starts_with(path)) + || virtual_environment.is_some_and(|environment| environment.starts_with(path)) => + { + true + } + ChangeEvent::Created { + path, + kind: CreatedKind::Directory, + } + | ChangeEvent::Deleted { + path, + kind: DeletedKind::Directory | DeletedKind::Any, + } => { + if path.file_name() == Some("site-packages") { + return true; + } + + let is_library = + |path: &SystemPath| matches!(path.file_name(), Some("lib" | "lib64" | "Lib")); + let library = if is_library(path) { + Some(path.as_path()) + } else { + path.parent().filter(|parent| is_library(parent)) + }; + library + .and_then(SystemPath::parent) + .is_some_and(may_be_environment_root) + } + _ => false, + } +} diff --git a/crates/ty_project/src/metadata.rs b/crates/ty_project/src/metadata.rs index 95e0ba704f..02451852d5 100644 --- a/crates/ty_project/src/metadata.rs +++ b/crates/ty_project/src/metadata.rs @@ -549,6 +549,13 @@ impl ProjectMetadata { .chain(self.fallback_options.as_deref()) } + /// Returns the configured environment or interpreter path, without resolving the full merged options. + pub(crate) fn configured_python_path(&self, system: &dyn System) -> Option { + self.options_in_precedence_order(&self.options, self.uv_workspace_options.as_deref()) + .find_map(|options| options.environment.as_ref()?.python.as_ref()) + .map(|path| path.absolute(self.root(), system)) + } + /// Loads the lower-precedence options from configuration files. /// /// This includes: diff --git a/crates/ty_project/src/metadata/options.rs b/crates/ty_project/src/metadata/options.rs index f79bca1000..10d34598a0 100644 --- a/crates/ty_project/src/metadata/options.rs +++ b/crates/ty_project/src/metadata/options.rs @@ -306,6 +306,9 @@ impl Options { python_version, python_platform, search_paths, + virtual_environment: python_environment + .filter(PythonEnvironment::is_virtual) + .map(|environment| environment.sys_prefix().to_path_buf()), }, diagnostics, )) diff --git a/crates/ty_project/src/watch/project_watcher.rs b/crates/ty_project/src/watch/project_watcher.rs index cf67e10213..7409637475 100644 --- a/crates/ty_project/src/watch/project_watcher.rs +++ b/crates/ty_project/src/watch/project_watcher.rs @@ -178,16 +178,43 @@ impl WatchPaths { } } -/// Watches are registered in project, module, then configuration order. On Linux, the last -/// registered watch determines the path reported for overlapping symlinks. +/// Watches are registered in project, Python environment, module, then configuration order. +/// On Linux, the last registered watch determines the path reported for overlapping symlinks. /// /// Project roots and explicitly included paths come first because project files are discovered -/// by walking them. Module search paths come next so imports are reported relative to their +/// by walking them. A virtual environment outside the project needs its own watch because +/// `pyvenv.cfg` can change the inferred Python version and import search paths. Watching its +/// `site-packages` alone does not cover that file. System environment roots are excluded to +/// avoid recursively watching a system prefix such as `/usr`. +/// +/// Module search paths follow the environment so imports are reported relative to their /// search roots, rather than through symlinks inside the project. Configuration paths come last /// so their events use the explicit paths checked for configuration changes. +/// +/// # Known limitations +/// +/// These limitations apply to virtual environments outside the project directory unless another +/// watch already covers the relevant paths. Environments inside the project are covered by the project watch. +/// +/// - Creating an initially missing environment: If environment resolution fails, there is no +/// resolved root to add to the watch list. Supporting later creation would require choosing an +/// existing ancestor to watch from an unresolved `environment.python` path, which can name +/// either a directory or an interpreter. We accept this limitation to avoid additional path +/// discovery and watch-registration logic for environments that could not be resolved. +/// - Deleting and recreating an environment: Deleting the environment can remove its native +/// watch, so its recreation goes unnoticed. Watching a surviving parent would cover this, but +/// our watches are recursive: the parent could contain every environment in a shared Poetry +/// directory, or even be the user's home directory. We accept this limitation to avoid watching +/// those unrelated directories. A nonrecursive parent watch would need additional logic to +/// register the environment watch again after recreation. #[salsa::tracked(returns(ref))] pub fn watch_paths(db: &dyn Db, project: Project) -> WatchPaths { let project_path = project.root(db); + let virtual_environment = project + .program_settings(db) + .virtual_environment + .as_deref() + .filter(|environment| !environment.starts_with(project_path)); // Watch both the project root and any paths provided by the user on the CLI (removing any redundant nested paths). // This is necessary to observe changes to files that are outside the project root. @@ -212,15 +239,16 @@ pub fn watch_paths(db: &dyn Db, project: Project) -> WatchPaths { } } - // The project watch covers search paths inside its root. Deduplicate the others so - // shared or nested search paths are registered once in stable order. - let unique_module_paths = ruff_db::system::deduplicate_nested_paths( - search_paths - .into_iter() - .filter(|path| !path.starts_with(project_path)), - ); + // The project and virtual environment watches cover search paths inside their roots. + // Deduplicate the remaining paths so shared or nested search paths are registered once. + let unique_module_paths = + ruff_db::system::deduplicate_nested_paths(search_paths.into_iter().filter(|path| { + !path.starts_with(project_path) + && virtual_environment.is_none_or(|environment| !path.starts_with(environment)) + })); let paths: Vec<_> = included_paths + .chain(virtual_environment) .chain(unique_module_paths) .chain(project.metadata(db).extra_configuration_paths()) .map(SystemPath::to_path_buf) diff --git a/crates/ty_python_core/src/db.rs b/crates/ty_python_core/src/db.rs index 93ba90c44f..42459247c9 100644 --- a/crates/ty_python_core/src/db.rs +++ b/crates/ty_python_core/src/db.rs @@ -166,6 +166,7 @@ pub(crate) mod tests { .context("Failed to write test files")?; let program_settings = ProgramSettings { + virtual_environment: None, python_version: PythonVersionWithSource { version: self.python_version, source: PythonVersionSource::default(), diff --git a/crates/ty_python_core/src/program.rs b/crates/ty_python_core/src/program.rs index f46480e1e6..a03dedd09c 100644 --- a/crates/ty_python_core/src/program.rs +++ b/crates/ty_python_core/src/program.rs @@ -1,7 +1,7 @@ use crate::{Db, platform::PythonPlatform}; use ruff_db::files::File; -use ruff_db::system::SystemPath; +use ruff_db::system::{SystemPath, SystemPathBuf}; use ruff_db::vendored::VendoredFileSystem; use ruff_python_ast::PythonVersion; use ty_module_resolver::{ResolverEnvironment, SearchPaths}; @@ -30,6 +30,7 @@ impl<'db> Program<'db> { python_version, python_platform, search_paths, + virtual_environment: _, } = settings; let resolver_environment = @@ -59,6 +60,11 @@ pub struct ProgramSettings { pub python_version: PythonVersionWithSource, pub python_platform: PythonPlatform, pub search_paths: SearchPaths, + /// The root of the resolved virtual environment, if any. File watchers use this to observe + /// `pyvenv.cfg` and directory changes without resolving the environment again. System Python + /// installations are very unlikely to be deleted and recreated, so we exclude them to avoid + /// recursively watching a system prefix such as `/usr`. + pub virtual_environment: Option, } impl ProgramSettings { @@ -67,6 +73,7 @@ impl ProgramSettings { python_version: PythonVersionWithSource::default(), python_platform: PythonPlatform::default(), search_paths: SearchPaths::empty(vendored), + virtual_environment: None, } } } diff --git a/crates/ty_python_semantic/src/db.rs b/crates/ty_python_semantic/src/db.rs index 3efc98bd86..571fa4e242 100644 --- a/crates/ty_python_semantic/src/db.rs +++ b/crates/ty_python_semantic/src/db.rs @@ -318,6 +318,9 @@ pub(crate) mod tests { }; let program_settings = ProgramSettings { + virtual_environment: self + .third_party_packages + .then(|| SystemPathBuf::from("/.venv")), python_version: PythonVersionWithSource { version: self.python_version, source: PythonVersionSource::default(), diff --git a/crates/ty_python_semantic/src/types/class/known.rs b/crates/ty_python_semantic/src/types/class/known.rs index 3a167b2397..51a2c55e6b 100644 --- a/crates/ty_python_semantic/src/types/class/known.rs +++ b/crates/ty_python_semantic/src/types/class/known.rs @@ -2235,11 +2235,10 @@ impl<'db> KnownClassLookupError<'db> { mod tests { use super::*; use crate::db::tests::{TestDbBuilder, setup_db}; - use crate::{PythonVersionSource, PythonVersionWithSource}; use strum::IntoEnumIterator; use ty_module_resolver::resolve_module_confident; use ty_python_core::TestProgramDb as _; - use ty_python_core::program::{Program, ProgramSettings}; + use ty_python_core::program::Program; #[test] fn known_class_roundtrip_from_str() { @@ -2335,22 +2334,12 @@ mod tests { classes.sort_unstable_by_key(|(_, version)| *version); let mut program = db.program(); - let mut current_version = program.python_version(&db); - let python_platform = program.python_platform(&db).clone(); - let search_paths = program.search_paths(&db).clone(); + let mut settings = db.program_settings().clone(); for (class, version_added) in classes { - if version_added != current_version { - let settings = ProgramSettings { - python_version: PythonVersionWithSource { - version: version_added, - source: PythonVersionSource::default(), - }, - python_platform: python_platform.clone(), - search_paths: search_paths.clone(), - }; + if version_added != settings.python_version.version { + settings.python_version.version = version_added; program = Program::from_settings(&db, &settings); - current_version = version_added; } // Check the class can be looked up successfully diff --git a/crates/ty_python_semantic/src/types/infer/tests.rs b/crates/ty_python_semantic/src/types/infer/tests.rs index 61c5063d05..4487443e0e 100644 --- a/crates/ty_python_semantic/src/types/infer/tests.rs +++ b/crates/ty_python_semantic/src/types/infer/tests.rs @@ -15,12 +15,11 @@ use ruff_python_ast::PythonVersion; use salsa::Database as _; use salsa::plumbing::AsId; use ty_python_core::definition::Definition; -use ty_python_core::program::{Program, ProgramSettings}; +use ty_python_core::program::Program; use ty_python_core::scope::FileScopeId; use ty_python_core::{ ProgramFile, TestProgramDb as _, global_scope, place_table, semantic_index, use_def_map, }; -use ty_site_packages::{PythonVersionSource, PythonVersionWithSource}; use super::*; @@ -119,39 +118,10 @@ fn same_file_at_different_python_versions() -> anyhow::Result<()> { db.write_dedented("src/py312_dependency.py", "value: int = 312")?; let file = system_path_to_file(&db, "src/main.py").expect("file to exist"); - let default_program = db.program(); - let search_paths = default_program.search_paths(&db).clone(); - let python_platform = default_program.python_platform(&db).clone(); - let py311 = ProgramFile::new( - &db, - file, - Program::from_settings( - &db, - &ProgramSettings { - python_version: PythonVersionWithSource { - version: PythonVersion::PY311, - source: PythonVersionSource::Default, - }, - python_platform: python_platform.clone(), - search_paths: search_paths.clone(), - }, - ), - ); - let py312 = ProgramFile::new( - &db, - file, - Program::from_settings( - &db, - &ProgramSettings { - python_version: PythonVersionWithSource { - version: PythonVersion::PY312, - source: PythonVersionSource::Default, - }, - python_platform, - search_paths, - }, - ), - ); + let py311 = db.program().program_file(&db, file); + let mut settings = db.program_settings().clone(); + settings.python_version.version = PythonVersion::PY312; + let py312 = Program::from_settings(&db, &settings).program_file(&db, file); let check = |file, expected_type, expect_invalid_syntax, expect_unresolved_import| { let diagnostics = crate::check_file_unwrap(&db, file); @@ -204,31 +174,16 @@ fn program_file_changes_with_python_version() -> anyhow::Result<()> { (program_file.as_id(), program_file.python_file(&db).as_id()) }; - let equivalent_program = Program::from_settings( - &db, - &ProgramSettings { - python_version: db.program_settings().python_version.clone(), - python_platform: program.python_platform(&db).clone(), - search_paths: program.search_paths(&db).clone(), - }, - ); + let mut settings = db.program_settings().clone(); + let equivalent_program = Program::from_settings(&db, &settings); assert_eq!(program, equivalent_program); assert_eq!( program_file_id, equivalent_program.program_file(&db, file).as_id() ); - let py312_program = Program::from_settings( - &db, - &ProgramSettings { - python_version: PythonVersionWithSource { - version: PythonVersion::PY312, - source: PythonVersionSource::Default, - }, - python_platform: program.python_platform(&db).clone(), - search_paths: program.search_paths(&db).clone(), - }, - ); + settings.python_version.version = PythonVersion::PY312; + let py312_program = Program::from_settings(&db, &settings); let program_file = py312_program.program_file(&db, file); assert_ne!(program_file_id, program_file.as_id()); diff --git a/crates/ty_server/tests/e2e/file_watching.rs b/crates/ty_server/tests/e2e/file_watching.rs index 84b19d0502..8992e3582c 100644 --- a/crates/ty_server/tests/e2e/file_watching.rs +++ b/crates/ty_server/tests/e2e/file_watching.rs @@ -180,8 +180,8 @@ fn picks_up_edits_to_unwatched_pth_files() -> Result<()> { assert_json_snapshot!(watcher_snapshot(&watches), @r#" [ "file:///src :: **", - "file:///old :: **", - "file:///venv/ :: **" + "file:///venv :: **", + "file:///old :: **" ] "#); server.open_text_document(main, &source, 1); @@ -218,8 +218,8 @@ fn picks_up_edits_to_unwatched_pth_files() -> Result<()> { assert_json_snapshot!(watcher_snapshot(&watches), @r#" [ "file:///src :: **", - "file:///old :: **", - "file:///venv/ :: **" + "file:///venv :: **", + "file:///old :: **" ] "#); server.acknowledge_request(request_id); @@ -230,8 +230,8 @@ fn picks_up_edits_to_unwatched_pth_files() -> Result<()> { assert_json_snapshot!(watcher_snapshot(&watches), @r#" [ "file:///src :: **", - "file:///new :: **", - "file:///venv/ :: **" + "file:///venv :: **", + "file:///new :: **" ] "#); assert_eq!(server.acknowledge_unregistration()?, stale_pth_id); diff --git a/crates/ty_site_packages/src/lib.rs b/crates/ty_site_packages/src/lib.rs index 7bb209482d..2bb387857a 100644 --- a/crates/ty_site_packages/src/lib.rs +++ b/crates/ty_site_packages/src/lib.rs @@ -296,35 +296,16 @@ impl PythonEnvironment { PythonEnvironment::new(path, origin, system) } - if let Ok(virtual_env) = system.env_var(EnvVars::VIRTUAL_ENV) { - return resolve_environment( - system, - SystemPath::new(&virtual_env), - SysPrefixPathOrigin::VirtualEnvVar, - ) - .map(Some); - } - - if let Some(conda_env) = conda_environment_from_env(system, CondaEnvironmentKind::Child) { - return resolve_environment(system, &conda_env, SysPrefixPathOrigin::CondaPrefixVar) - .map(Some); - } - - if let Some(project_root) = project_root { - tracing::debug!("Discovering virtual environment in `{project_root}`"); - let virtual_env_directory = project_root.join(".venv"); - - match PythonEnvironment::new( - &virtual_env_directory, - SysPrefixPathOrigin::LocalVenv, - system, - ) { + if let Some((path, origin)) = Self::virtual_environment_candidate(project_root, system) { + let is_local_venv = matches!(origin, SysPrefixPathOrigin::LocalVenv); + match resolve_environment(system, &path, origin) { Ok(environment) => return Ok(Some(environment)), + Err(err) if !is_local_venv => return Err(err), Err(err) => { - if system.is_directory(&virtual_env_directory) { + if system.is_directory(&path) { tracing::debug!( "Ignoring automatically detected virtual environment at `{}`: {}", - &virtual_env_directory, + &path, err ); } @@ -346,6 +327,24 @@ impl PythonEnvironment { Ok(None) } + /// Returns the virtual environment location to try and its origin, without checking whether + /// it exists. `VIRTUAL_ENV` takes precedence over a child Conda environment, followed by the + /// project's `.venv`. System Python fallbacks are handled by [`Self::discover`]. + pub fn virtual_environment_candidate( + project_root: Option<&SystemPath>, + system: &dyn System, + ) -> Option<(SystemPathBuf, SysPrefixPathOrigin)> { + if let Ok(virtual_env) = system.env_var(EnvVars::VIRTUAL_ENV) { + return Some((virtual_env.into(), SysPrefixPathOrigin::VirtualEnvVar)); + } + + if let Some(conda_env) = conda_environment_from_env(system, CondaEnvironmentKind::Child) { + return Some((conda_env, SysPrefixPathOrigin::CondaPrefixVar)); + } + + project_root.map(|root| (root.join(".venv"), SysPrefixPathOrigin::LocalVenv)) + } + pub fn new( path: impl AsRef, origin: SysPrefixPathOrigin, diff --git a/crates/ty_test/src/lib.rs b/crates/ty_test/src/lib.rs index f07f95a4b2..be3849839e 100644 --- a/crates/ty_test/src/lib.rs +++ b/crates/ty_test/src/lib.rs @@ -246,29 +246,25 @@ fn run_test( let configuration = test.configuration(); - let site_packages_paths = if configuration.dependencies().is_some() { + let python = if configuration.dependencies().is_some() { // If dependencies were specified, use the venv we just set up - let environment = PythonEnvironment::new( - &venv_for_external_dependencies, - SysPrefixPathOrigin::PythonCliFlag, - db.system(), - ) - .expect("Python environment to point to a valid path"); - environment - .site_packages_paths(db.system()) - .expect("Python environment to be valid") - .into_vec() - } else if let Some(python) = configuration.python() { - let environment = - PythonEnvironment::new(python, SysPrefixPathOrigin::PythonCliFlag, db.system()) - .expect("Python environment to point to a valid path"); - environment - .site_packages_paths(db.system()) - .expect("Python environment to be valid") - .into_vec() + Some(venv_for_external_dependencies.as_path()) } else { - vec![] + configuration.python() }; + let python_environment = python.map(|python| { + PythonEnvironment::new(python, SysPrefixPathOrigin::PythonCliFlag, db.system()) + .expect("Python environment to point to a valid path") + }); + let site_packages_paths = python_environment + .as_ref() + .map(|environment| { + environment + .site_packages_paths(db.system()) + .expect("Python environment to be valid") + .into_vec() + }) + .unwrap_or_default(); // Make any relative extra-paths be relative to src_path let extra_paths = configuration @@ -286,6 +282,9 @@ fn run_test( .collect(); let settings = ProgramSettings { + virtual_environment: python_environment + .filter(PythonEnvironment::is_virtual) + .map(|environment| environment.sys_prefix().to_path_buf()), python_version: PythonVersionWithSource { version: python_version, source: PythonVersionSource::Cli, diff --git a/fuzz/fuzz_targets/ty_check_invalid_syntax.rs b/fuzz/fuzz_targets/ty_check_invalid_syntax.rs index 0416895942..ad9da8f457 100644 --- a/fuzz/fuzz_targets/ty_check_invalid_syntax.rs +++ b/fuzz/fuzz_targets/ty_check_invalid_syntax.rs @@ -16,7 +16,6 @@ use ruff_db::system::{ use ruff_db::vendored::VendoredFileSystem; use ruff_python_parser::{Mode, ParseOptions, parse_unchecked}; use ty_module_resolver::{Db as ModuleResolverDb, SearchPathSettings}; -use ty_python_core::platform::PythonPlatform; use ty_python_core::program::{FallibleStrategy, ProgramSettings}; use ty_python_core::{Db as _, ProgramFile, TestProgramDb}; use ty_python_semantic::dependency::DependencyMetadata; @@ -65,15 +64,11 @@ impl TestDb { .create_directory_all(&src_root) .unwrap(); - let program_settings = ProgramSettings { - python_version: PythonVersionWithSource::default(), - python_platform: PythonPlatform::default(), - search_paths: SearchPathSettings::new(vec![src_root]) - .to_search_paths(db.system(), db.vendored(), &FallibleStrategy) - .expect("Valid search path settings"), - }; - program_settings.search_paths.try_register_static_roots(&db); - db.program_settings = program_settings; + let search_paths = SearchPathSettings::new(vec![src_root]) + .to_search_paths(db.system(), db.vendored(), &FallibleStrategy) + .expect("Valid search path settings"); + search_paths.try_register_static_roots(&db); + db.program_settings.search_paths = search_paths; db } diff --git a/hawk.toml b/hawk.toml index ad09a63ae6..de8bf64cdc 100644 --- a/hawk.toml +++ b/hawk.toml @@ -574,14 +574,6 @@ kind = "inherent_method" level = "expect" reason = "TextSize intentionally provides both checked and saturating arithmetic" -[[override]] -lint = "hawk::dead_public" -crate = "ty_project" -item = "db::changes::ChangeResult::custom_stdlib_changed" -kind = "inherent_method" -level = "expect" -reason = "change results expose both project and custom-stdlib invalidation state" - [[override]] lint = "hawk::dead_public" crate = "ruff_python_ast"