diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 71f934756..340bec3c7 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -12,8 +12,10 @@ from test import support from test.support.script_helper import run_python_until_end -# Set the list of ways we expect to be able to ask for the "C" locale -EXPECTED_C_LOCALE_EQUIVALENTS = ["C", "invalid.ascii"] +# Set the list of ways we expect to be able to ask for the "C" locale. +# 'invalid.ascii' is an invalid LOCALE name and so should get turned in to the +# default locale, which is traditionally C. +EXPECTED_C_LOCALE_EQUIVALENTS = ["C", "POSIX", "invalid.ascii"] # Set our expectation for the default encoding used in the C locale # for the filesystem encoding and the standard streams @@ -21,22 +23,22 @@ EXPECTED_C_LOCALE_STREAM_ENCODING = "ascii" EXPECTED_C_LOCALE_FS_ENCODING = "ascii" # Set our expectation for the default locale used when none is specified +DEFAULT_LOCALE_IS_C = True EXPECT_COERCION_IN_DEFAULT_LOCALE = True TARGET_LOCALES = ["C.UTF-8", "C.utf8", "UTF-8"] # Apply some platform dependent overrides -if sys.platform.startswith("linux"): - if support.is_android: - # Android defaults to using UTF-8 for all system interfaces - EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" - EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" - else: - # Linux distros typically alias the POSIX locale directly to the C - # locale. - # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be - # able to check this case unconditionally - EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") +if sys.platform == "android": + # Android defaults to using UTF-8 for all system interfaces + EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" + EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" +elif support.linked_to_musl(): + # MUSL defaults to utf-8 unless the C locale is set explicitly. + EXPECTED_C_LOCALE_EQUIVALENTS = ["C"] + DEFAULT_LOCALE_IS_C = False + DEFAULT_ENCODING = 'utf-8' + EXPECT_COERCION_IN_DEFAULT_LOCALE = False elif sys.platform.startswith("aix"): # AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII EXPECTED_C_LOCALE_STREAM_ENCODING = "iso8859-1" @@ -112,12 +114,16 @@ class EncodingDetails(_EncodingDetails): ]) @classmethod - def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, env_vars): + def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, stream_errors, env_vars): """Returns expected child process details for a given encoding""" _stream = stream_encoding + ":{}" - # stdin and stdout should use surrogateescape either because the - # coercion triggered, or because the C locale was detected - stream_info = 2*[_stream.format("surrogateescape")] + if stream_errors is None: + # stdin and stdout should use surrogateescape either because the + # coercion triggered, or because the C locale was detected + stream_errors = "surrogateescape" + + stream_info = [_stream.format(stream_errors)] * 2 + # stderr should always use backslashreplace stream_info.append(_stream.format("backslashreplace")) expected_lang = env_vars.get("LANG", "not set") @@ -210,6 +216,7 @@ class _LocaleHandlingTestCase(unittest.TestCase): env_vars, expected_fs_encoding, expected_stream_encoding, + expected_stream_errors, expected_warnings, coercion_expected): """Check the C locale handling for the given process environment @@ -225,6 +232,7 @@ class _LocaleHandlingTestCase(unittest.TestCase): coercion_expected, expected_fs_encoding, expected_stream_encoding, + expected_stream_errors, env_vars ) self.assertEqual(encoding_details, expected_details) @@ -257,6 +265,7 @@ class LocaleConfigurationTests(_LocaleHandlingTestCase): "LC_CTYPE": "", "LC_ALL": "", "PYTHONCOERCECLOCALE": "", + "PYTHONIOENCODING": "", } for env_var in ("LANG", "LC_CTYPE"): for locale_to_set in AVAILABLE_TARGETS: @@ -273,10 +282,43 @@ class LocaleConfigurationTests(_LocaleHandlingTestCase): self._check_child_encoding_details(var_dict, expected_fs_encoding, expected_stream_encoding, + expected_stream_errors=None, expected_warnings=None, coercion_expected=False) + def test_with_ioencoding(self): + # Explicitly setting a target locale should give the same behaviour as + # is seen when implicitly coercing to that target locale + self.maxDiff = None + expected_fs_encoding = "utf-8" + expected_stream_encoding = "utf-8" + + base_var_dict = { + "LANG": "", + "LC_CTYPE": "", + "LC_ALL": "", + "PYTHONCOERCECLOCALE": "", + "PYTHONIOENCODING": "UTF-8", + } + for env_var in ("LANG", "LC_CTYPE"): + for locale_to_set in AVAILABLE_TARGETS: + # XXX (ncoghlan): LANG=UTF-8 doesn't appear to work as + # expected, so skip that combination for now + # See https://bugs.python.org/issue30672 for discussion + if env_var == "LANG" and locale_to_set == "UTF-8": + continue + + with self.subTest(env_var=env_var, + configured_locale=locale_to_set): + var_dict = base_var_dict.copy() + var_dict[env_var] = locale_to_set + self._check_child_encoding_details(var_dict, + expected_fs_encoding, + expected_stream_encoding, + expected_stream_errors="strict", + expected_warnings=None, + coercion_expected=False) @support.cpython_only @unittest.skipUnless(sysconfig.get_config_var("PY_COERCE_C_LOCALE"), @@ -316,15 +358,21 @@ class LocaleCoercionTests(_LocaleHandlingTestCase): "LC_CTYPE": "", "LC_ALL": "", "PYTHONCOERCECLOCALE": "", + "PYTHONIOENCODING": "", } base_var_dict.update(extra_vars) if coerce_c_locale is not None: base_var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale # Check behaviour for the default locale + _fs_encoding = fs_encoding + _stream_encoding = stream_encoding + if not DEFAULT_LOCALE_IS_C and 'LC_ALL' not in extra_vars: + _fs_encoding = _stream_encoding = DEFAULT_ENCODING with self.subTest(default_locale=True, PYTHONCOERCECLOCALE=coerce_c_locale): - if EXPECT_COERCION_IN_DEFAULT_LOCALE: + if (EXPECT_COERCION_IN_DEFAULT_LOCALE + or (not DEFAULT_LOCALE_IS_C and 'LC_ALL' in extra_vars)): _expected_warnings = expected_warnings _coercion_expected = coercion_expected else: @@ -338,8 +386,9 @@ class LocaleCoercionTests(_LocaleHandlingTestCase): _expected_warnings == [CLI_COERCION_WARNING]): _expected_warnings = None self._check_child_encoding_details(base_var_dict, - fs_encoding, - stream_encoding, + _fs_encoding, + _stream_encoding, + None, _expected_warnings, _coercion_expected) @@ -348,13 +397,15 @@ class LocaleCoercionTests(_LocaleHandlingTestCase): for env_var in ("LANG", "LC_CTYPE"): with self.subTest(env_var=env_var, nominal_locale=locale_to_set, - PYTHONCOERCECLOCALE=coerce_c_locale): + PYTHONCOERCECLOCALE=coerce_c_locale, + PYTHONIOENCODING=""): var_dict = base_var_dict.copy() var_dict[env_var] = locale_to_set # Check behaviour on successful coercion self._check_child_encoding_details(var_dict, fs_encoding, stream_encoding, + None, expected_warnings, coercion_expected) @@ -411,8 +462,9 @@ class LocaleCoercionTests(_LocaleHandlingTestCase): loc = locale.setlocale(locale.LC_CTYPE, "") except locale.Error as e: self.skipTest(str(e)) - if loc == "C": - self.skipTest("test requires LC_CTYPE locale different than C") + if loc in ("C", "POSIX"): + self.skipTest("test requires LC_CTYPE locale different " + "than C and POSIX") if loc in TARGET_LOCALES : self.skipTest("coerced LC_CTYPE locale: %s" % loc) diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py index 3990eb8be..cf8df09d5 100644 --- a/Lib/test/test_utf8_mode.py +++ b/Lib/test/test_utf8_mode.py @@ -66,7 +66,7 @@ class UTF8ModeTests(unittest.TestCase): PYTHONLEGACYWINDOWSFSENCODING='1') self.assertEqual(out, '0') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailureIf(MS_WINDOWS, "TODO: RUSTPYTHON") def test_env_var(self): code = 'import sys; print(sys.flags.utf8_mode)' @@ -91,8 +91,8 @@ class UTF8ModeTests(unittest.TestCase): # the UTF-8 mode if not self.posix_locale(): # PYTHONUTF8 should be ignored if -E is used - out = self.get_output('-E', '-c', code, PYTHONUTF8='1') - self.assertEqual(out, '0') + out = self.get_output('-E', '-c', code, PYTHONUTF8='0') + self.assertEqual(out, '1') # invalid mode out = self.get_output('-c', code, PYTHONUTF8='xxx', failure=True) @@ -119,11 +119,10 @@ class UTF8ModeTests(unittest.TestCase): # PYTHONLEGACYWINDOWSFSENCODING disables the UTF-8 mode # and has the priority over -X utf8 and PYTHONUTF8 out = self.get_output('-X', 'utf8', '-c', code, - PYTHONUTF8='strict', + PYTHONUTF8='xxx', PYTHONLEGACYWINDOWSFSENCODING='1') self.assertEqual(out, 'mbcs/replace') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_stdio(self): code = textwrap.dedent(''' import sys diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index f265b731b..bb46b8694 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1568,6 +1568,25 @@ impl VirtualMachine { { let io = import::import_builtin(self, "_io")?; + // CPython normalizes configured stdio encodings to CodecInfo.name. + #[cfg(all(feature = "host_env", feature = "stdio"))] + let stdio_encoding: Option> = self + .state + .config + .settings + .stdio_encoding + .as_deref() + .map(|encoding| { + self.state + .codec_registry + .lookup(encoding, self)? + .as_tuple() + .as_object() + .get_attr("name", self)? + .try_into_value(self) + }) + .transpose()?; + // Full stdio: FileIO → BufferedWriter → TextIOWrapper #[cfg(all(feature = "host_env", feature = "stdio"))] let make_stdio = |name: &str, fd: i32, write: bool| -> PyResult { @@ -1594,7 +1613,7 @@ impl VirtualMachine { let line_buffering = buffered_stdio && (isatty || fd == 2); let newline = if cfg!(windows) { None } else { Some("\n") }; - let encoding = self.state.config.settings.stdio_encoding.as_deref(); + let encoding = stdio_encoding.clone(); // stderr always uses backslashreplace (ignores stdio_errors) let errors = if fd == 2 { Some("backslashreplace") diff --git a/scripts/update_lib/patch_spec.py b/scripts/update_lib/patch_spec.py index bf7d0f411..6b7c1f93f 100644 --- a/scripts/update_lib/patch_spec.py +++ b/scripts/update_lib/patch_spec.py @@ -635,11 +635,21 @@ def select_remaining_patches( pending = list(existing) omitted = [] for marker in remaining.get(cls, {}).get(method, []): - if marker not in pending: + original = marker + if ( + original not in pending + and marker.ut_method == UtMethod.ExpectedFailureIf + and marker.cond + ): + # A condition only retires part of an unconditional failure. + original = marker._replace( + ut_method=UtMethod.ExpectedFailure, cond=None + ) + if original not in pending: raise ValueError( "Marker override cannot add, change, or reorder markers" ) - index = pending.index(marker) + index = pending.index(original) omitted.extend(pending[:index]) pending = pending[index + 1 :] omitted.extend(pending) diff --git a/scripts/update_lib/tests/test_migrate.py b/scripts/update_lib/tests/test_migrate.py index 5b1daa19f..5ab1bd8c3 100644 --- a/scripts/update_lib/tests/test_migrate.py +++ b/scripts/update_lib/tests/test_migrate.py @@ -601,6 +601,66 @@ class MarkerOverrideTests(unittest.TestCase): with self.subTest(markers=markers), self.assertRaises(ValueError): select_remaining_patches(current, document, Path(document["target"])) + def test_cli_narrows_unconditional_failure(self): + def narrow(document): + document["patches"]["Example"]["test_known"] = [ + { + "ut_method": "expectedFailureIf", + "cond": "MS_WINDOWS", + "reason": "old failure", + } + ] + + status, result = self.invoke(narrow) + self.assertEqual(status, 0) + self.assertIn( + 'unittest.expectedFailureIf(MS_WINDOWS, "TODO: RUSTPYTHON; old failure")', + result, + ) + self.assertIn('unittest.skip("TODO: RUSTPYTHON; retained guard")', result) + self.assertIn("self.assertEqual(1, 1)", result) + + def test_narrowing_preserves_reason_order_and_existing_targets(self): + failure = PatchSpec(UtMethod.ExpectedFailure, reason="known failure") + narrowed = failure._replace( + ut_method=UtMethod.ExpectedFailureIf, cond="MS_WINDOWS" + ) + success = PatchSpec(UtMethod.ExpectedSuccess, reason="known success") + current = {"Example": {"test_known": [failure, success]}} + for markers in ( + [narrowed, success, narrowed], + [narrowed._replace(reason="changed"), success], + [success, narrowed], + [narrowed._replace(ut_method=UtMethod.SkipIf), success], + [narrowed._replace(ut_method=UtMethod.ExpectedSuccess), success], + [narrowed._replace(cond=None), success], + ): + document = { + "target": "Lib/test/test_example.py", + "patches": patches_to_json({"Example": {"test_known": markers}}), + } + with self.subTest(markers=markers), self.assertRaises(ValueError): + select_remaining_patches(current, document, Path(document["target"])) + + document["patches"] = patches_to_json({"Example": {"test_new": [narrowed]}}) + with self.assertRaises(ValueError): + select_remaining_patches(current, document, Path(document["target"])) + + def test_conditional_failure_cannot_be_broadened_or_changed(self): + conditional = PatchSpec(UtMethod.ExpectedFailureIf, cond="MS_WINDOWS") + current = {"Example": {"test_known": [conditional]}} + for marker in ( + conditional._replace(ut_method=UtMethod.ExpectedFailure, cond=None), + conditional._replace(cond="True"), + conditional._replace(cond="not MS_WINDOWS"), + ): + document = { + "target": "Lib/test/test_example.py", + "patches": patches_to_json({"Example": {"test_known": [marker]}}), + } + with self.subTest(marker=marker), self.assertRaises(ValueError): + select_remaining_patches(current, document, Path(document["target"])) + class ConditionalMarkerOverrideTests(unittest.TestCase): decorator = ( diff --git a/src/settings.rs b/src/settings.rs index 3c744f4aa..a7c1591b4 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -295,24 +295,6 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { settings.check_hash_pycs_mode = args.check_hash_based_pycs; - if let Some(val) = get_env("PYTHONUTF8") - && let Some(val_str) = val.to_str() - && !val_str.is_empty() - { - settings.utf8_mode = match val_str { - "1" => 1, - "0" => 0, - _ => { - error!( - "Fatal Python error: config_init_utf8_mode: \ - PYTHONUTF8=N: N is missing or invalid\n\ - Python runtime state: preinitialized" - ); - std::process::exit(1); - } - }; - } - if let Some(val) = get_env("PYTHON_LAZY_IMPORTS").filter(|val| !val.is_empty()) { settings.lazy_imports = parse_lazy_imports(val.to_str(), "PYTHON_LAZY_IMPORTS"); } @@ -330,6 +312,7 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { let mut lazy_imports_option_seen = false; let mut import_time_option_seen = false; let mut frozen_modules_option_seen = false; + let mut utf8_option_seen = false; let xopts = args.implementation_option.into_iter().map(|s| { let (name, value) = match s.split_once('=') { Some((name, value)) => (name.to_owned(), Some(value)), @@ -348,20 +331,21 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { } "faulthandler" => settings.faulthandler = true, "warn_default_encoding" => settings.warn_default_encoding = true, - "utf8" => { + "utf8" if !utf8_option_seen => { settings.utf8_mode = match value { None => 1, Some("1") => 1, Some("0") => 0, _ => { error!( - "Fatal Python error: config_init_utf8_mode: \ - -X utf8=n: n is missing or invalid\n\ + "Fatal Python error: preconfig_init_utf8_mode: \ + invalid -X utf8 option value\n\ Python runtime state: preinitialized" ); std::process::exit(1); } }; + utf8_option_seen = true; } "no_sig_int" => settings.install_signal_handlers = false, "no_debug_ranges" => settings.code_debug_ranges = false, @@ -417,6 +401,22 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { }); settings.xoptions.extend(xopts); + if !utf8_option_seen && let Some(val) = get_env("PYTHONUTF8").filter(|value| !value.is_empty()) + { + settings.utf8_mode = match val.to_str() { + Some("1") => 1, + Some("0") => 0, + _ => { + error!( + "Fatal Python error: preconfig_init_utf8_mode: \ + invalid PYTHONUTF8 environment variable value\n\ + Python runtime state: preinitialized" + ); + std::process::exit(1); + } + }; + } + // CPython validates the first frozen_modules option before initialization, // then getpath's config dictionary round-trip keeps the last occurrence. if let Some((_, value)) = settings