Fix UTF-8 startup option precedence and refresh canonical tests (#9422)

* Fix UTF-8 startup option precedence and refresh canonical tests

Resolve the first explicit UTF-8 option before environment validation and align startup diagnostics with CPython 3.15. Refresh canonical encoding tests through update_lib and narrow the existing environment test failure to Windows. Support only monotonic expected-failure narrowing in updater metadata with focused regression tests.

Assisted-by: OpenAI Codex: model version unavailable

* Normalize configured standard-stream codec names

Resolve explicit stdio encodings through CodecInfo.name before creating standard streams, matching CPython initialization. Preserve ordinary TextIOWrapper spelling and existing standard-stream error policies; leave canonical tests unchanged.

Assisted-by: OpenAI Codex: model version unavailable

* Retire passing standard-stream UTF-8 expected failure

The configured-codec normalization fix makes canonical test_stdio pass in both Linux CI attempts. Remove only its stale expectedFailure through update_lib, preserving canonical assertions and all other platform guards.

Assisted-by: OpenAI Codex: model version unavailable

---------

Co-authored-by: Codex <codex@openai.com>

Lib/test/test_c_locale_coercion.py

@@ -12,8 +12,10 @@ from test import support
 from test.support.script_helper import run_python_until_end
 
 
-# Set the list of ways we expect to be able to ask for the "C" locale
-EXPECTED_C_LOCALE_EQUIVALENTS = ["C", "invalid.ascii"]
+# Set the list of ways we expect to be able to ask for the "C" locale.
+# 'invalid.ascii' is an invalid LOCALE name and so should get turned in to the
+# default locale, which is traditionally C.
+EXPECTED_C_LOCALE_EQUIVALENTS = ["C", "POSIX", "invalid.ascii"]
 
 # Set our expectation for the default encoding used in the C locale
 # for the filesystem encoding and the standard streams
@@ -21,22 +23,22 @@ EXPECTED_C_LOCALE_STREAM_ENCODING = "ascii"
 EXPECTED_C_LOCALE_FS_ENCODING = "ascii"
 
 # Set our expectation for the default locale used when none is specified
+DEFAULT_LOCALE_IS_C = True
 EXPECT_COERCION_IN_DEFAULT_LOCALE = True
 
 TARGET_LOCALES = ["C.UTF-8", "C.utf8", "UTF-8"]
 
 # Apply some platform dependent overrides
-if sys.platform.startswith("linux"):
-    if support.is_android:
-        # Android defaults to using UTF-8 for all system interfaces
-        EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8"
-        EXPECTED_C_LOCALE_FS_ENCODING = "utf-8"
-    else:
-        # Linux distros typically alias the POSIX locale directly to the C
-        # locale.
-        # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be
-        #       able to check this case unconditionally
-        EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX")
+if sys.platform == "android":
+    # Android defaults to using UTF-8 for all system interfaces
+    EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8"
+    EXPECTED_C_LOCALE_FS_ENCODING = "utf-8"
+elif support.linked_to_musl():
+    # MUSL defaults to utf-8 unless the C locale is set explicitly.
+    EXPECTED_C_LOCALE_EQUIVALENTS = ["C"]
+    DEFAULT_LOCALE_IS_C = False
+    DEFAULT_ENCODING = 'utf-8'
+    EXPECT_COERCION_IN_DEFAULT_LOCALE = False
 elif sys.platform.startswith("aix"):
     # AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII
     EXPECTED_C_LOCALE_STREAM_ENCODING = "iso8859-1"
@@ -112,12 +114,16 @@ class EncodingDetails(_EncodingDetails):
     ])
 
     @classmethod
-    def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, env_vars):
+    def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, stream_errors, env_vars):
         """Returns expected child process details for a given encoding"""
         _stream = stream_encoding + ":{}"
-        # stdin and stdout should use surrogateescape either because the
-        # coercion triggered, or because the C locale was detected
-        stream_info = 2*[_stream.format("surrogateescape")]
+        if stream_errors is None:
+            # stdin and stdout should use surrogateescape either because the
+            # coercion triggered, or because the C locale was detected
+            stream_errors = "surrogateescape"
+
+        stream_info = [_stream.format(stream_errors)] * 2
+
         # stderr should always use backslashreplace
         stream_info.append(_stream.format("backslashreplace"))
         expected_lang = env_vars.get("LANG", "not set")
@@ -210,6 +216,7 @@ class _LocaleHandlingTestCase(unittest.TestCase):
                                       env_vars,
                                       expected_fs_encoding,
                                       expected_stream_encoding,
+                                      expected_stream_errors,
                                       expected_warnings,
                                       coercion_expected):
         """Check the C locale handling for the given process environment
@@ -225,6 +232,7 @@ class _LocaleHandlingTestCase(unittest.TestCase):
             coercion_expected,
             expected_fs_encoding,
             expected_stream_encoding,
+            expected_stream_errors,
             env_vars
         )
         self.assertEqual(encoding_details, expected_details)
@@ -257,6 +265,7 @@ class LocaleConfigurationTests(_LocaleHandlingTestCase):
             "LC_CTYPE": "",
             "LC_ALL": "",
             "PYTHONCOERCECLOCALE": "",
+            "PYTHONIOENCODING": "",
         }
         for env_var in ("LANG", "LC_CTYPE"):
             for locale_to_set in AVAILABLE_TARGETS:
@@ -273,10 +282,43 @@ class LocaleConfigurationTests(_LocaleHandlingTestCase):
                     self._check_child_encoding_details(var_dict,
                                                        expected_fs_encoding,
                                                        expected_stream_encoding,
+                                                       expected_stream_errors=None,
                                                        expected_warnings=None,
                                                        coercion_expected=False)
 
+    def test_with_ioencoding(self):
+        # Explicitly setting a target locale should give the same behaviour as
+        # is seen when implicitly coercing to that target locale
+        self.maxDiff = None
 
+        expected_fs_encoding = "utf-8"
+        expected_stream_encoding = "utf-8"
+
+        base_var_dict = {
+            "LANG": "",
+            "LC_CTYPE": "",
+            "LC_ALL": "",
+            "PYTHONCOERCECLOCALE": "",
+            "PYTHONIOENCODING": "UTF-8",
+        }
+        for env_var in ("LANG", "LC_CTYPE"):
+            for locale_to_set in AVAILABLE_TARGETS:
+                # XXX (ncoghlan): LANG=UTF-8 doesn't appear to work as
+                #                 expected, so skip that combination for now
+                # See https://bugs.python.org/issue30672 for discussion
+                if env_var == "LANG" and locale_to_set == "UTF-8":
+                    continue
+
+                with self.subTest(env_var=env_var,
+                                  configured_locale=locale_to_set):
+                    var_dict = base_var_dict.copy()
+                    var_dict[env_var] = locale_to_set
+                    self._check_child_encoding_details(var_dict,
+                                                       expected_fs_encoding,
+                                                       expected_stream_encoding,
+                                                       expected_stream_errors="strict",
+                                                       expected_warnings=None,
+                                                       coercion_expected=False)
 
 @support.cpython_only
 @unittest.skipUnless(sysconfig.get_config_var("PY_COERCE_C_LOCALE"),
@@ -316,15 +358,21 @@ class LocaleCoercionTests(_LocaleHandlingTestCase):
             "LC_CTYPE": "",
             "LC_ALL": "",
             "PYTHONCOERCECLOCALE": "",
+            "PYTHONIOENCODING": "",
         }
         base_var_dict.update(extra_vars)
         if coerce_c_locale is not None:
             base_var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale
 
         # Check behaviour for the default locale
+        _fs_encoding = fs_encoding
+        _stream_encoding = stream_encoding
+        if not DEFAULT_LOCALE_IS_C and 'LC_ALL' not in extra_vars:
+            _fs_encoding = _stream_encoding = DEFAULT_ENCODING
         with self.subTest(default_locale=True,
                           PYTHONCOERCECLOCALE=coerce_c_locale):
-            if EXPECT_COERCION_IN_DEFAULT_LOCALE:
+            if (EXPECT_COERCION_IN_DEFAULT_LOCALE
+                    or (not DEFAULT_LOCALE_IS_C and 'LC_ALL' in extra_vars)):
                 _expected_warnings = expected_warnings
                 _coercion_expected = coercion_expected
             else:
@@ -338,8 +386,9 @@ class LocaleCoercionTests(_LocaleHandlingTestCase):
                     _expected_warnings == [CLI_COERCION_WARNING]):
                 _expected_warnings = None
             self._check_child_encoding_details(base_var_dict,
-                                               fs_encoding,
-                                               stream_encoding,
+                                               _fs_encoding,
+                                               _stream_encoding,
+                                               None,
                                                _expected_warnings,
                                                _coercion_expected)
 
@@ -348,13 +397,15 @@ class LocaleCoercionTests(_LocaleHandlingTestCase):
             for env_var in ("LANG", "LC_CTYPE"):
                 with self.subTest(env_var=env_var,
                                   nominal_locale=locale_to_set,
-                                  PYTHONCOERCECLOCALE=coerce_c_locale):
+                                  PYTHONCOERCECLOCALE=coerce_c_locale,
+                                  PYTHONIOENCODING=""):
                     var_dict = base_var_dict.copy()
                     var_dict[env_var] = locale_to_set
                     # Check behaviour on successful coercion
                     self._check_child_encoding_details(var_dict,
                                                        fs_encoding,
                                                        stream_encoding,
+                                                       None,
                                                        expected_warnings,
                                                        coercion_expected)
 
@@ -411,8 +462,9 @@ class LocaleCoercionTests(_LocaleHandlingTestCase):
             loc = locale.setlocale(locale.LC_CTYPE, "")
         except locale.Error as e:
             self.skipTest(str(e))
-        if loc == "C":
-            self.skipTest("test requires LC_CTYPE locale different than C")
+        if loc in ("C", "POSIX"):
+            self.skipTest("test requires LC_CTYPE locale different "
+                          "than C and POSIX")
         if loc in TARGET_LOCALES :
             self.skipTest("coerced LC_CTYPE locale: %s" % loc)
 

Lib/test/test_utf8_mode.py

@@ -66,7 +66,7 @@ class UTF8ModeTests(unittest.TestCase):
                                   PYTHONLEGACYWINDOWSFSENCODING='1')
             self.assertEqual(out, '0')
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
+    @unittest.expectedFailureIf(MS_WINDOWS, "TODO: RUSTPYTHON")
     def test_env_var(self):
         code = 'import sys; print(sys.flags.utf8_mode)'
 
@@ -91,8 +91,8 @@ class UTF8ModeTests(unittest.TestCase):
         # the UTF-8 mode
         if not self.posix_locale():
             # PYTHONUTF8 should be ignored if -E is used
-            out = self.get_output('-E', '-c', code, PYTHONUTF8='1')
-            self.assertEqual(out, '0')
+            out = self.get_output('-E', '-c', code, PYTHONUTF8='0')
+            self.assertEqual(out, '1')
 
         # invalid mode
         out = self.get_output('-c', code, PYTHONUTF8='xxx', failure=True)
@@ -119,11 +119,10 @@ class UTF8ModeTests(unittest.TestCase):
             # PYTHONLEGACYWINDOWSFSENCODING disables the UTF-8 mode
             # and has the priority over -X utf8 and PYTHONUTF8
             out = self.get_output('-X', 'utf8', '-c', code,
-                                  PYTHONUTF8='strict',
+                                  PYTHONUTF8='xxx',
                                   PYTHONLEGACYWINDOWSFSENCODING='1')
             self.assertEqual(out, 'mbcs/replace')
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_stdio(self):
         code = textwrap.dedent('''
             import sys

crates/vm/src/vm/mod.rs

@@ -1568,6 +1568,25 @@ impl VirtualMachine {
             {
                 let io = import::import_builtin(self, "_io")?;
 
+                // CPython normalizes configured stdio encodings to CodecInfo.name.
+                #[cfg(all(feature = "host_env", feature = "stdio"))]
+                let stdio_encoding: Option<PyRef<PyUtf8Str>> = self
+                    .state
+                    .config
+                    .settings
+                    .stdio_encoding
+                    .as_deref()
+                    .map(|encoding| {
+                        self.state
+                            .codec_registry
+                            .lookup(encoding, self)?
+                            .as_tuple()
+                            .as_object()
+                            .get_attr("name", self)?
+                            .try_into_value(self)
+                    })
+                    .transpose()?;
+
                 // Full stdio: FileIO → BufferedWriter → TextIOWrapper
                 #[cfg(all(feature = "host_env", feature = "stdio"))]
                 let make_stdio = |name: &str, fd: i32, write: bool| -> PyResult<PyObjectRef> {
@@ -1594,7 +1613,7 @@ impl VirtualMachine {
                     let line_buffering = buffered_stdio && (isatty || fd == 2);
 
                     let newline = if cfg!(windows) { None } else { Some("\n") };
-                    let encoding = self.state.config.settings.stdio_encoding.as_deref();
+                    let encoding = stdio_encoding.clone();
                     // stderr always uses backslashreplace (ignores stdio_errors)
                     let errors = if fd == 2 {
                         Some("backslashreplace")

scripts/update_lib/patch_spec.py

@@ -635,11 +635,21 @@ def select_remaining_patches(
             pending = list(existing)
             omitted = []
             for marker in remaining.get(cls, {}).get(method, []):
-                if marker not in pending:
+                original = marker
+                if (
+                    original not in pending
+                    and marker.ut_method == UtMethod.ExpectedFailureIf
+                    and marker.cond
+                ):
+                    # A condition only retires part of an unconditional failure.
+                    original = marker._replace(
+                        ut_method=UtMethod.ExpectedFailure, cond=None
+                    )
+                if original not in pending:
                     raise ValueError(
                         "Marker override cannot add, change, or reorder markers"
                     )
-                index = pending.index(marker)
+                index = pending.index(original)
                 omitted.extend(pending[:index])
                 pending = pending[index + 1 :]
             omitted.extend(pending)

scripts/update_lib/tests/test_migrate.py

@@ -601,6 +601,66 @@ class MarkerOverrideTests(unittest.TestCase):
             with self.subTest(markers=markers), self.assertRaises(ValueError):
                 select_remaining_patches(current, document, Path(document["target"]))
 
+    def test_cli_narrows_unconditional_failure(self):
+        def narrow(document):
+            document["patches"]["Example"]["test_known"] = [
+                {
+                    "ut_method": "expectedFailureIf",
+                    "cond": "MS_WINDOWS",
+                    "reason": "old failure",
+                }
+            ]
+
+        status, result = self.invoke(narrow)
+        self.assertEqual(status, 0)
+        self.assertIn(
+            'unittest.expectedFailureIf(MS_WINDOWS, "TODO: RUSTPYTHON; old failure")',
+            result,
+        )
+        self.assertIn('unittest.skip("TODO: RUSTPYTHON; retained guard")', result)
+        self.assertIn("self.assertEqual(1, 1)", result)
+
+    def test_narrowing_preserves_reason_order_and_existing_targets(self):
+        failure = PatchSpec(UtMethod.ExpectedFailure, reason="known failure")
+        narrowed = failure._replace(
+            ut_method=UtMethod.ExpectedFailureIf, cond="MS_WINDOWS"
+        )
+        success = PatchSpec(UtMethod.ExpectedSuccess, reason="known success")
+        current = {"Example": {"test_known": [failure, success]}}
+        for markers in (
+            [narrowed, success, narrowed],
+            [narrowed._replace(reason="changed"), success],
+            [success, narrowed],
+            [narrowed._replace(ut_method=UtMethod.SkipIf), success],
+            [narrowed._replace(ut_method=UtMethod.ExpectedSuccess), success],
+            [narrowed._replace(cond=None), success],
+        ):
+            document = {
+                "target": "Lib/test/test_example.py",
+                "patches": patches_to_json({"Example": {"test_known": markers}}),
+            }
+            with self.subTest(markers=markers), self.assertRaises(ValueError):
+                select_remaining_patches(current, document, Path(document["target"]))
+
+        document["patches"] = patches_to_json({"Example": {"test_new": [narrowed]}})
+        with self.assertRaises(ValueError):
+            select_remaining_patches(current, document, Path(document["target"]))
+
+    def test_conditional_failure_cannot_be_broadened_or_changed(self):
+        conditional = PatchSpec(UtMethod.ExpectedFailureIf, cond="MS_WINDOWS")
+        current = {"Example": {"test_known": [conditional]}}
+        for marker in (
+            conditional._replace(ut_method=UtMethod.ExpectedFailure, cond=None),
+            conditional._replace(cond="True"),
+            conditional._replace(cond="not MS_WINDOWS"),
+        ):
+            document = {
+                "target": "Lib/test/test_example.py",
+                "patches": patches_to_json({"Example": {"test_known": [marker]}}),
+            }
+            with self.subTest(marker=marker), self.assertRaises(ValueError):
+                select_remaining_patches(current, document, Path(document["target"]))
+
 
 class ConditionalMarkerOverrideTests(unittest.TestCase):
     decorator = (

src/settings.rs

@@ -295,24 +295,6 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> {
 
     settings.check_hash_pycs_mode = args.check_hash_based_pycs;
 
-    if let Some(val) = get_env("PYTHONUTF8")
-        && let Some(val_str) = val.to_str()
-        && !val_str.is_empty()
-    {
-        settings.utf8_mode = match val_str {
-            "1" => 1,
-            "0" => 0,
-            _ => {
-                error!(
-                    "Fatal Python error: config_init_utf8_mode: \
-                     PYTHONUTF8=N: N is missing or invalid\n\
-                     Python runtime state: preinitialized"
-                );
-                std::process::exit(1);
-            }
-        };
-    }
-
     if let Some(val) = get_env("PYTHON_LAZY_IMPORTS").filter(|val| !val.is_empty()) {
         settings.lazy_imports = parse_lazy_imports(val.to_str(), "PYTHON_LAZY_IMPORTS");
     }
@@ -330,6 +312,7 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> {
     let mut lazy_imports_option_seen = false;
     let mut import_time_option_seen = false;
     let mut frozen_modules_option_seen = false;
+    let mut utf8_option_seen = false;
     let xopts = args.implementation_option.into_iter().map(|s| {
         let (name, value) = match s.split_once('=') {
             Some((name, value)) => (name.to_owned(), Some(value)),
@@ -348,20 +331,21 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> {
             }
             "faulthandler" => settings.faulthandler = true,
             "warn_default_encoding" => settings.warn_default_encoding = true,
-            "utf8" => {
+            "utf8" if !utf8_option_seen => {
                 settings.utf8_mode = match value {
                     None => 1,
                     Some("1") => 1,
                     Some("0") => 0,
                     _ => {
                         error!(
-                            "Fatal Python error: config_init_utf8_mode: \
-                             -X utf8=n: n is missing or invalid\n\
+                            "Fatal Python error: preconfig_init_utf8_mode: \
+                             invalid -X utf8 option value\n\
                              Python runtime state: preinitialized"
                         );
                         std::process::exit(1);
                     }
                 };
+                utf8_option_seen = true;
             }
             "no_sig_int" => settings.install_signal_handlers = false,
             "no_debug_ranges" => settings.code_debug_ranges = false,
@@ -417,6 +401,22 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> {
     });
     settings.xoptions.extend(xopts);
 
+    if !utf8_option_seen && let Some(val) = get_env("PYTHONUTF8").filter(|value| !value.is_empty())
+    {
+        settings.utf8_mode = match val.to_str() {
+            Some("1") => 1,
+            Some("0") => 0,
+            _ => {
+                error!(
+                    "Fatal Python error: preconfig_init_utf8_mode: \
+                     invalid PYTHONUTF8 environment variable value\n\
+                     Python runtime state: preinitialized"
+                );
+                std::process::exit(1);
+            }
+        };
+    }
+
     // CPython validates the first frozen_modules option before initialization,
     // then getpath's config dictionary round-trip keeps the last occurrence.
     if let Some((_, value)) = settings