From 620faab734c558f937fee89d7895e3a445a0761a Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 30 May 2026 14:16:10 +0300 Subject: [PATCH] Update some tests to 3.14.5 (#7990) * Update `test_embed.py` to 3.14.5 * Update `test_cmd_line*` to 3.14.5 * Update unicode related tests * Update `test_script_helper.py` * Update `test_gc.py` --- Lib/test/_test_embed_structseq.py | 62 + Lib/test/test_cmd_line.py | 473 +++- Lib/test/test_cmd_line_script.py | 92 +- Lib/test/test_embed.py | 2032 +++++++++++++++++ Lib/test/test_gc.py | 177 +- .../data/circular_imports/subpkg2/__init__.py | 1 - .../test_import/data/package/submodule.py | 1 - .../test_import/data/package2/submodule2.py | 1 - Lib/test/test_import/data/unwritable/x.py | 1 - Lib/test/test_script_helper.py | 6 +- Lib/test/test_ucn.py | 39 +- Lib/test/test_unicode_file_functions.py | 4 +- Lib/test/test_unicode_identifiers.py | 5 +- 13 files changed, 2672 insertions(+), 222 deletions(-) create mode 100644 Lib/test/_test_embed_structseq.py create mode 100644 Lib/test/test_embed.py diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py new file mode 100644 index 000000000..4cac84d7a --- /dev/null +++ b/Lib/test/_test_embed_structseq.py @@ -0,0 +1,62 @@ +import sys +import types +import unittest + + +# bpo-46417: Test that structseq types used by the sys module are still +# valid when Py_Finalize()/Py_Initialize() are called multiple times. +class TestStructSeq(unittest.TestCase): + # test PyTypeObject members + def check_structseq(self, obj_type): + # ob_refcnt + self.assertGreaterEqual(sys.getrefcount(obj_type), 1) + # tp_base + self.assertIsSubclass(obj_type, tuple) + # tp_bases + self.assertEqual(obj_type.__bases__, (tuple,)) + # tp_dict + self.assertIsInstance(obj_type.__dict__, types.MappingProxyType) + # tp_mro + self.assertEqual(obj_type.__mro__, (obj_type, tuple, object)) + # tp_name + self.assertIsInstance(type.__name__, str) + # tp_subclasses + self.assertEqual(obj_type.__subclasses__(), []) + + def test_sys_attrs(self): + for attr_name in ( + 'flags', # FlagsType + 'float_info', # FloatInfoType + 'hash_info', # Hash_InfoType + 'int_info', # Int_InfoType + 'thread_info', # ThreadInfoType + 'version_info', # VersionInfoType + ): + with self.subTest(attr=attr_name): + attr = getattr(sys, attr_name) + self.check_structseq(type(attr)) + + def test_sys_funcs(self): + func_names = ['get_asyncgen_hooks'] # AsyncGenHooksType + if hasattr(sys, 'getwindowsversion'): + func_names.append('getwindowsversion') # WindowsVersionType + for func_name in func_names: + with self.subTest(func=func_name): + func = getattr(sys, func_name) + obj = func() + self.check_structseq(type(obj)) + + +try: + unittest.main( + module=( + '__main__' + if __name__ == '__main__' + # Avoiding a circular import: + else sys.modules['test._test_embed_structseq'] + ) + ) +except SystemExit as exc: + if exc.args[0] != 0: + raise +print("Tests passed") diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 8aa6dad27..8a0831ee4 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -3,17 +3,23 @@ # See test_cmd_line_script.py for testing of script execution import os +import re import subprocess import sys +import sysconfig import tempfile import textwrap import unittest from test import support from test.support import os_helper +from test.support import force_not_colorized +from test.support import threading_helper from test.support.script_helper import ( spawn_python, kill_python, assert_python_ok, assert_python_failure, interpreter_requires_environment ) +from textwrap import dedent + if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") @@ -33,11 +39,13 @@ class CmdLineTest(unittest.TestCase): def verify_valid_flag(self, cmd_line): rc, out, err = assert_python_ok(cmd_line) - self.assertTrue(out == b'' or out.endswith(b'\n')) + if out != b'': + self.assertEndsWith(out, b'\n') self.assertNotIn(b'Traceback', out) self.assertNotIn(b'Traceback', err) return out + @support.cpython_only def test_help(self): self.verify_valid_flag('-h') self.verify_valid_flag('-?') @@ -48,20 +56,28 @@ class CmdLineTest(unittest.TestCase): self.assertNotIn(b'-X dev', out) self.assertLess(len(lines), 50) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only def test_help_env(self): out = self.verify_valid_flag('--help-env') self.assertIn(b'PYTHONHOME', out) + # Env vars in each section should be sorted alphabetically + # (ignoring underscores so PYTHON_FOO and PYTHONFOO intermix naturally) + sort_key = lambda name: name.replace(b'_', b'').lower() + sections = out.split(b'These variables have equivalent') + for section in sections: + envvars = re.findall(rb'^(PYTHON\w+)', section, re.MULTILINE) + self.assertEqual(envvars, sorted(envvars, key=sort_key), + "env vars should be sorted alphabetically") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only def test_help_xoptions(self): out = self.verify_valid_flag('--help-xoptions') self.assertIn(b'-X dev', out) + options = re.findall(rb'^-X (\w+)', out, re.MULTILINE) + self.assertEqual(options, sorted(options), + "options should be sorted alphabetically") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only def test_help_all(self): out = self.verify_valid_flag('--help-all') lines = out.splitlines() @@ -80,12 +96,13 @@ class CmdLineTest(unittest.TestCase): def test_site_flag(self): self.verify_valid_flag('-S') + @support.cpython_only def test_version(self): version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii") for switch in '-V', '--version', '-VV': rc, out, err = assert_python_ok(switch) - self.assertFalse(err.startswith(version)) - self.assertTrue(out.startswith(version)) + self.assertNotStartsWith(err, version) + self.assertStartsWith(out, version) def test_verbose(self): # -v causes imports to write to stderr. If the write to @@ -145,8 +162,7 @@ class CmdLineTest(unittest.TestCase): else: self.assertEqual(err, b'') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only def test_xoption_frozen_modules(self): tests = { ('=on', 'FrozenImporter'), @@ -161,6 +177,18 @@ class CmdLineTest(unittest.TestCase): res = assert_python_ok(*cmd) self.assertRegex(res.out.decode('utf-8'), expected) + @support.cpython_only + def test_env_var_frozen_modules(self): + tests = { + ('on', 'FrozenImporter'), + ('off', 'SourceFileLoader'), + } + for raw, expected in tests: + cmd = ['-c', 'import os; print(os.__spec__.loader, end="")'] + with self.subTest(raw): + res = assert_python_ok(*cmd, PYTHON_FROZEN_MODULES=raw) + self.assertRegex(res.out.decode('utf-8'), expected) + def test_run_module(self): # Test expected operation of the '-m' switch # Switch needs an argument @@ -173,6 +201,7 @@ class CmdLineTest(unittest.TestCase): # All good if module is located and run successfully assert_python_ok('-m', 'timeit', '-n', '1') + @unittest.expectedFailureIf(support.is_android, "TODO: RUSTPYTHON") def test_run_module_bug1764407(self): # -m and -i need to play well together # Runs the timeit module and checks the __main__ @@ -184,6 +213,14 @@ class CmdLineTest(unittest.TestCase): self.assertTrue(data.find(b'1 loop') != -1) self.assertTrue(data.find(b'__main__.Timer') != -1) + @support.cpython_only + def test_null_byte_in_interactive_mode(self): + # gh-140594: Fix an out of bounds read when a single NUL character + # is read from the standard input in interactive mode. + proc = spawn_python('-i') + proc.communicate(b'\x00', timeout=support.SHORT_TIMEOUT) + self.assertEqual(proc.returncode, 0) + def test_relativedir_bug46421(self): # Test `python -m unittest` with a relative directory beginning with ./ # Note: We have to switch to the project's top module's directory, as per @@ -222,8 +259,7 @@ class CmdLineTest(unittest.TestCase): # command line, but how subprocess does decode bytes to unicode. Python # doesn't decode the command line because Windows provides directly the # arguments as unicode (using wmain() instead of main()). - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(sys.platform == 'win32', 'Windows has a native unicode API') def test_undecodable_code(self): @@ -259,8 +295,7 @@ class CmdLineTest(unittest.TestCase): if not stdout.startswith(pattern): raise AssertionError("%a doesn't start with %a" % (stdout, pattern)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(sys.platform == 'win32', 'Windows has a native unicode API') def test_invalid_utf8_arg(self): @@ -272,12 +307,10 @@ class CmdLineTest(unittest.TestCase): # Test with default config, in the C locale, in the Python UTF-8 Mode. code = 'import sys, os; s=os.fsencode(sys.argv[1]); print(ascii(s))' - # TODO: RUSTPYTHON def run_default(arg): cmd = [sys.executable, '-c', code, arg] return subprocess.run(cmd, stdout=subprocess.PIPE, text=True) - # TODO: RUSTPYTHON def run_c_locale(arg): cmd = [sys.executable, '-c', code, arg] env = dict(os.environ) @@ -285,7 +318,6 @@ class CmdLineTest(unittest.TestCase): return subprocess.run(cmd, stdout=subprocess.PIPE, text=True, env=env) - # TODO: RUSTPYTHON def run_utf8_mode(arg): cmd = [sys.executable, '-X', 'utf8', '-c', code, arg] return subprocess.run(cmd, stdout=subprocess.PIPE, text=True) @@ -330,6 +362,8 @@ class CmdLineTest(unittest.TestCase): self.assertEqual(stdout, expected) self.assertEqual(p.returncode, 0) + @unittest.skipIf(os.environ.get("PYTHONUNBUFFERED", "0") != "0", + "Python stdio buffering is disabled.") def test_non_interactive_output_buffering(self): code = textwrap.dedent(""" import sys @@ -369,7 +403,7 @@ class CmdLineTest(unittest.TestCase): p.stdin.flush() data, rc = _kill_python_and_exit_code(p) self.assertEqual(rc, 0) - self.assertTrue(data.startswith(b'x'), data) + self.assertStartsWith(data, b'x') def test_large_PYTHONPATH(self): path1 = "ABCDE" * 100 @@ -406,8 +440,7 @@ class CmdLineTest(unittest.TestCase): # for empty and unset PYTHONPATH self.assertEqual(out1, out2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_displayhook_unencodable(self): for encoding in ('ascii', 'latin-1', 'utf-8'): env = os.environ.copy() @@ -475,6 +508,7 @@ class CmdLineTest(unittest.TestCase): self.assertRegex(err.decode('ascii', 'ignore'), 'SyntaxError') self.assertEqual(b'', out) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_stdout_flush_at_shutdown(self): # Issue #5319: if stdout.flush() fails at shutdown, an error should # be printed out. @@ -486,8 +520,9 @@ class CmdLineTest(unittest.TestCase): rc, out, err = assert_python_failure('-c', code) self.assertEqual(b'', out) self.assertEqual(120, rc) - self.assertRegex(err.decode('ascii', 'ignore'), - 'Exception ignored in.*\nOSError: .*') + self.assertIn(b'Exception ignored while flushing sys.stdout:\n' + b'OSError: '.replace(b'\n', os.linesep.encode()), + err) def test_closed_stdout(self): # Issue #13444: if stdout has been explicitly closed, we should @@ -525,23 +560,19 @@ class CmdLineTest(unittest.TestCase): self.assertEqual(err, b'') self.assertEqual(p.returncode, 42) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_no_stdin(self): self._test_no_stdio(['stdin']) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_no_stdout(self): self._test_no_stdio(['stdout']) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_no_stderr(self): self._test_no_stdio(['stderr']) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_no_std_streams(self): self._test_no_stdio(['stdin', 'stdout', 'stderr']) @@ -593,8 +624,7 @@ class CmdLineTest(unittest.TestCase): print("del sys.modules['__main__']", file=script) assert_python_ok(filename) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only def test_unknown_options(self): rc, out, err = assert_python_failure('-E', '-z') self.assertIn(b'Unknown option: -z', err) @@ -641,6 +671,7 @@ class CmdLineTest(unittest.TestCase): cwd=tmpdir) self.assertEqual(out.strip(), b"ok") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_sys_flags_set(self): # Issue 31845: a startup refactoring broke reading flags from env vars for value, expected in (("", 0), ("1", 1), ("text", 1), ("2", 2)): @@ -650,22 +681,19 @@ class CmdLineTest(unittest.TestCase): PYTHONDONTWRITEBYTECODE=value, PYTHONVERBOSE=value, ) - dont_write_bytecode = int(bool(value)) + expected_bool = int(bool(value)) code = ( "import sys; " "sys.stderr.write(str(sys.flags)); " f"""sys.exit(not ( - sys.flags.debug == sys.flags.optimize == - sys.flags.verbose == - {expected} - and sys.flags.dont_write_bytecode == {dont_write_bytecode} + sys.flags.optimize == sys.flags.verbose == {expected} + and sys.flags.debug == sys.flags.dont_write_bytecode == {expected_bool} ))""" ) with self.subTest(envar_value=value): assert_python_ok('-c', code, **env_vars) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_set_pycache_prefix(self): # sys.pycache_prefix can be set from either -X pycache_prefix or # PYTHONPYCACHEPREFIX env var, with the former taking precedence. @@ -711,6 +739,7 @@ class CmdLineTest(unittest.TestCase): self.assertEqual(proc.returncode, 0, proc) return proc.stdout.rstrip() + @support.cpython_only def test_xdev(self): # sys.flags.dev_mode code = "import sys; print(sys.flags.dev_mode)" @@ -747,22 +776,24 @@ class CmdLineTest(unittest.TestCase): # Memory allocator debug hooks try: - import _testcapi + import _testinternalcapi # noqa: F401 except ImportError: pass else: - code = "import _testcapi; print(_testcapi.pymem_getallocatorsname())" + code = "import _testinternalcapi; print(_testinternalcapi.pymem_getallocatorsname())" with support.SuppressCrashReport(): out = self.run_xdev("-c", code, check_exitcode=False) if support.with_pymalloc(): alloc_name = "pymalloc_debug" + elif support.Py_GIL_DISABLED: + alloc_name = "mimalloc_debug" else: alloc_name = "malloc_debug" self.assertEqual(out, alloc_name) # Faulthandler try: - import faulthandler + import faulthandler # noqa: F401 except ImportError: pass else: @@ -791,8 +822,7 @@ class CmdLineTest(unittest.TestCase): self.assertEqual(proc.returncode, 0, proc) return proc.stdout.rstrip() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_warnings_filter_precedence(self): expected_filters = ("error::BytesWarning " "once::UserWarning " @@ -815,7 +845,7 @@ class CmdLineTest(unittest.TestCase): self.assertEqual(out, expected_filters) def check_pythonmalloc(self, env_var, name): - code = 'import _testcapi; print(_testcapi.pymem_getallocatorsname())' + code = 'import _testinternalcapi; print(_testinternalcapi.pymem_getallocatorsname())' env = dict(os.environ) env.pop('PYTHONDEVMODE', None) if env_var is not None: @@ -831,12 +861,16 @@ class CmdLineTest(unittest.TestCase): self.assertEqual(proc.stdout.rstrip(), name) self.assertEqual(proc.returncode, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only def test_pythonmalloc(self): # Test the PYTHONMALLOC environment variable + malloc = not support.Py_GIL_DISABLED pymalloc = support.with_pymalloc() - if pymalloc: + mimalloc = support.with_mimalloc() + if support.Py_GIL_DISABLED: + default_name = 'mimalloc_debug' if support.Py_DEBUG else 'mimalloc' + default_name_debug = 'mimalloc_debug' + elif pymalloc: default_name = 'pymalloc_debug' if support.Py_DEBUG else 'pymalloc' default_name_debug = 'pymalloc_debug' else: @@ -846,21 +880,28 @@ class CmdLineTest(unittest.TestCase): tests = [ (None, default_name), ('debug', default_name_debug), - ('malloc', 'malloc'), - ('malloc_debug', 'malloc_debug'), ] + if malloc: + tests.extend([ + ('malloc', 'malloc'), + ('malloc_debug', 'malloc_debug'), + ]) if pymalloc: tests.extend(( ('pymalloc', 'pymalloc'), ('pymalloc_debug', 'pymalloc_debug'), )) + if mimalloc: + tests.extend(( + ('mimalloc', 'mimalloc'), + ('mimalloc_debug', 'mimalloc_debug'), + )) for env_var, name in tests: with self.subTest(env_var=env_var, name=name): self.check_pythonmalloc(env_var, name) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pythondevmode_env(self): # Test the PYTHONDEVMODE environment variable code = "import sys; print(sys.flags.dev_mode)" @@ -879,6 +920,138 @@ class CmdLineTest(unittest.TestCase): self.assertEqual(proc.stdout.rstrip(), 'True') self.assertEqual(proc.returncode, 0, proc) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_python_gil(self): + cases = [ + # (env, opt, expected, msg) + ('1', None, '1', "PYTHON_GIL=1"), + (None, '1', '1', "-X gil=1"), + ] + + if support.Py_GIL_DISABLED: + cases.extend( + [ + (None, None, 'None', "no options set"), + ('0', None, '0', "PYTHON_GIL=0"), + ('1', '0', '0', "-X gil=0 overrides PYTHON_GIL=1"), + (None, '0', '0', "-X gil=0"), + ] + ) + else: + cases.extend( + [ + (None, None, '1', '-X gil=0 (unsupported by this build)'), + ('1', None, '1', 'PYTHON_GIL=0 (unsupported by this build)'), + ] + ) + code = "import sys; print(sys.flags.gil)" + environ = dict(os.environ) + + for env, opt, expected, msg in cases: + with self.subTest(msg, env=env, opt=opt): + environ.pop('PYTHON_GIL', None) + if env is not None: + environ['PYTHON_GIL'] = env + extra_args = [] + if opt is not None: + extra_args = ['-X', f'gil={opt}'] + + proc = subprocess.run([sys.executable, *extra_args, '-c', code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, env=environ) + self.assertEqual(proc.returncode, 0, proc) + self.assertEqual(proc.stdout.rstrip(), expected) + self.assertEqual(proc.stderr, '') + + def test_python_asyncio_debug(self): + code = "import asyncio; print(asyncio.new_event_loop().get_debug())" + rc, out, err = assert_python_ok('-c', code, PYTHONASYNCIODEBUG='1') + self.assertIn(b'True', out) + + @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") + def test_python_dump_refs(self): + code = 'import sys; sys._clear_internal_caches()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFS='1') + self.assertEqual(rc, 0) + + @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") + def test_python_dump_refs_file(self): + with tempfile.NamedTemporaryFile() as dump_file: + code = 'import sys; sys._clear_internal_caches()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFSFILE=dump_file.name) + self.assertEqual(rc, 0) + with open(dump_file.name, 'r') as file: + contents = file.read() + self.assertIn('Remaining objects', contents) + + @unittest.expectedFailureIf(sys.platform == "darwin", "TODO: RUSTPYTHON") + @unittest.skipUnless(sys.platform == 'darwin', 'PYTHONEXECUTABLE only works on macOS') + def test_python_executable(self): + code = 'import sys; print(sys.executable)' + expected = "/busr/bbin/bpython" + rc, out, err = assert_python_ok('-c', code, PYTHONEXECUTABLE=expected) + self.assertIn(expected.encode(), out) + + @unittest.expectedFailureIf(support.MS_WINDOWS, "TODO: RUSTPYTHON") + @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') + def test_python_legacy_windows_fs_encoding(self): + code = "import sys; print(sys.getfilesystemencoding())" + expected = 'mbcs' + rc, out, err = assert_python_ok('-c', code, PYTHONLEGACYWINDOWSFSENCODING='1') + self.assertIn(expected.encode(), out) + + @unittest.expectedFailureIf(support.MS_WINDOWS, "TODO: RUSTPYTHON") + @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') + def test_python_legacy_windows_stdio(self): + # Test that _WindowsConsoleIO is used when PYTHONLEGACYWINDOWSSTDIO + # is not set. + # We cannot use PIPE becase it prevents creating new console. + # So we use exit code. + code = "import sys; sys.exit(type(sys.stdout.buffer.raw).__name__ != '_WindowsConsoleIO')" + env = os.environ.copy() + env["PYTHONLEGACYWINDOWSSTDIO"] = "" + p = subprocess.run([sys.executable, "-c", code], + creationflags=subprocess.CREATE_NEW_CONSOLE, + env=env) + self.assertEqual(p.returncode, 0) + + # Then test that FIleIO is used when PYTHONLEGACYWINDOWSSTDIO is set. + code = "import sys; sys.exit(type(sys.stdout.buffer.raw).__name__ != 'FileIO')" + env["PYTHONLEGACYWINDOWSSTDIO"] = "1" + p = subprocess.run([sys.executable, "-c", code], + creationflags=subprocess.CREATE_NEW_CONSOLE, + env=env) + self.assertEqual(p.returncode, 0) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()), + "PYTHONMALLOCSTATS doesn't work with ASAN") + def test_python_malloc_stats(self): + code = "pass" + rc, out, err = assert_python_ok('-c', code, PYTHONMALLOCSTATS='1') + self.assertIn(b'Small block threshold', err) + + def test_python_user_base(self): + code = "import site; print(site.USER_BASE)" + expected = "/custom/userbase" + rc, out, err = assert_python_ok('-c', code, PYTHONUSERBASE=expected) + self.assertIn(expected.encode(), out) + + def test_python_basic_repl(self): + # Currently this only tests that the env var is set. See test_pyrepl.test_python_basic_repl. + code = "import os; print('PYTHON_BASIC_REPL' in os.environ)" + expected = "True" + rc, out, err = assert_python_ok('-c', code, PYTHON_BASIC_REPL='1') + self.assertIn(expected.encode(), out) + + @unittest.skipUnless(sysconfig.get_config_var('HAVE_PERF_TRAMPOLINE'), "Requires HAVE_PERF_TRAMPOLINE support") + def test_python_perf_jit_support(self): + code = "import sys; print(sys.is_stack_trampoline_active())" + expected = "True" + rc, out, err = assert_python_ok('-c', code, PYTHON_PERF_JIT_SUPPORT='1') + self.assertIn(expected.encode(), out) + @unittest.skipUnless(sys.platform == 'win32', 'bpo-32457 only applies on Windows') def test_argv0_normalization(self): @@ -891,16 +1064,15 @@ class CmdLineTest(unittest.TestCase): self.assertEqual(proc.returncode, 0, proc) self.assertEqual(proc.stdout.strip(), b'0') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only def test_parsing_error(self): args = [sys.executable, '-I', '--unknown-option'] proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - err_msg = "unknown option --unknown-option\nusage: " - self.assertTrue(proc.stderr.startswith(err_msg), proc.stderr) + err_msg = "Unknown option: --unknown-option\nusage: " + self.assertStartsWith(proc.stderr, err_msg) self.assertNotEqual(proc.returncode, 0) def test_int_max_str_digits(self): @@ -915,11 +1087,8 @@ class CmdLineTest(unittest.TestCase): assert_python_failure('-c', code, PYTHONINTMAXSTRDIGITS='foo') assert_python_failure('-c', code, PYTHONINTMAXSTRDIGITS='100') - def res2int(res): - out = res.out.strip().decode("utf-8") - return tuple(int(i) for i in out.split()) - res = assert_python_ok('-c', code) + res2int = self.res2int current_max = sys.get_int_max_str_digits() self.assertEqual(res2int(res), (current_max, current_max)) res = assert_python_ok('-X', 'int_max_str_digits=0', '-c', code) @@ -939,6 +1108,181 @@ class CmdLineTest(unittest.TestCase): ) self.assertEqual(res2int(res), (6000, 6000)) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_cmd_dedent(self): + # test that -c auto-dedents its arguments + test_cases = [ + ( + """ + print('space-auto-dedent') + """, + "space-auto-dedent", + ), + ( + dedent( + """ + ^^^print('tab-auto-dedent') + """ + ).replace("^", "\t"), + "tab-auto-dedent", + ), + ( + dedent( + """ + ^^if 1: + ^^^^print('mixed-auto-dedent-1') + ^^print('mixed-auto-dedent-2') + """ + ).replace("^", "\t \t"), + "mixed-auto-dedent-1\nmixed-auto-dedent-2", + ), + ( + ''' + data = """$ + + this data has an empty newline above and a newline with spaces below $ + $ + """$ + if 1: $ + print(repr(data))$ + '''.replace( + "$", "" + ), + # Note: entirely blank lines are normalized to \n, even if they + # are part of a data string. This is consistent with + # textwrap.dedent behavior, but might not be intuitive. + "'\\n\\nthis data has an empty newline above and a newline with spaces below \\n\\n'", + ), + ( + '', + '', + ), + ( + ' \t\n\t\n \t\t\t \t\t \t\n\t\t \n\n\n\t\t\t ', + '', + ), + ] + for code, expected in test_cases: + # Run the auto-dedent case + args1 = sys.executable, '-c', code + proc1 = subprocess.run(args1, stdout=subprocess.PIPE) + self.assertEqual(proc1.returncode, 0, proc1) + output1 = proc1.stdout.strip().decode(encoding='utf-8') + + # Manually dedent beforehand, check the result is the same. + args2 = sys.executable, '-c', dedent(code) + proc2 = subprocess.run(args2, stdout=subprocess.PIPE) + self.assertEqual(proc2.returncode, 0, proc2) + output2 = proc2.stdout.strip().decode(encoding='utf-8') + + self.assertEqual(output1, output2) + self.assertEqual(output1.replace('\r\n', '\n'), expected) + + def test_cmd_dedent_failcase(self): + # Mixing tabs and spaces is not allowed + from textwrap import dedent + template = dedent( + ''' + -+if 1: + +-++ print('will fail') + ''') + code = template.replace('-', ' ').replace('+', '\t') + assert_python_failure('-c', code) + code = template.replace('-', '\t').replace('+', ' ') + assert_python_failure('-c', code) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_cpu_count(self): + code = "import os; print(os.cpu_count(), os.process_cpu_count())" + res = assert_python_ok('-X', 'cpu_count=4321', '-c', code) + self.assertEqual(self.res2int(res), (4321, 4321)) + res = assert_python_ok('-c', code, PYTHON_CPU_COUNT='1234') + self.assertEqual(self.res2int(res), (1234, 1234)) + + def test_cpu_count_default(self): + code = "import os; print(os.cpu_count(), os.process_cpu_count())" + res = assert_python_ok('-X', 'cpu_count=default', '-c', code) + self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) + res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHON_CPU_COUNT='1234') + self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) + res = assert_python_ok('-c', code, PYTHON_CPU_COUNT='default') + self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_import_time(self): + # os is not imported at startup + code = 'import os; import os' + + for case in 'importtime', 'importtime=1', 'importtime=true': + res = assert_python_ok('-X', case, '-c', code) + res_err = res.err.decode('utf-8') + self.assertRegex(res_err, r'import time: \s*\d+ \| \s*\d+ \| \s*os') + self.assertNotRegex(res_err, r'import time: cached\s* \| cached\s* \| os') + + res = assert_python_ok('-X', 'importtime=2', '-c', code) + res_err = res.err.decode('utf-8') + self.assertRegex(res_err, r'import time: \s*\d+ \| \s*\d+ \| \s*os') + self.assertRegex(res_err, r'import time: cached\s* \| cached\s* \| os') + + assert_python_failure('-X', 'importtime=-1', '-c', code) + assert_python_failure('-X', 'importtime=3', '-c', code) + + def res2int(self, res): + out = res.out.strip().decode("utf-8") + return tuple(int(i) for i in out.split()) + + @unittest.skipUnless(support.Py_GIL_DISABLED, + "PYTHON_TLBC and -X tlbc" + " only supported in Py_GIL_DISABLED builds") + @threading_helper.requires_working_threading() + def test_disable_thread_local_bytecode(self): + code = """if 1: + import threading + def test(x, y): + return x + y + t = threading.Thread(target=test, args=(1,2)) + t.start() + t.join()""" + assert_python_ok("-W", "always", "-X", "tlbc=0", "-c", code) + assert_python_ok("-W", "always", "-c", code, PYTHON_TLBC="0") + + @unittest.skipUnless(support.Py_GIL_DISABLED, + "PYTHON_TLBC and -X tlbc" + " only supported in Py_GIL_DISABLED builds") + @threading_helper.requires_working_threading() + def test_enable_thread_local_bytecode(self): + code = """if 1: + import threading + def test(x, y): + return x + y + t = threading.Thread(target=test, args=(1,2)) + t.start() + t.join()""" + # The functionality of thread-local bytecode is tested more extensively + # in test_thread_local_bytecode + assert_python_ok("-W", "always", "-X", "tlbc=1", "-c", code) + assert_python_ok("-W", "always", "-c", code, PYTHON_TLBC="1") + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipUnless(support.Py_GIL_DISABLED, + "PYTHON_TLBC and -X tlbc" + " only supported in Py_GIL_DISABLED builds") + def test_invalid_thread_local_bytecode(self): + rc, out, err = assert_python_failure("-X", "tlbc") + self.assertIn(b"tlbc=n: n is missing or invalid", err) + rc, out, err = assert_python_failure("-X", "tlbc=foo") + self.assertIn(b"tlbc=n: n is missing or invalid", err) + rc, out, err = assert_python_failure("-X", "tlbc=-1") + self.assertIn(b"tlbc=n: n is missing or invalid", err) + rc, out, err = assert_python_failure("-X", "tlbc=2") + self.assertIn(b"tlbc=n: n is missing or invalid", err) + rc, out, err = assert_python_failure(PYTHON_TLBC="foo") + self.assertIn(b"PYTHON_TLBC=N: N is missing or invalid", err) + rc, out, err = assert_python_failure(PYTHON_TLBC="-1") + self.assertIn(b"PYTHON_TLBC=N: N is missing or invalid", err) + rc, out, err = assert_python_failure(PYTHON_TLBC="2") + self.assertIn(b"PYTHON_TLBC=N: N is missing or invalid", err) + @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') @@ -979,6 +1323,7 @@ class IgnoreEnvironmentTest(unittest.TestCase): class SyntaxErrorTests(unittest.TestCase): + @force_not_colorized def check_string(self, code): proc = subprocess.run([sys.executable, "-"], input=code, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -986,13 +1331,11 @@ class SyntaxErrorTests(unittest.TestCase): self.assertNotEqual(proc.stderr, None) self.assertIn(b"\nSyntaxError", proc.stderr) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailureIf(not support.is_android, "TODO: RUSTPYTHON") def test_tokenizer_error_with_stdin(self): self.check_string(b"(1+2+3") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailureIf(not support.is_android, "TODO: RUSTPYTHON") def test_decoding_error_at_the_end_of_the_line(self): self.check_string(br"'\u1f'") diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index f977b97bc..8b8c452f6 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -14,8 +14,7 @@ import io import textwrap from test import support -from test.support import import_helper -from test.support import os_helper +from test.support import import_helper, is_apple, os_helper from test.support.script_helper import ( make_pkg, make_script, make_zip_pkg, make_zip_script, assert_python_ok, assert_python_failure, spawn_python, kill_python) @@ -89,6 +88,8 @@ def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, importlib.invalidate_caches() return to_return + +@support.force_not_colorized_test_class class CmdLineTest(unittest.TestCase): def _check_output(self, script_name, exit_code, data, expected_file, expected_argv0, @@ -152,15 +153,13 @@ class CmdLineTest(unittest.TestCase): print('Expected output: %r' % expected_msg) self.assertIn(expected_msg.encode('utf-8'), err) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dash_c_loader(self): rc, out, err = assert_python_ok("-c", "print(__loader__)") expected = repr(importlib.machinery.BuiltinImporter).encode("utf-8") self.assertIn(expected, out) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_stdin_loader(self): # Unfortunately, there's no way to automatically test the fully # interactive REPL, since that code path only gets executed when @@ -207,21 +206,23 @@ class CmdLineTest(unittest.TestCase): stderr = p.stderr if separate_stderr else p.stdout self.assertIn(b'Traceback ', stderr.readline()) self.assertIn(b'File ""', stderr.readline()) + self.assertIn(b'1/0', stderr.readline()) + self.assertIn(b' ~^~', stderr.readline()) self.assertIn(b'ZeroDivisionError', stderr.readline()) - @unittest.skip("TODO: RUSTPYTHON, test hang in middle") + @unittest.skip("TODO: RUSTPYTHON; test hang in middle") def test_repl_stdout_flush(self): self.check_repl_stdout_flush() - @unittest.skip("TODO: RUSTPYTHON, test hang in middle") + @unittest.skip("TODO: RUSTPYTHON; test hang in middle") def test_repl_stdout_flush_separate_stderr(self): self.check_repl_stdout_flush(True) - @unittest.skip("TODO: RUSTPYTHON, test hang in middle") + @unittest.skip("TODO: RUSTPYTHON; test hang in middle") def test_repl_stderr_flush(self): self.check_repl_stderr_flush() - @unittest.skip("TODO: RUSTPYTHON, test hang in middle") + @unittest.skip("TODO: RUSTPYTHON; test hang in middle") def test_repl_stderr_flush_separate_stderr(self): self.check_repl_stderr_flush(True) @@ -233,8 +234,7 @@ class CmdLineTest(unittest.TestCase): importlib.machinery.SourceFileLoader, expected_cwd=script_dir) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_script_abspath(self): # pass the script using the relative path, expect the absolute path # in __file__ @@ -390,8 +390,7 @@ class CmdLineTest(unittest.TestCase): "be directly executed") self._check_import_error(["-m", "test_pkg"], msg, cwd=script_dir) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue8202(self): # Make sure package __init__ modules see "-m" in sys.argv0 while # searching for the module to execute @@ -554,7 +553,7 @@ class CmdLineTest(unittest.TestCase): script = textwrap.dedent("""\ try: raise ValueError - except: + except ValueError: raise NameError from None """) with os_helper.temp_dir() as script_dir: @@ -562,18 +561,23 @@ class CmdLineTest(unittest.TestCase): exitcode, stdout, stderr = assert_python_failure(script_name) text = stderr.decode('ascii').split('\n') self.assertEqual(len(text), 5) - self.assertTrue(text[0].startswith('Traceback')) - self.assertTrue(text[1].startswith(' File ')) - self.assertTrue(text[3].startswith('NameError')) + self.assertStartsWith(text[0], 'Traceback') + self.assertStartsWith(text[1], ' File ') + self.assertStartsWith(text[3], 'NameError') - @unittest.expectedFailureIf(sys.platform == "linux", "TODO: RUSTPYTHON") + @unittest.expectedFailureIf(sys.platform in ("android", "linux"), "TODO: RUSTPYTHON") def test_non_ascii(self): - # Mac OS X denies the creation of a file with an invalid UTF-8 name. + # Apple platforms deny the creation of a file with an invalid UTF-8 name. # Windows allows creating a name with an arbitrary bytes name, but # Python cannot a undecodable bytes argument to a subprocess. - # WASI does not permit invalid UTF-8 names. - if (os_helper.TESTFN_UNDECODABLE - and sys.platform not in ('win32', 'darwin', 'emscripten', 'wasi')): + # Emscripten/WASI does not permit invalid UTF-8 names. + if ( + os_helper.TESTFN_UNDECODABLE + and sys.platform not in { + "win32", "emscripten", "wasi" + } + and not is_apple + ): name = os.fsdecode(os_helper.TESTFN_UNDECODABLE) elif os_helper.TESTFN_NONASCII: name = os_helper.TESTFN_NONASCII @@ -641,8 +645,7 @@ class CmdLineTest(unittest.TestCase): self.assertNotIn("\f", text) self.assertIn("\n 1 + 1 = 2\n ^^^^^\n", text) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_syntaxerror_multi_line_fstring(self): script = 'foo = f"""{}\nfoo"""\n' with os_helper.temp_dir() as script_dir: @@ -657,8 +660,7 @@ class CmdLineTest(unittest.TestCase): ], ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_syntaxerror_invalid_escape_sequence_multi_line(self): script = 'foo = """\\q"""\n' with os_helper.temp_dir() as script_dir: @@ -669,13 +671,13 @@ class CmdLineTest(unittest.TestCase): self.assertEqual( stderr.splitlines()[-3:], [ b' foo = """\\q"""', - b' ^^^^^^^^', - b'SyntaxError: invalid escape sequence \'\\q\'' + b' ^^', + b'SyntaxError: "\\q" is an invalid escape sequence. ' + b'Did you mean "\\\\q"? A raw string is also an option.' ], ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_syntaxerror_null_bytes(self): script = "x = '\0' nothing to see here\n';import os;os.system('echo pwnd')\n" with os_helper.temp_dir() as script_dir: @@ -688,8 +690,7 @@ class CmdLineTest(unittest.TestCase): ], ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_syntaxerror_null_bytes_in_multiline_string(self): scripts = ["\n'''\nmultilinestring\0\n'''", "\nf'''\nmultilinestring\0\n'''"] # Both normal and f-strings with os_helper.temp_dir() as script_dir: @@ -703,6 +704,26 @@ class CmdLineTest(unittest.TestCase): ] ) + def test_source_lines_are_shown_when_running_source(self): + _, _, stderr = assert_python_failure("-c", "1/0") + expected_lines = [ + b'Traceback (most recent call last):', + b' File "", line 1, in ', + b' 1/0', + b' ~^~', + b'ZeroDivisionError: division by zero'] + self.assertEqual(stderr.splitlines(), expected_lines) + + def test_syntaxerror_does_not_crash(self): + script = "nonlocal x\n" + with os_helper.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, 'script', script) + exitcode, stdout, stderr = assert_python_failure(script_name) + text = io.TextIOWrapper(io.BytesIO(stderr), 'ascii').read() + # It used to crash in https://github.com/python/cpython/issues/111132 + self.assertEndsWith(text, + 'SyntaxError: nonlocal declaration not allowed at module level\n') + def test_consistent_sys_path_for_direct_execution(self): # This test case ensures that the following all give the same # sys.path configuration: @@ -771,8 +792,7 @@ class CmdLineTest(unittest.TestCase): traceback_lines = stderr.decode().splitlines() self.assertIn("No module named script_pkg", traceback_lines[-1]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_nonexisting_script(self): # bpo-34783: "./python script.py" must not crash # if the script file doesn't exist. @@ -788,11 +808,11 @@ class CmdLineTest(unittest.TestCase): self.assertIn(": can't open file ", err) self.assertNotEqual(proc.returncode, 0) + @unittest.skipIf(sys.platform.startswith("darwin"), "TODO: RUSTPYTHON; Problems with Mac os descriptor") @unittest.skipUnless(os.path.exists('/dev/fd/0'), 'requires /dev/fd platform') @unittest.skipIf(sys.platform.startswith("freebsd") and os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, "Requires fdescfs mounted on /dev/fd on FreeBSD") - @unittest.skipIf(sys.platform.startswith("darwin"), "TODO: RUSTPYTHON Problems with Mac os descriptor") def test_script_as_dev_fd(self): # GH-87235: On macOS passing a non-trivial script to /dev/fd/N can cause # problems because all open /dev/fd/N file descriptors share the same diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py new file mode 100644 index 000000000..a4a328aa3 --- /dev/null +++ b/Lib/test/test_embed.py @@ -0,0 +1,2032 @@ +# Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs) +from test import support +from test.support import import_helper, os_helper, threading_helper, MS_WINDOWS +import unittest + +from collections import namedtuple +import contextlib +import json +import os +import os.path +import re +import shutil +import subprocess +import sys +import sysconfig +import tempfile +import textwrap + +if not support.has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + + +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + + +MACOS = (sys.platform == 'darwin') +PYMEM_ALLOCATOR_NOT_SET = 0 +PYMEM_ALLOCATOR_DEBUG = 2 +PYMEM_ALLOCATOR_MALLOC = 3 +PYMEM_ALLOCATOR_MIMALLOC = 7 +if support.Py_GIL_DISABLED: + ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MIMALLOC +else: + ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MALLOC + +Py_STATS = hasattr(sys, '_stats_on') + +# _PyCoreConfig_InitCompatConfig() +API_COMPAT = 1 +# _PyCoreConfig_InitPythonConfig() +API_PYTHON = 2 +# _PyCoreConfig_InitIsolatedConfig() +API_ISOLATED = 3 + +INIT_LOOPS = 4 +MAX_HASH_SEED = 4294967295 + +ABI_THREAD = 't' if support.Py_GIL_DISABLED else '' +# PLATSTDLIB_LANDMARK copied from Modules/getpath.py +if os.name == 'nt': + PLATSTDLIB_LANDMARK = f'{sys.platlibdir}' +else: + VERSION_MAJOR = sys.version_info.major + VERSION_MINOR = sys.version_info.minor + PLATSTDLIB_LANDMARK = (f'{sys.platlibdir}/python{VERSION_MAJOR}.' + f'{VERSION_MINOR}{ABI_THREAD}/lib-dynload') + +DEFAULT_THREAD_INHERIT_CONTEXT = 1 if support.Py_GIL_DISABLED else 0 +DEFAULT_CONTEXT_AWARE_WARNINGS = 1 if support.Py_GIL_DISABLED else 0 + +# If we are running from a build dir, but the stdlib has been installed, +# some tests need to expect different results. +STDLIB_INSTALL = os.path.join(sys.prefix, sys.platlibdir, + f'python{sys.version_info.major}.{sys.version_info.minor}') +if not os.path.isfile(os.path.join(STDLIB_INSTALL, 'os.py')): + STDLIB_INSTALL = None + +def debug_build(program): + program = os.path.basename(program) + name = os.path.splitext(program)[0] + return name.casefold().endswith("_d".casefold()) + + +def remove_python_envvars(): + env = dict(os.environ) + # Remove PYTHON* environment variables to get deterministic environment + for key in list(env): + if key.startswith('PYTHON'): + del env[key] + return env + + +class EmbeddingTestsMixin: + def setUp(self): + exename = "_testembed" + builddir = os.path.dirname(sys.executable) + if MS_WINDOWS: + ext = ("_d" if debug_build(sys.executable) else "") + ".exe" + exename += ext + exepath = builddir + else: + exepath = os.path.join(builddir, 'Programs') + self.test_exe = exe = os.path.join(exepath, exename) + if not os.path.exists(exe): + self.skipTest("%r doesn't exist" % exe) + # This is needed otherwise we get a fatal error: + # "Py_Initialize: Unable to get the locale encoding + # LookupError: no codec search functions registered: can't find encoding" + self.oldcwd = os.getcwd() + os.chdir(builddir) + + def tearDown(self): + os.chdir(self.oldcwd) + + def run_embedded_interpreter(self, *args, env=None, + timeout=None, returncode=0, input=None, + cwd=None): + """Runs a test in the embedded interpreter""" + cmd = [self.test_exe] + cmd.extend(args) + if env is not None and MS_WINDOWS: + # Windows requires at least the SYSTEMROOT environment variable to + # start Python. + env = env.copy() + env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] + + p = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + env=env, + cwd=cwd) + try: + (out, err) = p.communicate(input=input, timeout=timeout) + except: + p.terminate() + p.wait() + raise + if p.returncode != returncode and support.verbose: + print(f"--- {cmd} failed ---") + print(f"stdout:\n{out}") + print(f"stderr:\n{err}") + print("------") + + self.assertEqual(p.returncode, returncode, + "bad returncode %d, stderr is %r" % + (p.returncode, err)) + return out, err + + def run_repeated_init_and_subinterpreters(self): + out, err = self.run_embedded_interpreter("test_repeated_init_and_subinterpreters") + self.assertEqual(err, "") + + # The output from _testembed looks like this: + # --- Pass 1 --- + # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 + # interp 1 <0x1d4f690>, thread state <0x1d35350>: id(modules) = 139650431165784 + # interp 2 <0x1d5a690>, thread state <0x1d99ed0>: id(modules) = 139650413140368 + # interp 3 <0x1d4f690>, thread state <0x1dc3340>: id(modules) = 139650412862200 + # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 + # --- Pass 2 --- + # ... + + interp_pat = (r"^interp (\d+) <(0x[\dA-F]+)>, " + r"thread state <(0x[\dA-F]+)>: " + r"id\(modules\) = ([\d]+)$") + Interp = namedtuple("Interp", "id interp tstate modules") + + numloops = 1 + current_run = [] + for line in out.splitlines(): + if line == "--- Pass {} ---".format(numloops): + self.assertEqual(len(current_run), 0) + if support.verbose > 1: + print(line) + numloops += 1 + continue + + self.assertLess(len(current_run), 5) + match = re.match(interp_pat, line) + if match is None: + self.assertRegex(line, interp_pat) + + # Parse the line from the loop. The first line is the main + # interpreter and the 3 afterward are subinterpreters. + interp = Interp(*match.groups()) + if support.verbose > 2: + # 5 lines per pass is super-spammy, so limit that to -vvv + print(interp) + self.assertTrue(interp.interp) + self.assertTrue(interp.tstate) + self.assertTrue(interp.modules) + current_run.append(interp) + + # The last line in the loop should be the same as the first. + if len(current_run) == 5: + main = current_run[0] + self.assertEqual(interp, main) + yield current_run + current_run = [] + + +class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase): + maxDiff = 100 * 50 + + def test_subinterps_main(self): + for run in self.run_repeated_init_and_subinterpreters(): + main = run[0] + + self.assertEqual(main.id, '0') + + def test_subinterps_different_ids(self): + for run in self.run_repeated_init_and_subinterpreters(): + main, *subs, _ = run + + mainid = int(main.id) + for i, sub in enumerate(subs): + self.assertEqual(sub.id, str(mainid + i + 1)) + + def test_subinterps_distinct_state(self): + for run in self.run_repeated_init_and_subinterpreters(): + main, *subs, _ = run + + if '0x0' in main: + # XXX Fix on Windows (and other platforms): something + # is going on with the pointers in Programs/_testembed.c. + # interp.interp is 0x0 and interp.modules is the same + # between interpreters. + raise unittest.SkipTest('platform prints pointers as 0x0') + + for sub in subs: + # A new subinterpreter may have the same + # PyInterpreterState pointer as a previous one if + # the earlier one has already been destroyed. So + # we compare with the main interpreter. The same + # applies to tstate. + self.assertNotEqual(sub.interp, main.interp) + self.assertNotEqual(sub.tstate, main.tstate) + self.assertNotEqual(sub.modules, main.modules) + + def test_repeated_init_and_inittab(self): + out, err = self.run_embedded_interpreter("test_repeated_init_and_inittab") + self.assertEqual(err, "") + + lines = [f"--- Pass {i} ---" for i in range(1, INIT_LOOPS+1)] + lines = "\n".join(lines) + "\n" + self.assertEqual(out, lines) + + def test_inittab_submodule_multiphase(self): + out, err = self.run_embedded_interpreter("test_inittab_submodule_multiphase") + self.assertEqual(err, "") + self.assertEqual(out, + "\n" + "\n" + "Hello from sub-module\n" + "mp_pkg.mp_submod.mp_submod_exec_slot_ran='yes'\n" + "mp_pkg.mp_pkg_exec_slot_ran='yes'\n" + ) + + def test_inittab_submodule_singlephase(self): + out, err = self.run_embedded_interpreter("test_inittab_submodule_singlephase") + self.assertEqual(self._nogil_filtered_err(err, "sp_pkg"), "") + self.assertEqual(out, + "\n" + "\n" + "Hello from sub-module\n" + ) + + def test_forced_io_encoding(self): + # Checks forced configuration of embedded interpreter IO streams + env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") + out, err = self.run_embedded_interpreter("test_forced_io_encoding", env=env) + if support.verbose > 1: + print() + print(out) + print(err) + expected_stream_encoding = "utf-8" + expected_errors = "surrogateescape" + expected_output = '\n'.join([ + "--- Use defaults ---", + "Expected encoding: default", + "Expected errors: default", + "stdin: {in_encoding}:{errors}", + "stdout: {out_encoding}:{errors}", + "stderr: {out_encoding}:backslashreplace", + "--- Set errors only ---", + "Expected encoding: default", + "Expected errors: ignore", + "stdin: {in_encoding}:ignore", + "stdout: {out_encoding}:ignore", + "stderr: {out_encoding}:backslashreplace", + "--- Set encoding only ---", + "Expected encoding: iso8859-1", + "Expected errors: default", + "stdin: iso8859-1:{errors}", + "stdout: iso8859-1:{errors}", + "stderr: iso8859-1:backslashreplace", + "--- Set encoding and errors ---", + "Expected encoding: iso8859-1", + "Expected errors: replace", + "stdin: iso8859-1:replace", + "stdout: iso8859-1:replace", + "stderr: iso8859-1:backslashreplace"]) + expected_output = expected_output.format( + in_encoding=expected_stream_encoding, + out_encoding=expected_stream_encoding, + errors=expected_errors) + # This is useful if we ever trip over odd platform behaviour + self.maxDiff = None + self.assertEqual(out.strip(), expected_output) + + def test_pre_initialization_api(self): + """ + Checks some key parts of the C-API that need to work before the runtime + is initialized (via Py_Initialize()). + """ + env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) + out, err = self.run_embedded_interpreter("test_pre_initialization_api", env=env) + if support.verbose > 1: + print() + print(out) + print(err) + if MS_WINDOWS: + expected_path = self.test_exe + else: + expected_path = os.path.join(os.getcwd(), "spam") + expected_output = f"sys.executable: {expected_path}\n" + self.assertIn(expected_output, out) + self.assertEqual(err, '') + + def test_pre_initialization_sys_options(self): + """ + Checks that sys.warnoptions and sys._xoptions can be set before the + runtime is initialized (otherwise they won't be effective). + """ + env = remove_python_envvars() + env['PYTHONPATH'] = os.pathsep.join(sys.path) + out, err = self.run_embedded_interpreter( + "test_pre_initialization_sys_options", env=env) + if support.verbose > 1: + print() + print(out) + print(err) + expected_output = ( + "sys.warnoptions: ['once', 'module', 'default']\n" + "sys._xoptions: {'not_an_option': '1', 'also_not_an_option': '2'}\n" + "warnings.filters[:3]: ['default', 'module', 'once']\n" + ) + self.assertIn(expected_output, out) + self.assertEqual(err, '') + + def test_bpo20891(self): + """ + bpo-20891: Calling PyGILState_Ensure in a non-Python thread must not + crash. + """ + out, err = self.run_embedded_interpreter("test_bpo20891") + self.assertEqual(out, '') + self.assertEqual(err, '') + + def test_initialize_twice(self): + """ + bpo-33932: Calling Py_Initialize() twice should do nothing (and not + crash!). + """ + out, err = self.run_embedded_interpreter("test_initialize_twice") + self.assertEqual(out, '') + self.assertEqual(err, '') + + def test_initialize_pymain(self): + """ + bpo-34008: Calling Py_Main() after Py_Initialize() must not fail. + """ + out, err = self.run_embedded_interpreter("test_initialize_pymain") + self.assertEqual(out.rstrip(), "Py_Main() after Py_Initialize: sys.argv=['-c', 'arg2']") + self.assertEqual(err, '') + + def test_run_main(self): + out, err = self.run_embedded_interpreter("test_run_main") + self.assertEqual(out.rstrip(), "Py_RunMain(): sys.argv=['-c', 'arg2']") + self.assertEqual(err, '') + + def test_run_main_loop(self): + # bpo-40413: Calling Py_InitializeFromConfig()+Py_RunMain() multiple + # times must not crash. + nloop = 5 + out, err = self.run_embedded_interpreter("test_run_main_loop") + self.assertEqual(out, "Py_RunMain(): sys.argv=['-c', 'arg2']\n" * nloop) + self.assertEqual(err, '') + + def test_finalize_structseq(self): + # bpo-46417: Py_Finalize() clears structseq static types. Check that + # sys attributes using struct types still work when + # Py_Finalize()/Py_Initialize() is called multiple times. + # print() calls type->tp_repr(instance) and so checks that the types + # are still working properly. + script = support.findfile('_test_embed_structseq.py') + with open(script, encoding="utf-8") as fp: + code = fp.read() + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS) + + def test_simple_initialization_api(self): + # _testembed now uses Py_InitializeFromConfig by default + # This case specifically checks Py_Initialize(Ex) still works + out, err = self.run_embedded_interpreter("test_repeated_simple_init") + self.assertEqual(out, 'Finalized\n' * INIT_LOOPS) + + @support.requires_specialization + @unittest.skipUnless(support.TEST_MODULES_ENABLED, "requires test modules") + def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self): + # https://github.com/python/cpython/issues/92031 + + _testinternalcapi = import_helper.import_module("_testinternalcapi") + + code = textwrap.dedent(f"""\ + import dis + import importlib._bootstrap + import opcode + import test.test_dis + import test.support + + def is_specialized(f): + for instruction in dis.get_instructions(f, adaptive=True): + opname = instruction.opname + if ( + opname in opcode._specialized_opmap + # Exclude superinstructions: + and "__" not in opname + # LOAD_CONST_IMMORTAL is "specialized", but is + # inserted during quickening. + and opname != "LOAD_CONST_IMMORTAL" + ): + return True + return False + + func = importlib._bootstrap._handle_fromlist + + # "copy" the code to un-specialize it: + test.support.reset_code(func) + + assert not is_specialized(func), "specialized instructions found" + + for _ in range({_testinternalcapi.SPECIALIZATION_THRESHOLD}): + func(importlib._bootstrap, ["x"], lambda *args: None) + + assert is_specialized(func), "no specialized instructions found" + + print("Tests passed") + """) + run = self.run_embedded_interpreter + out, err = run("test_repeated_init_exec", code) + self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS) + + def test_ucnhash_capi_reset(self): + # bpo-47182: unicodeobject.c:ucnhash_capi was not reset on shutdown. + code = "print('\\N{digit nine}')" + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, '9\n' * INIT_LOOPS) + + def test_datetime_reset_strptime(self): + code = ( + "import datetime;" + "d = datetime.datetime.strptime('2000-01-01', '%Y-%m-%d');" + "print(d.strftime('%Y%m%d'))" + ) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, '20000101\n' * INIT_LOOPS) + + def test_static_types_inherited_slots(self): + script = textwrap.dedent(""" + import test.support + results = [] + for cls in test.support.iter_builtin_types(): + for attr, _ in test.support.iter_slot_wrappers(cls): + wrapper = getattr(cls, attr) + res = (cls, attr, wrapper) + results.append(res) + results = ((repr(c), a, repr(w)) for c, a, w in results) + """) + def collate_results(raw): + results = {} + for cls, attr, wrapper in raw: + key = cls, attr + assert key not in results, (results, key, wrapper) + results[key] = wrapper + return results + + ns = {} + exec(script, ns, ns) + main_results = collate_results(ns['results']) + del ns + + script += textwrap.dedent(""" + import json + import sys + text = json.dumps(list(results)) + print(text, file=sys.stderr) + """) + out, err = self.run_embedded_interpreter( + "test_repeated_init_exec", script, script) + _results = err.split('--- Loop #')[1:] + (_embedded, _reinit, + ) = [json.loads(res.rpartition(' ---\n')[-1]) for res in _results] + embedded_results = collate_results(_embedded) + reinit_results = collate_results(_reinit) + + for key, expected in main_results.items(): + cls, attr = key + for src, results in [ + ('embedded', embedded_results), + ('reinit', reinit_results), + ]: + with self.subTest(src, cls=cls, slotattr=attr): + actual = results.pop(key) + self.assertEqual(actual, expected) + self.maxDiff = None + self.assertEqual(embedded_results, {}) + self.assertEqual(reinit_results, {}) + + self.assertEqual(out, '') + + def test_getargs_reset_static_parser(self): + # Test _PyArg_Parser initializations via _PyArg_UnpackKeywords() + # https://github.com/python/cpython/issues/122334 + code = textwrap.dedent(""" + try: + import _ssl + except ModuleNotFoundError: + _ssl = None + if _ssl is not None: + _ssl.txt2obj(txt='1.3') + print('1') + + import _queue + _queue.SimpleQueue().put_nowait(item=None) + print('2') + + import _zoneinfo + _zoneinfo.ZoneInfo.clear_cache(only_keys=['Foo/Bar']) + print('3') + """) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, '1\n2\n3\n' * INIT_LOOPS) + + @staticmethod + def _nogil_filtered_err(err: str, mod_name: str) -> str: + if not support.Py_GIL_DISABLED: + return err + + # the test imports a singlephase init extension, so it emits a warning + # under the free-threaded build + expected_runtime_warning = ( + "RuntimeWarning: The global interpreter lock (GIL)" + f" has been enabled to load module '{mod_name}'" + ) + filtered_err_lines = [ + line + for line in err.strip().splitlines() + if expected_runtime_warning not in line + ] + return "\n".join(filtered_err_lines) + + +def config_dev_mode(preconfig, config): + preconfig['allocator'] = PYMEM_ALLOCATOR_DEBUG + preconfig['dev_mode'] = 1 + config['dev_mode'] = 1 + config['warnoptions'] = ['default'] + config['faulthandler'] = 1 + + +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") +class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): + maxDiff = 4096 + UTF8_MODE_ERRORS = ('surrogatepass' if MS_WINDOWS else 'surrogateescape') + + # Marker to read the default configuration: get_default_config() + GET_DEFAULT_CONFIG = object() + + # Marker to ignore a configuration parameter + IGNORE_CONFIG = object() + + PRE_CONFIG_COMPAT = { + '_config_init': API_COMPAT, + 'allocator': PYMEM_ALLOCATOR_NOT_SET, + 'parse_argv': False, + 'configure_locale': True, + 'coerce_c_locale': False, + 'coerce_c_locale_warn': False, + 'utf8_mode': False, + } + if MS_WINDOWS: + PRE_CONFIG_COMPAT.update({ + 'legacy_windows_fs_encoding': False, + }) + PRE_CONFIG_PYTHON = dict(PRE_CONFIG_COMPAT, + _config_init=API_PYTHON, + parse_argv=True, + coerce_c_locale=GET_DEFAULT_CONFIG, + utf8_mode=GET_DEFAULT_CONFIG, + ) + PRE_CONFIG_ISOLATED = dict(PRE_CONFIG_COMPAT, + _config_init=API_ISOLATED, + configure_locale=False, + isolated=True, + use_environment=False, + utf8_mode=False, + dev_mode=False, + coerce_c_locale=False, + ) + + COPY_PRE_CONFIG = [ + 'dev_mode', + 'isolated', + 'use_environment', + ] + + CONFIG_COMPAT = { + '_config_init': API_COMPAT, + 'isolated': False, + 'use_environment': True, + 'dev_mode': False, + + 'install_signal_handlers': True, + 'use_hash_seed': False, + 'hash_seed': 0, + 'int_max_str_digits': sys.int_info.default_max_str_digits, + 'cpu_count': -1, + 'faulthandler': False, + 'tracemalloc': 0, + 'perf_profiling': 0, + 'import_time': 0, + 'thread_inherit_context': DEFAULT_THREAD_INHERIT_CONTEXT, + 'context_aware_warnings': DEFAULT_CONTEXT_AWARE_WARNINGS, + 'code_debug_ranges': True, + 'show_ref_count': False, + 'dump_refs': False, + 'dump_refs_file': None, + 'malloc_stats': False, + + 'filesystem_encoding': GET_DEFAULT_CONFIG, + 'filesystem_errors': GET_DEFAULT_CONFIG, + + 'pycache_prefix': None, + 'program_name': GET_DEFAULT_CONFIG, + 'parse_argv': False, + 'argv': [""], + 'orig_argv': [], + + 'xoptions': {}, + 'warnoptions': [], + + 'pythonpath_env': None, + 'home': None, + 'executable': GET_DEFAULT_CONFIG, + 'base_executable': GET_DEFAULT_CONFIG, + + 'prefix': GET_DEFAULT_CONFIG, + 'base_prefix': GET_DEFAULT_CONFIG, + 'exec_prefix': GET_DEFAULT_CONFIG, + 'base_exec_prefix': GET_DEFAULT_CONFIG, + 'module_search_paths': GET_DEFAULT_CONFIG, + 'module_search_paths_set': True, + 'platlibdir': sys.platlibdir, + 'stdlib_dir': GET_DEFAULT_CONFIG, + + 'site_import': True, + 'bytes_warning': 0, + 'warn_default_encoding': False, + 'inspect': False, + 'interactive': False, + 'optimization_level': 0, + 'parser_debug': False, + 'write_bytecode': True, + 'verbose': 0, + 'quiet': False, + 'remote_debug': True, + 'user_site_directory': True, + 'configure_c_stdio': False, + 'buffered_stdio': True, + + 'stdio_encoding': GET_DEFAULT_CONFIG, + 'stdio_errors': GET_DEFAULT_CONFIG, + + 'skip_source_first_line': False, + 'run_command': None, + 'run_module': None, + 'run_filename': None, + 'sys_path_0': None, + + '_install_importlib': True, + 'check_hash_pycs_mode': 'default', + 'pathconfig_warnings': True, + '_init_main': True, + 'use_frozen_modules': not support.Py_DEBUG, + 'safe_path': False, + '_is_python_build': IGNORE_CONFIG, + } + if Py_STATS: + CONFIG_COMPAT['_pystats'] = False + if support.Py_DEBUG: + CONFIG_COMPAT['run_presite'] = None + if support.Py_GIL_DISABLED: + CONFIG_COMPAT['enable_gil'] = -1 + CONFIG_COMPAT['tlbc_enabled'] = GET_DEFAULT_CONFIG + if MS_WINDOWS: + CONFIG_COMPAT.update({ + 'legacy_windows_stdio': False, + }) + if support.is_apple: + CONFIG_COMPAT['use_system_logger'] = False + + CONFIG_PYTHON = dict(CONFIG_COMPAT, + _config_init=API_PYTHON, + configure_c_stdio=True, + parse_argv=True, + ) + CONFIG_ISOLATED = dict(CONFIG_COMPAT, + _config_init=API_ISOLATED, + isolated=True, + use_environment=False, + user_site_directory=False, + safe_path=True, + dev_mode=False, + install_signal_handlers=False, + use_hash_seed=False, + faulthandler=False, + tracemalloc=False, + perf_profiling=0, + pathconfig_warnings=False, + ) + if MS_WINDOWS: + CONFIG_ISOLATED['legacy_windows_stdio'] = False + + # global config + DEFAULT_GLOBAL_CONFIG = { + 'Py_HasFileSystemDefaultEncoding': 0, + 'Py_HashRandomizationFlag': 1, + '_Py_HasFileSystemDefaultEncodeErrors': 0, + } + COPY_GLOBAL_PRE_CONFIG = [ + ('Py_UTF8Mode', 'utf8_mode'), + ] + COPY_GLOBAL_CONFIG = [ + # Copy core config to global config for expected values + # True means that the core config value is inverted (0 => 1 and 1 => 0) + ('Py_BytesWarningFlag', 'bytes_warning'), + ('Py_DebugFlag', 'parser_debug'), + ('Py_DontWriteBytecodeFlag', 'write_bytecode', True), + ('Py_FileSystemDefaultEncodeErrors', 'filesystem_errors'), + ('Py_FileSystemDefaultEncoding', 'filesystem_encoding'), + ('Py_FrozenFlag', 'pathconfig_warnings', True), + ('Py_IgnoreEnvironmentFlag', 'use_environment', True), + ('Py_InspectFlag', 'inspect'), + ('Py_InteractiveFlag', 'interactive'), + ('Py_IsolatedFlag', 'isolated'), + ('Py_NoSiteFlag', 'site_import', True), + ('Py_NoUserSiteDirectory', 'user_site_directory', True), + ('Py_OptimizeFlag', 'optimization_level'), + ('Py_QuietFlag', 'quiet'), + ('Py_UnbufferedStdioFlag', 'buffered_stdio', True), + ('Py_VerboseFlag', 'verbose'), + ] + if MS_WINDOWS: + COPY_GLOBAL_PRE_CONFIG.extend(( + ('Py_LegacyWindowsFSEncodingFlag', 'legacy_windows_fs_encoding'), + )) + COPY_GLOBAL_CONFIG.extend(( + ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'), + )) + + EXPECTED_CONFIG = None + + @classmethod + def tearDownClass(cls): + # clear cache + cls.EXPECTED_CONFIG = None + + def main_xoptions(self, xoptions_list): + xoptions = {} + for opt in xoptions_list: + if '=' in opt: + key, value = opt.split('=', 1) + xoptions[key] = value + else: + xoptions[opt] = True + return xoptions + + def _get_expected_config_impl(self): + env = remove_python_envvars() + code = textwrap.dedent(''' + import json + import sys + import _testinternalcapi + + configs = _testinternalcapi.get_configs() + + data = json.dumps(configs) + data = data.encode('utf-8') + sys.stdout.buffer.write(data) + sys.stdout.buffer.flush() + ''') + + # Use -S to not import the site module: get the proper configuration + # when test_embed is run from a venv (bpo-35313) + args = [sys.executable, '-S', '-c', code] + proc = subprocess.run(args, env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if proc.returncode: + raise Exception(f"failed to get the default config: " + f"stdout={proc.stdout!r} stderr={proc.stderr!r}") + stdout = proc.stdout.decode('utf-8') + # ignore stderr + try: + return json.loads(stdout) + except json.JSONDecodeError: + self.fail(f"fail to decode stdout: {stdout!r}") + + def _get_expected_config(self): + cls = InitConfigTests + if cls.EXPECTED_CONFIG is None: + cls.EXPECTED_CONFIG = self._get_expected_config_impl() + + # get a copy + configs = {} + for config_key, config_value in cls.EXPECTED_CONFIG.items(): + config = {} + for key, value in config_value.items(): + if isinstance(value, list): + value = value.copy() + config[key] = value + configs[config_key] = config + return configs + + def get_expected_config(self, expected_preconfig, expected, + env, api, modify_path_cb=None): + configs = self._get_expected_config() + + pre_config = configs['pre_config'] + for key, value in expected_preconfig.items(): + if value is self.GET_DEFAULT_CONFIG: + expected_preconfig[key] = pre_config[key] + + if not expected_preconfig['configure_locale'] or api == API_COMPAT: + # there is no easy way to get the locale encoding before + # setlocale(LC_CTYPE, "") is called: don't test encodings + for key in ('filesystem_encoding', 'filesystem_errors', + 'stdio_encoding', 'stdio_errors'): + expected[key] = self.IGNORE_CONFIG + + if not expected_preconfig['configure_locale']: + # UTF-8 Mode depends on the locale. There is no easy way + # to guess if UTF-8 Mode will be enabled or not if the locale + # is not configured. + expected_preconfig['utf8_mode'] = self.IGNORE_CONFIG + + if expected_preconfig['utf8_mode'] == 1: + if expected['filesystem_encoding'] is self.GET_DEFAULT_CONFIG: + expected['filesystem_encoding'] = 'utf-8' + if expected['filesystem_errors'] is self.GET_DEFAULT_CONFIG: + expected['filesystem_errors'] = self.UTF8_MODE_ERRORS + if expected['stdio_encoding'] is self.GET_DEFAULT_CONFIG: + expected['stdio_encoding'] = 'utf-8' + if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG: + expected['stdio_errors'] = 'surrogateescape' + + if MS_WINDOWS: + default_executable = self.test_exe + elif expected['program_name'] is not self.GET_DEFAULT_CONFIG: + default_executable = os.path.abspath(expected['program_name']) + else: + default_executable = os.path.join(os.getcwd(), '_testembed') + if expected['executable'] is self.GET_DEFAULT_CONFIG: + expected['executable'] = default_executable + if expected['base_executable'] is self.GET_DEFAULT_CONFIG: + expected['base_executable'] = default_executable + if expected['program_name'] is self.GET_DEFAULT_CONFIG: + expected['program_name'] = './_testembed' + + config = configs['config'] + for key, value in expected.items(): + if value is self.GET_DEFAULT_CONFIG: + expected[key] = config[key] + + if expected['module_search_paths'] is not self.IGNORE_CONFIG: + pythonpath_env = expected['pythonpath_env'] + if pythonpath_env is not None: + paths = pythonpath_env.split(os.path.pathsep) + expected['module_search_paths'] = [*paths, *expected['module_search_paths']] + if modify_path_cb is not None: + expected['module_search_paths'] = expected['module_search_paths'].copy() + modify_path_cb(expected['module_search_paths']) + + for key in self.COPY_PRE_CONFIG: + if key not in expected_preconfig: + expected_preconfig[key] = expected[key] + + def check_pre_config(self, configs, expected): + pre_config = dict(configs['pre_config']) + for key, value in list(expected.items()): + if value is self.IGNORE_CONFIG: + pre_config.pop(key, None) + del expected[key] + self.assertEqual(pre_config, expected) + + def check_config(self, configs, expected): + config = dict(configs['config']) + if MS_WINDOWS: + value = config.get(key := 'program_name') + if value and isinstance(value, str): + value = value[:len(value.lower().removesuffix('.exe'))] + if debug_build(sys.executable): + value = value[:len(value.lower().removesuffix('_d'))] + config[key] = value + for key, value in list(expected.items()): + if value is self.IGNORE_CONFIG: + config.pop(key, None) + del expected[key] + # Resolve bool/int mismatches to reduce noise in diffs + if isinstance(value, (bool, int)) and isinstance(config.get(key), (bool, int)): + expected[key] = type(config[key])(expected[key]) + self.assertEqual(config, expected) + + def check_global_config(self, configs): + pre_config = configs['pre_config'] + config = configs['config'] + + expected = dict(self.DEFAULT_GLOBAL_CONFIG) + for item in self.COPY_GLOBAL_CONFIG: + if len(item) == 3: + global_key, core_key, opposite = item + expected[global_key] = 0 if config[core_key] else 1 + else: + global_key, core_key = item + expected[global_key] = config[core_key] + for item in self.COPY_GLOBAL_PRE_CONFIG: + if len(item) == 3: + global_key, core_key, opposite = item + expected[global_key] = 0 if pre_config[core_key] else 1 + else: + global_key, core_key = item + expected[global_key] = pre_config[core_key] + + self.assertEqual(configs['global_config'], expected) + + def check_all_configs(self, testname, expected_config=None, + expected_preconfig=None, + modify_path_cb=None, + stderr=None, *, api, preconfig_api=None, + env=None, ignore_stderr=False, cwd=None): + new_env = remove_python_envvars() + if env is not None: + new_env.update(env) + env = new_env + + if preconfig_api is None: + preconfig_api = api + if preconfig_api == API_ISOLATED: + default_preconfig = self.PRE_CONFIG_ISOLATED + elif preconfig_api == API_PYTHON: + default_preconfig = self.PRE_CONFIG_PYTHON + else: + default_preconfig = self.PRE_CONFIG_COMPAT + if expected_preconfig is None: + expected_preconfig = {} + expected_preconfig = dict(default_preconfig, **expected_preconfig) + + if expected_config is None: + expected_config = {} + + if api == API_PYTHON: + default_config = self.CONFIG_PYTHON + elif api == API_ISOLATED: + default_config = self.CONFIG_ISOLATED + else: + default_config = self.CONFIG_COMPAT + expected_config = dict(default_config, **expected_config) + + self.get_expected_config(expected_preconfig, + expected_config, + env, + api, modify_path_cb) + + out, err = self.run_embedded_interpreter(testname, + env=env, cwd=cwd) + if stderr is None and not expected_config['verbose']: + stderr = "" + if stderr is not None and not ignore_stderr: + self.assertEqual(err.rstrip(), stderr) + try: + configs = json.loads(out) + except json.JSONDecodeError: + self.fail(f"fail to decode stdout: {out!r}") + + self.check_pre_config(configs, expected_preconfig) + self.check_config(configs, expected_config) + self.check_global_config(configs) + return configs + + @unittest.skipIf(support.check_bolt_optimized, "segfaults on BOLT instrumented binaries") + def test_init_default_config(self): + self.check_all_configs("test_init_initialize_config", api=API_COMPAT) + + def test_preinit_compat_config(self): + self.check_all_configs("test_preinit_compat_config", api=API_COMPAT) + + def test_init_compat_config(self): + self.check_all_configs("test_init_compat_config", api=API_COMPAT) + + def test_init_global_config(self): + preconfig = { + 'utf8_mode': True, + } + config = { + 'program_name': './globalvar', + 'site_import': False, + 'bytes_warning': True, + 'warnoptions': ['default::BytesWarning'], + 'inspect': True, + 'interactive': True, + 'optimization_level': 2, + 'write_bytecode': False, + 'verbose': True, + 'quiet': True, + 'buffered_stdio': False, + 'remote_debug': True, + 'user_site_directory': False, + 'pathconfig_warnings': False, + } + self.check_all_configs("test_init_global_config", config, preconfig, + api=API_COMPAT) + + def test_init_from_config(self): + preconfig = { + 'allocator': ALLOCATOR_FOR_CONFIG, + 'utf8_mode': True, + } + config = { + 'install_signal_handlers': False, + 'use_hash_seed': True, + 'hash_seed': 123, + 'tracemalloc': 2, + 'perf_profiling': 0, + 'import_time': 2, + 'code_debug_ranges': False, + 'show_ref_count': True, + 'malloc_stats': True, + + 'stdio_encoding': 'iso8859-1', + 'stdio_errors': 'replace', + + 'pycache_prefix': 'conf_pycache_prefix', + 'program_name': './conf_program_name', + 'argv': ['-c', 'arg2'], + 'orig_argv': ['python3', + '-W', 'cmdline_warnoption', + '-X', 'cmdline_xoption', + '-c', 'pass', + 'arg2'], + 'parse_argv': True, + 'xoptions': { + 'config_xoption1': '3', + 'config_xoption2': '', + 'config_xoption3': True, + 'cmdline_xoption': True, + }, + 'warnoptions': [ + 'cmdline_warnoption', + 'default::BytesWarning', + 'config_warnoption', + ], + 'run_command': 'pass\n', + + 'site_import': False, + 'bytes_warning': 1, + 'inspect': True, + 'interactive': True, + 'optimization_level': 2, + 'write_bytecode': False, + 'verbose': 1, + 'quiet': True, + 'remote_debug': True, + 'configure_c_stdio': True, + 'buffered_stdio': False, + 'user_site_directory': False, + 'faulthandler': True, + 'platlibdir': 'my_platlibdir', + 'module_search_paths': self.IGNORE_CONFIG, + 'safe_path': True, + 'int_max_str_digits': 31337, + 'cpu_count': 4321, + + 'check_hash_pycs_mode': 'always', + 'pathconfig_warnings': False, + } + if Py_STATS: + config['_pystats'] = 1 + self.check_all_configs("test_init_from_config", config, preconfig, + api=API_COMPAT) + + @unittest.skipIf(support.check_bolt_optimized, "segfaults on BOLT instrumented binaries") + def test_init_compat_env(self): + preconfig = { + 'allocator': ALLOCATOR_FOR_CONFIG, + } + config = { + 'use_hash_seed': True, + 'hash_seed': 42, + 'tracemalloc': 2, + 'import_time': 1, + 'code_debug_ranges': False, + 'malloc_stats': True, + 'inspect': True, + 'optimization_level': 2, + 'pythonpath_env': '/my/path', + 'pycache_prefix': 'env_pycache_prefix', + 'write_bytecode': False, + 'verbose': 1, + 'buffered_stdio': False, + 'stdio_encoding': 'iso8859-1', + 'stdio_errors': 'replace', + 'user_site_directory': False, + 'faulthandler': True, + 'warnoptions': ['EnvVar'], + 'platlibdir': 'env_platlibdir', + 'module_search_paths': self.IGNORE_CONFIG, + 'safe_path': True, + 'int_max_str_digits': 4567, + 'perf_profiling': 1, + } + if Py_STATS: + config['_pystats'] = 1 + self.check_all_configs("test_init_compat_env", config, preconfig, + api=API_COMPAT) + + @unittest.skipIf(support.check_bolt_optimized, "segfaults on BOLT instrumented binaries") + def test_init_python_env(self): + preconfig = { + 'allocator': ALLOCATOR_FOR_CONFIG, + 'utf8_mode': 1, + } + config = { + 'use_hash_seed': True, + 'hash_seed': 42, + 'tracemalloc': 2, + 'import_time': 1, + 'code_debug_ranges': False, + 'malloc_stats': True, + 'inspect': True, + 'optimization_level': 2, + 'pythonpath_env': '/my/path', + 'pycache_prefix': 'env_pycache_prefix', + 'write_bytecode': False, + 'verbose': 1, + 'buffered_stdio': False, + 'stdio_encoding': 'iso8859-1', + 'stdio_errors': 'replace', + 'user_site_directory': False, + 'faulthandler': True, + 'warnoptions': ['EnvVar'], + 'platlibdir': 'env_platlibdir', + 'module_search_paths': self.IGNORE_CONFIG, + 'safe_path': True, + 'int_max_str_digits': 4567, + 'perf_profiling': 1, + } + if Py_STATS: + config['_pystats'] = True + self.check_all_configs("test_init_python_env", config, preconfig, + api=API_PYTHON) + + def test_init_env_dev_mode(self): + preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG) + config = dict(dev_mode=1, + faulthandler=1, + warnoptions=['default']) + self.check_all_configs("test_init_env_dev_mode", config, preconfig, + api=API_COMPAT) + + def test_init_env_dev_mode_alloc(self): + preconfig = dict(allocator=ALLOCATOR_FOR_CONFIG) + config = dict(dev_mode=1, + faulthandler=1, + warnoptions=['default']) + self.check_all_configs("test_init_env_dev_mode_alloc", config, preconfig, + api=API_COMPAT) + + def test_init_dev_mode(self): + preconfig = {} + config = { + 'faulthandler': True, + 'dev_mode': True, + 'warnoptions': ['default'], + } + config_dev_mode(preconfig, config) + self.check_all_configs("test_init_dev_mode", config, preconfig, + api=API_PYTHON) + + def test_preinit_parse_argv(self): + # Pre-initialize implicitly using argv: make sure that -X dev + # is used to configure the allocation in preinitialization + preconfig = {} + config = { + 'argv': ['script.py'], + 'orig_argv': ['python3', '-X', 'dev', '-P', 'script.py'], + 'run_filename': os.path.abspath('script.py'), + 'dev_mode': True, + 'faulthandler': True, + 'warnoptions': ['default'], + 'xoptions': {'dev': True}, + 'safe_path': True, + } + config_dev_mode(preconfig, config) + self.check_all_configs("test_preinit_parse_argv", config, preconfig, + api=API_PYTHON) + + def test_preinit_dont_parse_argv(self): + # -X dev must be ignored by isolated preconfiguration + preconfig = { + 'isolated': False, + } + argv = ["python3", + "-E", "-I", "-P", + "-X", "dev", + "-X", "utf8", + "script.py"] + config = { + 'argv': argv, + 'orig_argv': argv, + 'isolated': False, + } + self.check_all_configs("test_preinit_dont_parse_argv", config, preconfig, + api=API_ISOLATED) + + def test_init_isolated_flag(self): + config = { + 'isolated': True, + 'safe_path': True, + 'use_environment': False, + 'user_site_directory': False, + } + self.check_all_configs("test_init_isolated_flag", config, api=API_PYTHON) + + def test_preinit_isolated1(self): + # _PyPreConfig.isolated=1, _PyCoreConfig.isolated not set + config = { + 'isolated': True, + 'safe_path': True, + 'use_environment': False, + 'user_site_directory': False, + } + self.check_all_configs("test_preinit_isolated1", config, api=API_COMPAT) + + def test_preinit_isolated2(self): + # _PyPreConfig.isolated=0, _PyCoreConfig.isolated=1 + config = { + 'isolated': True, + 'safe_path': True, + 'use_environment': False, + 'user_site_directory': False, + } + self.check_all_configs("test_preinit_isolated2", config, api=API_COMPAT) + + def test_preinit_isolated_config(self): + self.check_all_configs("test_preinit_isolated_config", api=API_ISOLATED) + + def test_init_isolated_config(self): + self.check_all_configs("test_init_isolated_config", api=API_ISOLATED) + + def test_preinit_python_config(self): + self.check_all_configs("test_preinit_python_config", api=API_PYTHON) + + def test_init_python_config(self): + self.check_all_configs("test_init_python_config", api=API_PYTHON) + + def test_init_dont_configure_locale(self): + # _PyPreConfig.configure_locale=0 + preconfig = { + 'configure_locale': 0, + 'coerce_c_locale': 0, + } + self.check_all_configs("test_init_dont_configure_locale", {}, preconfig, + api=API_PYTHON) + + @unittest.skip('as of 3.11 this test no longer works because ' + 'path calculations do not occur on read') + def test_init_read_set(self): + config = { + 'program_name': './init_read_set', + 'executable': 'my_executable', + 'base_executable': 'my_executable', + } + def modify_path(path): + path.insert(1, "test_path_insert1") + path.append("test_path_append") + self.check_all_configs("test_init_read_set", config, + api=API_PYTHON, + modify_path_cb=modify_path) + + def test_init_sys_add(self): + config = { + 'faulthandler': 1, + 'xoptions': { + 'config_xoption': True, + 'cmdline_xoption': True, + 'sysadd_xoption': True, + 'faulthandler': True, + }, + 'warnoptions': [ + 'ignore:::cmdline_warnoption', + 'ignore:::sysadd_warnoption', + 'ignore:::config_warnoption', + ], + 'orig_argv': ['python3', + '-W', 'ignore:::cmdline_warnoption', + '-X', 'cmdline_xoption'], + } + self.check_all_configs("test_init_sys_add", config, api=API_PYTHON) + + def test_init_run_main(self): + code = ('import _testinternalcapi, json; ' + 'print(json.dumps(_testinternalcapi.get_configs()))') + config = { + 'argv': ['-c', 'arg2'], + 'orig_argv': ['python3', '-c', code, 'arg2'], + 'program_name': './python3', + 'run_command': code + '\n', + 'parse_argv': True, + 'sys_path_0': '', + } + self.check_all_configs("test_init_run_main", config, api=API_PYTHON) + + def test_init_parse_argv(self): + config = { + 'parse_argv': True, + 'argv': ['-c', 'arg1', '-v', 'arg3'], + 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], + 'program_name': './argv0', + 'run_command': 'pass\n', + 'use_environment': False, + } + self.check_all_configs("test_init_parse_argv", config, api=API_PYTHON) + + def test_init_dont_parse_argv(self): + pre_config = { + 'parse_argv': 0, + } + config = { + 'parse_argv': False, + 'argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], + 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], + 'program_name': './argv0', + } + self.check_all_configs("test_init_dont_parse_argv", config, pre_config, + api=API_PYTHON) + + def default_program_name(self, config): + if MS_WINDOWS: + program_name = 'python' + executable = self.test_exe + else: + program_name = 'python3' + if MACOS: + executable = self.test_exe + else: + executable = shutil.which(program_name) or '' + config.update({ + 'program_name': program_name, + 'base_executable': executable, + 'executable': executable, + }) + + def test_init_setpath(self): + # Test Py_SetPath() + config = self._get_expected_config() + paths = config['config']['module_search_paths'] + + config = { + 'module_search_paths': paths, + 'prefix': '', + 'base_prefix': '', + 'exec_prefix': '', + 'base_exec_prefix': '', + # The current getpath.c doesn't determine the stdlib dir + # in this case. + 'stdlib_dir': '', + } + self.default_program_name(config) + env = {'TESTPATH': os.path.pathsep.join(paths)} + + self.check_all_configs("test_init_setpath", config, + api=API_COMPAT, env=env, + ignore_stderr=True) + + def test_init_setpath_config(self): + # Test Py_SetPath() with PyConfig + config = self._get_expected_config() + paths = config['config']['module_search_paths'] + + config = { + # set by Py_SetPath() + 'module_search_paths': paths, + 'prefix': '', + 'base_prefix': '', + 'exec_prefix': '', + 'base_exec_prefix': '', + # The current getpath.c doesn't determine the stdlib dir + # in this case. + 'stdlib_dir': '', + 'use_frozen_modules': not support.Py_DEBUG, + # overridden by PyConfig + 'program_name': 'conf_program_name', + 'base_executable': 'conf_executable', + 'executable': 'conf_executable', + } + env = {'TESTPATH': os.path.pathsep.join(paths)} + self.check_all_configs("test_init_setpath_config", config, + api=API_PYTHON, env=env, ignore_stderr=True) + + def module_search_paths(self, prefix=None, exec_prefix=None): + config = self._get_expected_config() + if prefix is None: + prefix = config['config']['prefix'] + if exec_prefix is None: + exec_prefix = config['config']['prefix'] + if MS_WINDOWS: + return config['config']['module_search_paths'] + else: + ver = sys.version_info + return [ + os.path.join(prefix, sys.platlibdir, + f'python{ver.major}{ver.minor}{ABI_THREAD}.zip'), + os.path.join(prefix, sys.platlibdir, + f'python{ver.major}.{ver.minor}{ABI_THREAD}'), + os.path.join(exec_prefix, sys.platlibdir, + f'python{ver.major}.{ver.minor}{ABI_THREAD}', 'lib-dynload'), + ] + + @contextlib.contextmanager + def tmpdir_with_python(self, subdir=None): + # Temporary directory with a copy of the Python program + with tempfile.TemporaryDirectory() as tmpdir: + # bpo-38234: On macOS and FreeBSD, the temporary directory + # can be symbolic link. For example, /tmp can be a symbolic link + # to /var/tmp. Call realpath() to resolve all symbolic links. + tmpdir = os.path.realpath(tmpdir) + if subdir: + tmpdir = os.path.normpath(os.path.join(tmpdir, subdir)) + os.makedirs(tmpdir) + + if MS_WINDOWS: + # Copy pythonXY.dll (or pythonXY_d.dll) + import fnmatch + exedir = os.path.dirname(self.test_exe) + for f in os.listdir(exedir): + if fnmatch.fnmatch(f, '*.dll'): + shutil.copyfile(os.path.join(exedir, f), os.path.join(tmpdir, f)) + + # Copy Python program + exec_copy = os.path.join(tmpdir, os.path.basename(self.test_exe)) + shutil.copyfile(self.test_exe, exec_copy) + shutil.copystat(self.test_exe, exec_copy) + self.test_exe = exec_copy + + yield tmpdir + + def test_init_setpythonhome(self): + # Test Py_SetPythonHome(home) with PYTHONPATH env var + config = self._get_expected_config() + paths = config['config']['module_search_paths'] + paths_str = os.path.pathsep.join(paths) + + for path in paths: + if not os.path.isdir(path): + continue + if os.path.exists(os.path.join(path, 'os.py')): + home = os.path.dirname(path) + break + else: + self.fail(f"Unable to find home in {paths!r}") + + prefix = exec_prefix = home + if MS_WINDOWS: + stdlib = os.path.join(home, "Lib") + # Because we are specifying 'home', module search paths + # are fairly static + expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] + else: + version = f'{sys.version_info.major}.{sys.version_info.minor}' + stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}') + expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) + + config = { + 'home': home, + 'module_search_paths': expected_paths, + 'prefix': prefix, + 'base_prefix': prefix, + 'exec_prefix': exec_prefix, + 'base_exec_prefix': exec_prefix, + 'pythonpath_env': paths_str, + 'stdlib_dir': stdlib, + } + self.default_program_name(config) + env = {'TESTHOME': home, 'PYTHONPATH': paths_str} + self.check_all_configs("test_init_setpythonhome", config, + api=API_COMPAT, env=env) + + def test_init_is_python_build_with_home(self): + # Test _Py_path_config._is_python_build configuration (gh-91985) + config = self._get_expected_config() + paths = config['config']['module_search_paths'] + paths_str = os.path.pathsep.join(paths) + + for path in paths: + if not os.path.isdir(path): + continue + if os.path.exists(os.path.join(path, 'os.py')): + home = os.path.dirname(path) + break + else: + self.fail(f"Unable to find home in {paths!r}") + + prefix = exec_prefix = home + if MS_WINDOWS: + stdlib = os.path.join(home, "Lib") + # Because we are specifying 'home', module search paths + # are fairly static + expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] + else: + version = f'{sys.version_info.major}.{sys.version_info.minor}' + stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}') + expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) + + config = { + 'home': home, + 'module_search_paths': expected_paths, + 'prefix': prefix, + 'base_prefix': prefix, + 'exec_prefix': exec_prefix, + 'base_exec_prefix': exec_prefix, + 'pythonpath_env': paths_str, + 'stdlib_dir': stdlib, + } + # The code above is taken from test_init_setpythonhome() + env = {'TESTHOME': home, 'PYTHONPATH': paths_str} + + env['NEGATIVE_ISPYTHONBUILD'] = '1' + config['_is_python_build'] = 0 + self.check_all_configs("test_init_is_python_build", config, + api=API_COMPAT, env=env) + + env['NEGATIVE_ISPYTHONBUILD'] = '0' + config['_is_python_build'] = 1 + exedir = os.path.dirname(sys.executable) + with open(os.path.join(exedir, 'pybuilddir.txt'), encoding='utf8') as f: + expected_paths[1 if MS_WINDOWS else 2] = os.path.normpath( + os.path.join(exedir, f'{f.read()}\n$'.splitlines()[0])) + if not MS_WINDOWS: + # PREFIX (default) is set when running in build directory + prefix = exec_prefix = sys.prefix + # stdlib calculation (/Lib) is not yet supported + expected_paths[0] = self.module_search_paths(prefix=prefix)[0] + config.update(prefix=prefix, base_prefix=prefix, + exec_prefix=exec_prefix, base_exec_prefix=exec_prefix) + self.check_all_configs("test_init_is_python_build", config, + api=API_COMPAT, env=env) + + def copy_paths_by_env(self, config): + all_configs = self._get_expected_config() + paths = all_configs['config']['module_search_paths'] + paths_str = os.path.pathsep.join(paths) + config['pythonpath_env'] = paths_str + env = {'PYTHONPATH': paths_str} + return env + + @unittest.skipIf(MS_WINDOWS, 'See test_init_pybuilddir_win32') + def test_init_pybuilddir(self): + # Test path configuration with pybuilddir.txt configuration file + + with self.tmpdir_with_python() as tmpdir: + # pybuilddir.txt is a sub-directory relative to the current + # directory (tmpdir) + vpath = sysconfig.get_config_var("VPATH") or '' + subdir = 'libdir' + libdir = os.path.join(tmpdir, subdir) + # The stdlib dir is dirname(executable) + VPATH + 'Lib' + stdlibdir = os.path.normpath(os.path.join(tmpdir, vpath, 'Lib')) + os.mkdir(libdir) + + filename = os.path.join(tmpdir, 'pybuilddir.txt') + with open(filename, "w", encoding="utf8") as fp: + fp.write(subdir) + + module_search_paths = self.module_search_paths() + module_search_paths[-2] = stdlibdir + module_search_paths[-1] = libdir + + executable = self.test_exe + config = { + 'base_exec_prefix': sysconfig.get_config_var("exec_prefix"), + 'base_prefix': sysconfig.get_config_var("prefix"), + 'base_executable': executable, + 'executable': executable, + 'module_search_paths': module_search_paths, + 'stdlib_dir': stdlibdir, + } + env = self.copy_paths_by_env(config) + self.check_all_configs("test_init_compat_config", config, + api=API_COMPAT, env=env, + ignore_stderr=True, cwd=tmpdir) + + @unittest.skipUnless(MS_WINDOWS, 'See test_init_pybuilddir') + def test_init_pybuilddir_win32(self): + # Test path configuration with pybuilddir.txt configuration file + + vpath = sysconfig.get_config_var("VPATH") + subdir = r'PCbuild\arch' + if os.path.normpath(vpath).count(os.sep) == 2: + subdir = os.path.join(subdir, 'instrumented') + + with self.tmpdir_with_python(subdir) as tmpdir: + # The prefix is dirname(executable) + VPATH + prefix = os.path.normpath(os.path.join(tmpdir, vpath)) + # The stdlib dir is dirname(executable) + VPATH + 'Lib' + stdlibdir = os.path.normpath(os.path.join(tmpdir, vpath, 'Lib')) + + filename = os.path.join(tmpdir, 'pybuilddir.txt') + with open(filename, "w", encoding="utf8") as fp: + fp.write(tmpdir) + + module_search_paths = self.module_search_paths() + module_search_paths[-3] = os.path.join(tmpdir, os.path.basename(module_search_paths[-3])) + module_search_paths[-2] = tmpdir + module_search_paths[-1] = stdlibdir + + executable = self.test_exe + config = { + 'base_exec_prefix': prefix, + 'base_prefix': prefix, + 'base_executable': executable, + 'executable': executable, + 'prefix': prefix, + 'exec_prefix': prefix, + 'module_search_paths': module_search_paths, + 'stdlib_dir': stdlibdir, + } + env = self.copy_paths_by_env(config) + self.check_all_configs("test_init_compat_config", config, + api=API_COMPAT, env=env, + ignore_stderr=False, cwd=tmpdir) + + def test_init_pyvenv_cfg(self): + # Test path configuration with pyvenv.cfg configuration file + + with self.tmpdir_with_python() as tmpdir, \ + tempfile.TemporaryDirectory() as pyvenv_home: + + ver = sys.version_info + base_prefix = sysconfig.get_config_var("prefix") + + # gh-128690: base_exec_prefix depends if PLATSTDLIB_LANDMARK exists + platstdlib = os.path.join(base_prefix, PLATSTDLIB_LANDMARK) + change_exec_prefix = not os.path.isdir(platstdlib) + + if not MS_WINDOWS: + lib_dynload = os.path.join(pyvenv_home, + sys.platlibdir, + f'python{ver.major}.{ver.minor}{ABI_THREAD}', + 'lib-dynload') + os.makedirs(lib_dynload) + else: + lib_folder = os.path.join(pyvenv_home, 'Lib') + os.makedirs(lib_folder) + # getpath.py uses Lib\os.py as the LANDMARK + shutil.copyfile( + os.path.join(support.STDLIB_DIR, 'os.py'), + os.path.join(lib_folder, 'os.py'), + ) + + filename = os.path.join(tmpdir, 'pyvenv.cfg') + with open(filename, "w", encoding="utf8") as fp: + print("home = %s" % pyvenv_home, file=fp) + print("include-system-site-packages = false", file=fp) + + paths = self.module_search_paths() + if not MS_WINDOWS: + if change_exec_prefix: + paths[-1] = lib_dynload + else: + paths = [ + os.path.join(tmpdir, os.path.basename(paths[0])), + pyvenv_home, + os.path.join(pyvenv_home, "Lib"), + ] + + executable = self.test_exe + base_executable = os.path.join(pyvenv_home, os.path.basename(executable)) + config = { + 'base_prefix': base_prefix, + 'exec_prefix': tmpdir, + 'prefix': tmpdir, + 'base_executable': base_executable, + 'executable': executable, + 'module_search_paths': paths, + } + if change_exec_prefix: + config['base_exec_prefix'] = pyvenv_home + if MS_WINDOWS: + config['base_prefix'] = pyvenv_home + config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib') + config['use_frozen_modules'] = bool(not support.Py_DEBUG) + else: + # cannot reliably assume stdlib_dir here because it + # depends too much on our build. But it ought to be found + config['stdlib_dir'] = self.IGNORE_CONFIG + config['use_frozen_modules'] = bool(not support.Py_DEBUG) + + env = self.copy_paths_by_env(config) + self.check_all_configs("test_init_compat_config", config, + api=API_COMPAT, env=env, + ignore_stderr=True, cwd=tmpdir) + + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') + def test_getpath_abspath_win32(self): + # Check _Py_abspath() is passed a backslashed path not to fall back to + # GetFullPathNameW() on startup, which (re-)normalizes the path overly. + # Currently, _Py_normpath() doesn't trim trailing dots and spaces. + CASES = [ + ("C:/a. . .", "C:\\a. . ."), + ("C:\\a. . .", "C:\\a. . ."), + ("\\\\?\\C:////a////b. . .", "\\\\?\\C:\\a\\b. . ."), + ("//a/b/c. . .", "\\\\a\\b\\c. . ."), + ("\\\\a\\b\\c. . .", "\\\\a\\b\\c. . ."), + ("a. . .", f"{os.getcwd()}\\a"), # relpath gets fully normalized + ] + out, err = self.run_embedded_interpreter( + "test_init_initialize_config", + env={**remove_python_envvars(), + "PYTHONPATH": os.path.pathsep.join(c[0] for c in CASES)} + ) + self.assertEqual(err, "") + try: + out = json.loads(out) + except json.JSONDecodeError: + self.fail(f"fail to decode stdout: {out!r}") + + results = out['config']["module_search_paths"] + for (_, expected), result in zip(CASES, results): + self.assertEqual(result, expected) + + def test_global_pathconfig(self): + # Test C API functions getting the path configuration: + # + # - Py_GetExecPrefix() + # - Py_GetPath() + # - Py_GetPrefix() + # - Py_GetProgramFullPath() + # - Py_GetProgramName() + # - Py_GetPythonHome() + # + # The global path configuration (_Py_path_config) must be a copy + # of the path configuration of PyInterpreter.config (PyConfig). + ctypes = import_helper.import_module('ctypes') + + def get_func(name): + func = getattr(ctypes.pythonapi, name) + func.argtypes = () + func.restype = ctypes.c_wchar_p + return func + + Py_GetPath = get_func('Py_GetPath') + Py_GetPrefix = get_func('Py_GetPrefix') + Py_GetExecPrefix = get_func('Py_GetExecPrefix') + Py_GetProgramName = get_func('Py_GetProgramName') + Py_GetProgramFullPath = get_func('Py_GetProgramFullPath') + Py_GetPythonHome = get_func('Py_GetPythonHome') + + config = _testinternalcapi.get_configs()['config'] + + self.assertEqual(tuple(Py_GetPath().split(os.path.pathsep)), + config['module_search_paths']) + self.assertEqual(Py_GetPrefix(), config['prefix']) + self.assertEqual(Py_GetExecPrefix(), config['exec_prefix']) + self.assertEqual(Py_GetProgramName(), config['program_name']) + self.assertEqual(Py_GetProgramFullPath(), config['executable']) + self.assertEqual(Py_GetPythonHome(), config['home']) + + def test_init_warnoptions(self): + # lowest to highest priority + warnoptions = [ + 'ignore:::PyConfig_Insert0', # PyWideStringList_Insert(0) + 'default', # PyConfig.dev_mode=1 + 'ignore:::env1', # PYTHONWARNINGS env var + 'ignore:::env2', # PYTHONWARNINGS env var + 'ignore:::cmdline1', # -W opt command line option + 'ignore:::cmdline2', # -W opt command line option + 'default::BytesWarning', # PyConfig.bytes_warnings=1 + 'ignore:::PySys_AddWarnOption1', # PySys_AddWarnOption() + 'ignore:::PySys_AddWarnOption2', # PySys_AddWarnOption() + 'ignore:::PyConfig_BeforeRead', # PyConfig.warnoptions + 'ignore:::PyConfig_AfterRead'] # PyWideStringList_Append() + preconfig = {} + config = { + 'bytes_warning': 1, + 'orig_argv': ['python3', + '-Wignore:::cmdline1', + '-Wignore:::cmdline2'], + } + config_dev_mode(preconfig, config) + config['warnoptions'] = warnoptions + self.check_all_configs("test_init_warnoptions", config, preconfig, + api=API_PYTHON) + + @unittest.skipIf(support.check_bolt_optimized, "segfaults on BOLT instrumented binaries") + def test_initconfig_api(self): + preconfig = { + 'configure_locale': True, + } + config = { + 'pycache_prefix': 'conf_pycache_prefix', + 'xoptions': {'faulthandler': True}, + 'hash_seed': 10, + 'use_hash_seed': True, + 'perf_profiling': 2, + } + config_dev_mode(preconfig, config) + # Temporarily enable ignore_stderr=True to ignore warnings on JIT builds + # See gh-126255 for more information + self.check_all_configs("test_initconfig_api", config, preconfig, + api=API_ISOLATED, ignore_stderr=True) + + def test_initconfig_get_api(self): + self.run_embedded_interpreter("test_initconfig_get_api") + + def test_initconfig_exit(self): + self.run_embedded_interpreter("test_initconfig_exit") + + def test_initconfig_module(self): + self.run_embedded_interpreter("test_initconfig_module") + + def test_get_argc_argv(self): + self.run_embedded_interpreter("test_get_argc_argv") + # ignore output + + def test_init_use_frozen_modules(self): + tests = { + ('=on', True), + ('=off', False), + ('=', True), + ('', True), + } + for raw, expected in tests: + optval = f'frozen_modules{raw}' + if raw.startswith('='): + xoption_value = raw[1:] + else: + xoption_value = True + config = { + 'parse_argv': True, + 'argv': ['-c'], + 'orig_argv': ['./argv0', '-X', optval, '-c', 'pass'], + 'program_name': './argv0', + 'run_command': 'pass\n', + 'use_environment': True, + 'xoptions': {'frozen_modules': xoption_value}, + 'use_frozen_modules': expected, + } + env = {'TESTFROZEN': raw[1:]} if raw else None + with self.subTest(repr(raw)): + self.check_all_configs("test_init_use_frozen_modules", config, + api=API_PYTHON, env=env) + + def test_init_main_interpreter_settings(self): + OBMALLOC = 1<<5 + EXTENSIONS = 1<<8 + THREADS = 1<<10 + DAEMON_THREADS = 1<<11 + FORK = 1<<15 + EXEC = 1<<16 + expected = { + # All optional features should be enabled. + 'feature_flags': + OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS, + 'own_gil': True, + } + out, err = self.run_embedded_interpreter( + 'test_init_main_interpreter_settings', + ) + self.assertEqual(err, '') + try: + out = json.loads(out) + except json.JSONDecodeError: + self.fail(f'fail to decode stdout: {out!r}') + + self.assertEqual(out, expected) + + @threading_helper.requires_working_threading() + def test_init_in_background_thread(self): + # gh-123022: Check that running Py_Initialize() in a background + # thread doesn't crash. + out, err = self.run_embedded_interpreter("test_init_in_background_thread") + self.assertEqual(err, "") + + +class AuditingTests(EmbeddingTestsMixin, unittest.TestCase): + def test_open_code_hook(self): + self.run_embedded_interpreter("test_open_code_hook") + + def test_audit(self): + self.run_embedded_interpreter("test_audit") + + def test_audit_tuple(self): + self.run_embedded_interpreter("test_audit_tuple") + + def test_audit_subinterpreter(self): + self.run_embedded_interpreter("test_audit_subinterpreter") + + def test_audit_run_command(self): + self.run_embedded_interpreter("test_audit_run_command", + timeout=support.SHORT_TIMEOUT, + returncode=1) + + def test_audit_run_file(self): + self.run_embedded_interpreter("test_audit_run_file", + timeout=support.SHORT_TIMEOUT, + returncode=1) + + def test_audit_run_interactivehook(self): + startup = os.path.join(self.oldcwd, os_helper.TESTFN) + ".py" + with open(startup, "w", encoding="utf-8") as f: + print("import sys", file=f) + print("sys.__interactivehook__ = lambda: None", file=f) + try: + env = {**remove_python_envvars(), "PYTHONSTARTUP": startup} + self.run_embedded_interpreter("test_audit_run_interactivehook", + timeout=support.SHORT_TIMEOUT, + returncode=10, env=env) + finally: + os.unlink(startup) + + def test_audit_run_startup(self): + startup = os.path.join(self.oldcwd, os_helper.TESTFN) + ".py" + with open(startup, "w", encoding="utf-8") as f: + print("pass", file=f) + try: + env = {**remove_python_envvars(), "PYTHONSTARTUP": startup} + self.run_embedded_interpreter("test_audit_run_startup", + timeout=support.SHORT_TIMEOUT, + returncode=10, env=env) + finally: + os.unlink(startup) + + def test_audit_run_stdin(self): + self.run_embedded_interpreter("test_audit_run_stdin", + timeout=support.SHORT_TIMEOUT, + returncode=1) + + def test_get_incomplete_frame(self): + self.run_embedded_interpreter("test_get_incomplete_frame") + + +class MiscTests(EmbeddingTestsMixin, unittest.TestCase): + def test_unicode_id_init(self): + # bpo-42882: Test that _PyUnicode_FromId() works + # when Python is initialized multiples times. + self.run_embedded_interpreter("test_unicode_id_init") + + # See bpo-44133 + @unittest.skipIf(os.name == 'nt', + 'Py_FrozenMain is not exported on Windows') + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_frozenmain(self): + env = dict(os.environ) + env['PYTHONUNBUFFERED'] = '1' + out, err = self.run_embedded_interpreter("test_frozenmain", env=env) + executable = os.path.realpath('./argv0') + expected = textwrap.dedent(f""" + Frozen Hello World + sys.argv ['./argv0', '-E', 'arg1', 'arg2'] + config program_name: ./argv0 + config executable: {executable} + config use_environment: True + config configure_c_stdio: True + config buffered_stdio: False + """).lstrip() + self.assertEqual(out, expected) + + @unittest.skipUnless(support.Py_DEBUG, + '-X showrefcount requires a Python debug build') + def test_no_memleak(self): + # bpo-1635741: Python must release all memory at exit + tests = ( + ('off', 'pass'), + ('on', 'pass'), + ('off', 'import __hello__'), + ('on', 'import __hello__'), + ) + for flag, stmt in tests: + xopt = f"frozen_modules={flag}" + cmd = [sys.executable, "-I", "-X", "showrefcount", "-X", xopt, "-c", stmt] + proc = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True) + self.assertEqual(proc.returncode, 0) + out = proc.stdout.rstrip() + match = re.match(r'^\[(-?\d+) refs, (-?\d+) blocks\]', out) + if not match: + self.fail(f"unexpected output: {out!a}") + refs = int(match.group(1)) + blocks = int(match.group(2)) + with self.subTest(frozen_modules=flag, stmt=stmt): + self.assertEqual(refs, 0, out) + self.assertEqual(blocks, 0, out) + + @unittest.skipUnless(support.Py_DEBUG, + '-X presite requires a Python debug build') + def test_presite(self): + cmd = [ + sys.executable, + "-I", "-X", "presite=test._test_embed_structseq", + "-c", "print('unique-python-message')", + ] + proc = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + self.assertEqual(proc.returncode, 0) + out = proc.stdout.strip() + self.assertIn("Tests passed", out) + self.assertIn("unique-python-message", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 879a2875a..855e1c79c 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -7,7 +7,7 @@ from test.support import (verbose, refcount_test, Py_GIL_DISABLED) from test.support.import_helper import import_module from test.support.os_helper import temp_dir, TESTFN, unlink -from test.support.script_helper import assert_python_ok, make_script, run_test_script +from test.support.script_helper import assert_python_ok, make_script from test.support import threading_helper, gc_threshold import gc @@ -236,8 +236,7 @@ class GCTests(unittest.TestCase): # is 3 because it includes f's code object. self.assertIn(gc.collect(), (2, 3)) - # TODO: RUSTPYTHON - weakref clear ordering differs from 3.15+ - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - weakref clear ordering differs from 3.15+ def test_function_tp_clear_leaves_consistent_state(self): # https://github.com/python/cpython/issues/91636 code = """if 1: @@ -311,28 +310,6 @@ class GCTests(unittest.TestCase): self.assertRegex(stdout, rb"""\A\s*func=None""") self.assertFalse(stderr) - # TODO: RUSTPYTHON - _datetime module not available - @unittest.expectedFailure - def test_datetime_weakref_cycle(self): - # https://github.com/python/cpython/issues/132413 - # If the weakref used by the datetime extension gets cleared by the GC (due to being - # in an unreachable cycle) then datetime functions would crash (get_module_state() - # was returning a NULL pointer). This bug is fixed by clearing weakrefs without - # callbacks *after* running finalizers. - code = """if 1: - import _datetime - class C: - def __del__(self): - print('__del__ called') - _datetime.timedelta(days=1) # crash? - - l = [C()] - l.append(l) - """ - rc, stdout, stderr = assert_python_ok("-c", code) - self.assertEqual(rc, 0) - self.assertEqual(stdout.strip(), b'__del__ called') - @refcount_test def test_frame(self): def f(): @@ -423,11 +400,19 @@ class GCTests(unittest.TestCase): # each call to collect(N) x = [] gc.collect(0) - # x is now in the old gen + # x is now in gen 1 a, b, c = gc.get_count() - # We don't check a since its exact values depends on + gc.collect(1) + # x is now in gen 2 + d, e, f = gc.get_count() + gc.collect(2) + # x is now in gen 3 + g, h, i = gc.get_count() + # We don't check a, d, g since their exact values depends on # internal implementation details of the interpreter. self.assertEqual((b, c), (1, 0)) + self.assertEqual((e, f), (0, 1)) + self.assertEqual((h, i), (0, 0)) def test_trashcan(self): class Ouch: @@ -682,8 +667,9 @@ class GCTests(unittest.TestCase): gc.collect() self.assertEqual(len(ouch), 2) # else the callbacks didn't run for x in ouch: - # The weakref should be cleared before executing the callback. - self.assertIsNone(x) + # If the callback resurrected one of these guys, the instance + # would be damaged, with an empty __dict__. + self.assertEqual(x, None) def test_bug21435(self): # This is a poor test - its only virtue is that it happened to @@ -845,20 +831,17 @@ class GCTests(unittest.TestCase): rc, out, err = assert_python_ok(TESTFN) self.assertEqual(out.strip(), b'__del__ called') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_get_stats(self): stats = gc.get_stats() self.assertEqual(len(stats), 3) for st in stats: self.assertIsInstance(st, dict) - self.assertEqual( - set(st), - {"collected", "collections", "uncollectable", "candidates", "duration"} - ) + self.assertEqual(set(st), + {"collected", "collections", "uncollectable"}) self.assertGreaterEqual(st["collected"], 0) self.assertGreaterEqual(st["collections"], 0) self.assertGreaterEqual(st["uncollectable"], 0) - self.assertGreaterEqual(st["candidates"], 0) - self.assertGreaterEqual(st["duration"], 0) # Check that collection counts are incremented correctly if gc.isenabled(): self.addCleanup(gc.enable) @@ -869,25 +852,11 @@ class GCTests(unittest.TestCase): self.assertEqual(new[0]["collections"], old[0]["collections"] + 1) self.assertEqual(new[1]["collections"], old[1]["collections"]) self.assertEqual(new[2]["collections"], old[2]["collections"]) - self.assertGreater(new[0]["duration"], old[0]["duration"]) - self.assertEqual(new[1]["duration"], old[1]["duration"]) - self.assertEqual(new[2]["duration"], old[2]["duration"]) - for stat in ["collected", "uncollectable", "candidates"]: - self.assertGreaterEqual(new[0][stat], old[0][stat]) - self.assertEqual(new[1][stat], old[1][stat]) - self.assertEqual(new[2][stat], old[2][stat]) gc.collect(2) - old, new = new, gc.get_stats() - self.assertEqual(new[0]["collections"], old[0]["collections"]) + new = gc.get_stats() + self.assertEqual(new[0]["collections"], old[0]["collections"] + 1) self.assertEqual(new[1]["collections"], old[1]["collections"]) self.assertEqual(new[2]["collections"], old[2]["collections"] + 1) - self.assertEqual(new[0]["duration"], old[0]["duration"]) - self.assertEqual(new[1]["duration"], old[1]["duration"]) - self.assertGreater(new[2]["duration"], old[2]["duration"]) - for stat in ["collected", "uncollectable", "candidates"]: - self.assertEqual(new[0][stat], old[0][stat]) - self.assertEqual(new[1][stat], old[1][stat]) - self.assertGreaterEqual(new[2][stat], old[2][stat]) def test_freeze(self): gc.freeze() @@ -911,10 +880,42 @@ class GCTests(unittest.TestCase): self.assertTrue( any(l is element for element in gc.get_objects(generation=0)) ) - gc.collect() + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=2)) + ) + gc.collect(generation=0) self.assertFalse( any(l is element for element in gc.get_objects(generation=0)) ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=2)) + ) + gc.collect(generation=1) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=2)) + ) + gc.collect(generation=2) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=2)) + ) del l gc.collect() @@ -1203,37 +1204,6 @@ class GCTests(unittest.TestCase): """) assert_python_ok("-c", source) - def test_do_not_cleanup_type_subclasses_before_finalization(self): - # See https://github.com/python/cpython/issues/135552 - # If we cleanup weakrefs for tp_subclasses before calling - # the finalizer (__del__) then the line `fail = BaseNode.next.next` - # should fail because we are trying to access a subclass - # attribute. But subclass type cache was not properly invalidated. - code = """ - class BaseNode: - def __del__(self): - BaseNode.next = BaseNode.next.next - fail = BaseNode.next.next - - class Node(BaseNode): - pass - - BaseNode.next = Node() - BaseNode.next.next = Node() - """ - # this test checks garbage collection while interp - # finalization - assert_python_ok("-c", textwrap.dedent(code)) - - code_inside_function = textwrap.dedent(F""" - def test(): - {textwrap.indent(code, ' ')} - - test() - """) - # this test checks regular garbage collection - assert_python_ok("-c", code_inside_function) - @unittest.skipUnless(Py_GIL_DISABLED, "requires free-threaded GC") @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") @@ -1252,16 +1222,14 @@ class GCTests(unittest.TestCase): # Use n // 2 just in case some other objects were collected. self.assertTrue(new_count - count > (n // 2)) - -class IncrementalGCTests(unittest.TestCase): + @requires_gil_enabled('need generational GC') @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") - @requires_gil_enabled("Free threading does not support incremental GC") - def test_incremental_gc_handles_fast_cycle_creation(self): - # Run this test in a fresh process. The number of alive objects (which can - # be from unit tests run before this one) can influence how quickly cyclic - # garbage is found. - script = support.findfile("_test_gc_fast_cycles.py") - run_test_script(script) + def test_heap_size(self): + count = _testinternalcapi.get_tracked_heap_size() + l = [] + self.assertEqual(count + 1, _testinternalcapi.get_tracked_heap_size()) + del l + self.assertEqual(count, _testinternalcapi.get_tracked_heap_size()) class GCCallbackTests(unittest.TestCase): @@ -1338,11 +1306,9 @@ class GCCallbackTests(unittest.TestCase): # Check that we got the right info dict for all callbacks for v in self.visit: info = v[2] - self.assertIn("generation", info) - self.assertIn("collected", info) - self.assertIn("uncollectable", info) - self.assertIn("candidates", info) - self.assertIn("duration", info) + self.assertTrue("generation" in info) + self.assertTrue("collected" in info) + self.assertTrue("uncollectable" in info) def test_collect_generation(self): self.preclean() @@ -1518,8 +1484,8 @@ class GCTogglingTests(unittest.TestCase): assert not detector.gc_happened while not detector.gc_happened: i += 1 - if i > 100000: - self.fail("gc didn't happen after 100000 iterations") + if i > 10000: + self.fail("gc didn't happen after 10000 iterations") self.assertEqual(len(ouch), 0) junk.append([]) # this will eventually trigger gc @@ -1530,7 +1496,6 @@ class GCTogglingTests(unittest.TestCase): self.assertEqual(x, None) @gc_threshold(1000, 0, 0) - @unittest.skipIf(Py_GIL_DISABLED, "requires GC generations or increments") def test_bug1055820d(self): # Corresponds to temp2d.py in the bug report. This is very much like # test_bug1055820c, but uses a __del__ method instead of a weakref @@ -1591,8 +1556,8 @@ class GCTogglingTests(unittest.TestCase): gc.collect() while not detector.gc_happened: i += 1 - if i > 50000: - self.fail("gc didn't happen after 50000 iterations") + if i > 10000: + self.fail("gc didn't happen after 10000 iterations") self.assertEqual(len(ouch), 0) junk.append([]) # this will eventually trigger gc @@ -1609,8 +1574,8 @@ class GCTogglingTests(unittest.TestCase): detector = GC_Detector() while not detector.gc_happened: i += 1 - if i > 100000: - self.fail("gc didn't happen after 100000 iterations") + if i > 10000: + self.fail("gc didn't happen after 10000 iterations") junk.append([]) # this will eventually trigger gc try: @@ -1620,11 +1585,11 @@ class GCTogglingTests(unittest.TestCase): detector = GC_Detector() while not detector.gc_happened: i += 1 - if i > 100000: + if i > 10000: break junk.append([]) # this may eventually trigger gc (if it is enabled) - self.assertEqual(i, 100001) + self.assertEqual(i, 10001) finally: gc.enable() diff --git a/Lib/test/test_import/data/circular_imports/subpkg2/__init__.py b/Lib/test/test_import/data/circular_imports/subpkg2/__init__.py index 8b1378917..e69de29bb 100644 --- a/Lib/test/test_import/data/circular_imports/subpkg2/__init__.py +++ b/Lib/test/test_import/data/circular_imports/subpkg2/__init__.py @@ -1 +0,0 @@ - diff --git a/Lib/test/test_import/data/package/submodule.py b/Lib/test/test_import/data/package/submodule.py index 8b1378917..e69de29bb 100644 --- a/Lib/test/test_import/data/package/submodule.py +++ b/Lib/test/test_import/data/package/submodule.py @@ -1 +0,0 @@ - diff --git a/Lib/test/test_import/data/package2/submodule2.py b/Lib/test/test_import/data/package2/submodule2.py index 8b1378917..e69de29bb 100644 --- a/Lib/test/test_import/data/package2/submodule2.py +++ b/Lib/test/test_import/data/package2/submodule2.py @@ -1 +0,0 @@ - diff --git a/Lib/test/test_import/data/unwritable/x.py b/Lib/test/test_import/data/unwritable/x.py index 8b1378917..e69de29bb 100644 --- a/Lib/test/test_import/data/unwritable/x.py +++ b/Lib/test/test_import/data/unwritable/x.py @@ -1 +0,0 @@ - diff --git a/Lib/test/test_script_helper.py b/Lib/test/test_script_helper.py index 4ade2cbc0..eeea6c484 100644 --- a/Lib/test/test_script_helper.py +++ b/Lib/test/test_script_helper.py @@ -3,7 +3,7 @@ import subprocess import sys import os -from test.support import script_helper +from test.support import script_helper, requires_subprocess import unittest from unittest import mock @@ -69,12 +69,12 @@ class TestScriptHelper(unittest.TestCase): self.assertNotIn('-E', popen_command) +@requires_subprocess() class TestScriptHelperEnvironment(unittest.TestCase): """Code coverage for interpreter_requires_environment().""" def setUp(self): - self.assertTrue( - hasattr(script_helper, '__cached_interp_requires_environment')) + self.assertHasAttr(script_helper, '__cached_interp_requires_environment') # Reset the private cached state. script_helper.__dict__['__cached_interp_requires_environment'] = None diff --git a/Lib/test/test_ucn.py b/Lib/test/test_ucn.py index 10f262cff..7821a41ea 100644 --- a/Lib/test/test_ucn.py +++ b/Lib/test/test_ucn.py @@ -88,6 +88,9 @@ class UnicodeNamesTest(unittest.TestCase): self.checkletter("HANGUL SYLLABLE HWEOK", "\ud6f8") self.checkletter("HANGUL SYLLABLE HIH", "\ud7a3") + self.checkletter("haNGul SYllABle WAe", '\uc65c') + self.checkletter("HAngUL syLLabLE waE", '\uc65c') + self.assertRaises(ValueError, unicodedata.name, "\ud7a4") def test_cjk_unified_ideographs(self): @@ -103,6 +106,36 @@ class UnicodeNamesTest(unittest.TestCase): self.checkletter("CJK UNIFIED IDEOGRAPH-2B81D", "\U0002B81D") self.checkletter("CJK UNIFIED IDEOGRAPH-3134A", "\U0003134A") + self.checkletter("cjK UniFIeD idEogRAph-3aBc", "\u3abc") + self.checkletter("CJk uNIfiEd IDeOGraPH-3AbC", "\u3abc") + self.checkletter("cjK UniFIeD idEogRAph-2aBcD", "\U0002abcd") + self.checkletter("CJk uNIfiEd IDeOGraPH-2AbCd", "\U0002abcd") + + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: got unexpected unicode + def test_tangut_ideographs(self): + self.checkletter("TANGUT IDEOGRAPH-17000", "\U00017000") + self.checkletter("TANGUT IDEOGRAPH-187F7", "\U000187f7") + self.checkletter("TANGUT IDEOGRAPH-18D00", "\U00018D00") + self.checkletter("TANGUT IDEOGRAPH-18D08", "\U00018d08") + self.checkletter("tangut ideograph-18d08", "\U00018d08") + + def test_egyptian_hieroglyphs(self): + self.checkletter("EGYPTIAN HIEROGLYPH-13460", "\U00013460") + self.checkletter("EGYPTIAN HIEROGLYPH-143FA", "\U000143fa") + self.checkletter("egyptian hieroglyph-143fa", "\U000143fa") + + def test_khitan_small_script_characters(self): + self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18B00", "\U00018b00") + self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CD5", "\U00018cd5") + self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CFF", "\U00018cff") + self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CFF", "\U00018cff") + self.checkletter("khitan small script character-18cff", "\U00018cff") + + def test_nushu_characters(self): + self.checkletter("NUSHU CHARACTER-1B170", "\U0001b170") + self.checkletter("NUSHU CHARACTER-1B2FB", "\U0001b2fb") + self.checkletter("nushu character-1b2fb", "\U0001b2fb") + def test_bmp_characters(self): for code in range(0x10000): char = chr(code) @@ -116,7 +149,7 @@ class UnicodeNamesTest(unittest.TestCase): self.checkletter("HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK", "\uFF9F") self.checkletter("FULLWIDTH LATIN SMALL LETTER A", "\uFF41") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_aliases(self): # Check that the aliases defined in the NameAliases.txt file work. # This should be updated when new aliases are added or the file @@ -157,7 +190,7 @@ class UnicodeNamesTest(unittest.TestCase): unicodedata.name(chr(cp)) self.assertEqual(str(cm.exception), 'no such name') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_named_sequences_sample(self): # Check a few named sequences. See #12753. sequences = [ @@ -174,7 +207,7 @@ class UnicodeNamesTest(unittest.TestCase): with self.assertRaises(KeyError): unicodedata.ucd_3_2_0.lookup(seqname) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_named_sequences_full(self): # Check all the named sequences def check_version(testfile): diff --git a/Lib/test/test_unicode_file_functions.py b/Lib/test/test_unicode_file_functions.py index 25c16e3a0..4a067d714 100644 --- a/Lib/test/test_unicode_file_functions.py +++ b/Lib/test/test_unicode_file_functions.py @@ -125,8 +125,8 @@ class UnicodeFileTests(unittest.TestCase): # open(), os.stat(), etc. don't raise any exception. @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "test fails on Emscripten/WASI when host platform is macOS." + support.is_wasi, + "test fails on WASI when host platform is macOS." ) def test_normalize(self): files = set(self.files) diff --git a/Lib/test/test_unicode_identifiers.py b/Lib/test/test_unicode_identifiers.py index 60cfdaabe..27749a080 100644 --- a/Lib/test/test_unicode_identifiers.py +++ b/Lib/test/test_unicode_identifiers.py @@ -17,11 +17,10 @@ class PEP3131Test(unittest.TestCase): 𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1 self.assertIn("Unicode", dir()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid(self): try: - from test.tokenizedata import badsyntax_3131 + from test.tokenizedata import badsyntax_3131 # noqa: F401 except SyntaxError as err: self.assertEqual(str(err), "invalid character '€' (U+20AC) (badsyntax_3131.py, line 2)")