gh-157384: Unify _Py_ext_module_loader_info initialization (GH-157758)

- Unify `_Py_ext_module_loader_info_init*` functions. The “core” one was entirely unused, and the difference between “builtin” and “extension” can be expressed in arguments.
- Consolidate getting & checking the module name from spec. (This improves error message wording for `imp.create_builtin` with empty name.)
- Allow non-ASCII names.

Core modules (`sys` and `builtins`) still don't go through the routine.

Co-authored-by: Itamar Oren <itamarost@gmail.com>
This commit is contained in:
Petr ViktorinandItamar Oren authored and GitHub committed 2026-10-23 22:01:22 +02:00
1 parent e548b472f3
commit 18c935f72d
11 files changed
+165 -179

No files matched your search

+1
View File
@@ -250,6 +250,7 @@ _PyStaticObjects_CheckAll(PyInterpreterState *interp) {
_PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(_asyncio_future_blocking), "_asyncio_future_blocking", 24);
_PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(_blksize), "_blksize", 8);
_PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(_bootstrap), "_bootstrap", 10);
_PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(_bootstrap_external), "_bootstrap_external", 19);
_PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(_check_retval_), "_check_retval_", 14);
_PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(_dealloc_warn), "_dealloc_warn", 13);
_PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(_feature_version), "_feature_version", 16);
+1
View File
@@ -262,6 +262,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(_asyncio_future_blocking)
STRUCT_FOR_ID(_blksize)
STRUCT_FOR_ID(_bootstrap)
STRUCT_FOR_ID(_bootstrap_external)
STRUCT_FOR_ID(_check_retval_)
STRUCT_FOR_ID(_dealloc_warn)
STRUCT_FOR_ID(_feature_version)
+5 -16
View File
@@ -51,9 +51,9 @@ typedef enum ext_module_kind {
} _Py_ext_module_kind;
typedef enum ext_module_origin {
_Py_ext_module_origin_CORE = 1,
_Py_ext_module_origin_BUILTIN = 2,
_Py_ext_module_origin_DYNAMIC = 3,
_Py_ext_module_origin_CORE = 1, // 'sys' & 'builtins'
_Py_ext_module_origin_BUILTIN = 2, // inittab or PyImport_CreateModuleFromInitfunc
_Py_ext_module_origin_DYNAMIC = 3, // loaded dynamically
} _Py_ext_module_origin;
struct hook_prefixes {
@@ -79,21 +79,10 @@ struct _Py_ext_module_loader_info {
extern void _Py_ext_module_loader_info_clear(
struct _Py_ext_module_loader_info *info);
extern int _Py_ext_module_loader_info_init(
struct _Py_ext_module_loader_info *info,
struct _Py_ext_module_loader_info *p_info,
PyObject *name,
PyObject *filename,
PyObject *spec,
_Py_ext_module_origin origin);
extern int _Py_ext_module_loader_info_init_for_core(
struct _Py_ext_module_loader_info *p_info,
PyObject *name);
extern int _Py_ext_module_loader_info_init_for_builtin(
struct _Py_ext_module_loader_info *p_info,
PyObject *name);
#ifdef HAVE_DYNAMIC_LOADING
extern int _Py_ext_module_loader_info_init_from_spec(
struct _Py_ext_module_loader_info *info,
PyObject *spec);
#endif
/* The result from running an extension module's init function.
* Not used for modules defined via PyModExport (slots array).
+1
View File
@@ -1537,6 +1537,7 @@ extern "C" {
INIT_ID(_asyncio_future_blocking), \
INIT_ID(_blksize), \
INIT_ID(_bootstrap), \
INIT_ID(_bootstrap_external), \
INIT_ID(_check_retval_), \
INIT_ID(_dealloc_warn), \
INIT_ID(_feature_version), \
+1
View File
@@ -214,6 +214,7 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
offsetof(struct _Py_global_strings, identifiers._py__asyncio_future_blocking._ascii.ob_base),
offsetof(struct _Py_global_strings, identifiers._py__blksize._ascii.ob_base),
offsetof(struct _Py_global_strings, identifiers._py__bootstrap._ascii.ob_base),
offsetof(struct _Py_global_strings, identifiers._py__bootstrap_external._ascii.ob_base),
offsetof(struct _Py_global_strings, identifiers._py__check_retval_._ascii.ob_base),
offsetof(struct _Py_global_strings, identifiers._py__dealloc_warn._ascii.ob_base),
offsetof(struct _Py_global_strings, identifiers._py__feature_version._ascii.ob_base),
+6
View File
@@ -293,6 +293,12 @@ class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase):
"sp_pkg.sp_submod.__name__='sp_pkg.sp_submod'\n"
"sys.modules[\"sp_pkg.sp_submod\"] is sp_pkg.sp_submod=True\n"
"\"sp_submod\" in sys.modules=False\n"
# Module with non-ASCII name. (The SystemError shows the internal
# punycode-encoded name: not ideal, but also not horrible.)
"created nonascii_mp_\N{SNAKE}: in sys.modules=False\n"
"ascii(mp.__name__)=\"'nonascii_mp_\\\\U0001f40d'\" mp.executed='yes'\n"
"SystemError: 'initialization of nonascii_sp__rf17k "
"did not return PyModuleDef'\n"
)
def test_inittab_submodule_multiphase(self):
+9 -2
View File
@@ -1264,6 +1264,13 @@ os.does_not_exist
origin = "a\x00b"
_imp.create_dynamic(Spec2())
def test_create_dynamic_no_origin(self):
with self.assertRaisesRegex(AttributeError, "no attribute 'origin'"):
class Spec:
name = "abc"
_imp.create_dynamic(Spec())
def test_create_builtin(self):
class Spec:
pass
@@ -1273,7 +1280,7 @@ os.does_not_exist
self.assertIs(_imp.create_builtin(spec), sys)
spec.name = None
with self.assertRaisesRegex(TypeError, 'name must be string, not NoneType'):
with self.assertRaisesRegex(TypeError, 'module name must be a string, not NoneType'):
_imp.create_builtin(spec)
# gh-142029
@@ -1283,7 +1290,7 @@ os.does_not_exist
# gh-142029
spec.name = ""
with self.assertRaisesRegex(ValueError, 'name must not be empty'):
with self.assertRaisesRegex(ValueError, 'module name must not be empty'):
_imp.create_builtin(spec)
def test_filter_syntax_warnings_by_module(self):
@@ -0,0 +1,3 @@
:c:func:`PyImport_CreateModuleFromInitfunc` now accepts non-ASCII module
names for multi-phase init modules, instead of raising
:exc:`UnicodeEncodeError`.
+58 -1
View File
@@ -2460,6 +2460,40 @@ PyInit_cmfi_sp_pkg(void)
return PyModuleDef_Init(&cmfi_sp_pkg_def);
}
// Modules with non-ASCII names: multi-phase init is supported,
// single-phase init is not.
static PyModuleDef_Slot cmfi_nonascii_mp_slots[] = {
{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
{Py_mod_gil, Py_MOD_GIL_NOT_USED},
{0, NULL},
};
static PyModuleDef cmfi_nonascii_mp_def = {
PyModuleDef_HEAD_INIT,
.m_name = "nonascii_mp_\xf0\x9f\x90\x8d",
.m_size = 0,
.m_slots = cmfi_nonascii_mp_slots,
};
static PyObject*
PyInit_cmfi_nonascii_mp(void)
{
return PyModuleDef_Init(&cmfi_nonascii_mp_def);
}
static PyModuleDef cmfi_nonascii_sp_def = {
PyModuleDef_HEAD_INIT,
.m_name = "nonascii_sp_\xf0\x9f\x90\x8d",
.m_size = -1,
};
static PyObject*
PyInit_cmfi_nonascii_sp(void)
{
return PyModule_Create(&cmfi_nonascii_sp_def);
}
static PyObject*
create_module(PyObject* self, PyObject* spec)
{
@@ -2487,6 +2521,14 @@ create_module(PyObject* self, PyObject* spec)
Py_DECREF(name);
return PyImport_CreateModuleFromInitfunc(spec, PyInit_cmfi_sp_submod);
}
if (PyUnicode_EqualToUTF8(name, "nonascii_mp_\xf0\x9f\x90\x8d")) {
Py_DECREF(name);
return PyImport_CreateModuleFromInitfunc(spec, PyInit_cmfi_nonascii_mp);
}
if (PyUnicode_EqualToUTF8(name, "nonascii_sp_\xf0\x9f\x90\x8d")) {
Py_DECREF(name);
return PyImport_CreateModuleFromInitfunc(spec, PyInit_cmfi_nonascii_sp);
}
PyErr_Format(PyExc_LookupError, "static module %R not found", name);
Py_DECREF(name);
return NULL;
@@ -2548,6 +2590,11 @@ test_create_module_from_initfunc(void)
L"print(f'{sp_pkg.sp_submod.__name__=}');"
L"print(f'{sys.modules[\"sp_pkg.sp_submod\"] is sp_pkg.sp_submod=}');"
L"print(f'{\"sp_submod\" in sys.modules=}');"
// Non-ASCII names: multi-phase init works, single-phase init doesn't
L"import importlib;"
L"mp = importlib.import_module('nonascii_mp_\\N{SNAKE}');"
L"print(f'{ascii(mp.__name__)=} {mp.executed=}');"
L"try_import('nonascii_sp_\\N{SNAKE}');"
};
PyConfig config;
if (PyImport_AppendInittab("create_static_module",
@@ -2568,7 +2615,10 @@ test_create_module_from_initfunc(void)
" @classmethod\n"
" def find_spec(cls, fullname, path, target=None):\n"
" if fullname in {'my_test_extension', 'embedded_ext',\n"
" 'sp_pkg', 'sp_pkg.sp_submod'}:\n"
" 'sp_pkg', 'sp_pkg.sp_submod',\n"
" 'nonascii_mp_\\N{SNAKE}',\n"
" 'nonascii_sp_\\N{SNAKE}',\n"
" }:\n"
" return spec_from_loader(fullname, cls, origin=cls._ORIGIN,\n"
" is_package=(fullname == 'sp_pkg'))\n"
" return None\n"
@@ -2583,6 +2633,13 @@ test_create_module_from_initfunc(void)
" create_static_module.exec_module(module)\n"
" module.executed = 'yes'\n"
"sys.meta_path.append(StaticExtensionImporter)\n"
"def try_import(name):\n"
" try:\n"
" importlib.import_module(name)\n"
" except SystemError as exc:\n"
" print(f'SystemError: {ascii(str(exc))}')\n"
" else:\n"
" print(f'no SystemError for {ascii(name)}!')\n"
);
if (result < 0) {
fprintf(stderr, "PyRun_SimpleString() failed\n");
+17 -60
View File
@@ -2507,27 +2507,16 @@ lookup_inittab_entry(const struct _Py_ext_module_loader_info* info)
static PyObject*
create_builtin(
PyThreadState *tstate, PyObject *name,
PyThreadState *tstate,
PyObject *name, /* if NULL, get from spec */
PyObject *spec,
PyModInitFunction initfunc)
{
struct _Py_ext_module_loader_info info;
if (_Py_ext_module_loader_info_init_for_builtin(&info, name) < 0) {
_Py_ext_module_origin origin = _Py_ext_module_origin_BUILTIN;
if (_Py_ext_module_loader_info_init(&info, name, spec, origin) < 0) {
return NULL;
}
if (initfunc != NULL) {
/* An explicitly provided init function (see
* PyImport_CreateModuleFromInitfunc()) may belong to a submodule
* whose single-phase init creates the module using only the last
* component of the name (as e.g. pybind11 does). Set the package
* context so that PyModule_Create() resolves the full name, the
* same as for dynamically loaded extensions. */
info.newcontext = PyUnicode_AsUTF8(info.name);
if (info.newcontext == NULL) {
_Py_ext_module_loader_info_clear(&info);
return NULL;
}
}
struct extensions_cache_value *cached = NULL;
PyObject *mod = import_find_extension(tstate, &info, &cached);
@@ -2558,7 +2547,7 @@ create_builtin(
struct _inittab *entry = lookup_inittab_entry(&info);
if (entry == NULL) {
mod = NULL;
_PyErr_SetModuleNotFoundError(name);
_PyErr_SetModuleNotFoundError(info.name);
goto finally;
}
@@ -2610,20 +2599,7 @@ PyImport_CreateModuleFromInitfunc(
PyThreadState *tstate = _PyThreadState_GET();
PyObject *name = PyObject_GetAttr(spec, &_Py_ID(name));
if (name == NULL) {
return NULL;
}
if (!PyUnicode_Check(name)) {
PyErr_Format(PyExc_TypeError,
"spec name must be string, not %T", name);
Py_DECREF(name);
return NULL;
}
PyObject *mod = create_builtin(tstate, name, spec, initfunc);
Py_DECREF(name);
PyObject *mod = create_builtin(tstate, NULL, spec, initfunc);
return mod;
}
@@ -2835,8 +2811,8 @@ PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co,
Py_FatalError("no current interpreter");
}
external= PyObject_GetAttrString(IMPORTLIB(interp),
"_bootstrap_external");
external= PyObject_GetAttr(IMPORTLIB(interp),
&_Py_ID(_bootstrap_external));
if (external != NULL) {
pathobj = PyObject_CallMethodOneArg(
external, &_Py_ID(_get_sourcefile), cpathobj);
@@ -2920,8 +2896,8 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname,
if (pathname == NULL) {
pathname = ((PyCodeObject *)co)->co_filename;
}
external = PyObject_GetAttrString(IMPORTLIB(tstate->interp),
"_bootstrap_external");
external = PyObject_GetAttr(IMPORTLIB(tstate->interp),
&_Py_ID(_bootstrap_external));
if (external == NULL) {
Py_DECREF(d);
return NULL;
@@ -3499,8 +3475,8 @@ PyObject *
_PyImport_GetImportlibExternalLoader(PyInterpreterState *interp,
const char *loader_name)
{
PyObject *bootstrap = PyObject_GetAttrString(IMPORTLIB(interp),
"_bootstrap_external");
PyObject *bootstrap = PyObject_GetAttr(IMPORTLIB(interp),
&_Py_ID(_bootstrap_external));
if (bootstrap == NULL) {
return NULL;
}
@@ -3513,8 +3489,8 @@ _PyImport_GetImportlibExternalLoader(PyInterpreterState *interp,
PyObject *
_PyImport_BlessMyLoader(PyInterpreterState *interp, PyObject *module_globals)
{
PyObject *external = PyObject_GetAttrString(IMPORTLIB(interp),
"_bootstrap_external");
PyObject *external = PyObject_GetAttr(IMPORTLIB(interp),
&_Py_ID(_bootstrap_external));
if (external == NULL) {
return NULL;
}
@@ -5085,27 +5061,7 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
{
PyThreadState *tstate = _PyThreadState_GET();
PyObject *name = PyObject_GetAttrString(spec, "name");
if (name == NULL) {
return NULL;
}
if (!PyUnicode_Check(name)) {
PyErr_Format(PyExc_TypeError,
"name must be string, not %.200s",
Py_TYPE(name)->tp_name);
Py_DECREF(name);
return NULL;
}
if (PyUnicode_GetLength(name) == 0) {
PyErr_Format(PyExc_ValueError, "name must not be empty");
Py_DECREF(name);
return NULL;
}
PyObject *mod = create_builtin(tstate, name, spec, NULL);
Py_DECREF(name);
PyObject *mod = create_builtin(tstate, NULL, spec, NULL);
return mod;
}
@@ -5437,7 +5393,8 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file)
PyThreadState *tstate = _PyThreadState_GET();
struct _Py_ext_module_loader_info info;
if (_Py_ext_module_loader_info_init_from_spec(&info, spec) < 0) {
_Py_ext_module_origin origin = _Py_ext_module_origin_DYNAMIC;
if (_Py_ext_module_loader_info_init(&info, NULL, spec, origin) < 0) {
return NULL;
}
+63 -100
View File
@@ -8,6 +8,7 @@
#include "pycore_moduleobject.h" // _PyModule_GetDefOrNull()
#include "pycore_pyerrors.h" // _PyErr_FormatFromCause()
#include "pycore_runtime.h" // _Py_ID()
#include "pycore_global_objects.h" // _Py_SINGLETON()
/***********************************/
@@ -28,8 +29,6 @@ static const struct hook_prefixes nonascii_prefixes = {
static PyObject *
get_encoded_name(PyObject *name, const struct hook_prefixes **hook_prefixes) {
PyObject *tmp;
PyObject *encoded = NULL;
PyObject *modname = NULL;
Py_ssize_t name_len, lastdot;
/* Get the short name (substring after last dot) */
@@ -51,33 +50,38 @@ get_encoded_name(PyObject *name, const struct hook_prefixes **hook_prefixes) {
}
/* Encode to ASCII or Punycode, as needed */
encoded = PyUnicode_AsEncodedString(name, "ascii", NULL);
if (encoded != NULL) {
PyObject *encoded;
if (PyUnicode_IS_ASCII(name)) {
*hook_prefixes = &ascii_only_prefixes;
encoded = PyUnicode_AsASCIIString(name);
} else {
if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) {
PyErr_Clear();
encoded = PyUnicode_AsEncodedString(name, "punycode", NULL);
if (encoded == NULL) {
goto error;
}
*hook_prefixes = &nonascii_prefixes;
} else {
goto error;
}
*hook_prefixes = &nonascii_prefixes;
encoded = PyUnicode_AsEncodedString(name, "punycode", NULL);
}
if (encoded == NULL) {
goto error;
}
/* Replace '-' by '_' */
modname = _PyObject_CallMethod(encoded, &_Py_ID(replace), "cc", '-', '_');
if (modname == NULL)
/* Call encoded.replace('-', '_') */
PyObject *modname = PyObject_VectorcallMethod(
&_Py_ID(replace),
(PyObject*[3]) {
encoded,
(PyObject *)&_Py_SINGLETON(bytes_characters)['-'],
(PyObject *)&_Py_SINGLETON(bytes_characters)['_'],
},
3 | PY_VECTORCALL_ARGUMENTS_OFFSET,
NULL);
Py_CLEAR(encoded);
if (modname == NULL) {
goto error;
}
Py_DECREF(name);
Py_DECREF(encoded);
return modname;
error:
Py_DECREF(name);
Py_XDECREF(encoded);
return NULL;
}
@@ -94,123 +98,82 @@ _Py_ext_module_loader_info_clear(struct _Py_ext_module_loader_info *info)
int
_Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info,
PyObject *name, PyObject *filename,
PyObject *name,
PyObject *spec,
_Py_ext_module_origin origin)
{
struct _Py_ext_module_loader_info info = {
.origin=origin,
};
assert(name != NULL);
if (!PyUnicode_Check(name)) {
PyErr_SetString(PyExc_TypeError,
"module name must be a string");
_Py_ext_module_loader_info_clear(&info);
return -1;
if (name == NULL) {
assert(spec != NULL);
info.name = PyObject_GetAttr(spec, &_Py_ID(name));
if (info.name == NULL) {
goto error;
}
}
else {
info.name = Py_NewRef(name);
}
if (!PyUnicode_Check(info.name)) {
PyErr_Format(PyExc_TypeError,
"module name must be a string, not %T",
info.name);
goto error;
}
if (PyUnicode_GetLength(info.name) == 0) {
PyErr_Format(PyExc_ValueError, "module name must not be empty");
goto error;
}
assert(PyUnicode_GetLength(name) > 0);
info.name = Py_NewRef(name);
info.name_encoded = get_encoded_name(info.name, &info.hook_prefixes);
if (info.name_encoded == NULL) {
_Py_ext_module_loader_info_clear(&info);
return -1;
goto error;
}
info.newcontext = PyUnicode_AsUTF8(info.name);
if (info.newcontext == NULL) {
_Py_ext_module_loader_info_clear(&info);
return -1;
goto error;
}
if (filename != NULL) {
if (!PyUnicode_Check(filename)) {
if (origin == _Py_ext_module_origin_DYNAMIC) {
assert(spec != NULL);
info.filename = PyObject_GetAttr(spec, &_Py_ID(origin));
if (info.filename == NULL) {
goto error;
}
if (!PyUnicode_Check(info.filename)) {
PyErr_SetString(PyExc_TypeError,
"module filename must be a string");
_Py_ext_module_loader_info_clear(&info);
return -1;
goto error;
}
info.filename = Py_NewRef(filename);
#ifndef MS_WINDOWS
info.filename_encoded = PyUnicode_EncodeFSDefault(info.filename);
if (info.filename_encoded == NULL) {
_Py_ext_module_loader_info_clear(&info);
return -1;
goto error;
}
#endif
info.path = info.filename;
}
else {
info.filename = NULL;
info.path = info.name;
}
*p_info = info;
return 0;
error:
_Py_ext_module_loader_info_clear(&info);
return -1;
}
int
_Py_ext_module_loader_info_init_for_builtin(
struct _Py_ext_module_loader_info *info,
PyObject *name)
{
assert(PyUnicode_Check(name));
assert(PyUnicode_GetLength(name) > 0);
PyObject *name_encoded = PyUnicode_AsEncodedString(name, "ascii", NULL);
if (name_encoded == NULL) {
return -1;
}
*info = (struct _Py_ext_module_loader_info){
.name=Py_NewRef(name),
.name_encoded=name_encoded,
/* We won't need filename. */
.path=name,
.origin=_Py_ext_module_origin_BUILTIN,
.hook_prefixes=&ascii_only_prefixes,
.newcontext=NULL,
};
return 0;
}
int
_Py_ext_module_loader_info_init_for_core(
struct _Py_ext_module_loader_info *info,
PyObject *name)
{
if (_Py_ext_module_loader_info_init_for_builtin(info, name) < 0) {
return -1;
}
info->origin = _Py_ext_module_origin_CORE;
return 0;
}
#ifdef HAVE_DYNAMIC_LOADING
int
_Py_ext_module_loader_info_init_from_spec(
struct _Py_ext_module_loader_info *p_info,
PyObject *spec)
{
PyObject *name = PyObject_GetAttrString(spec, "name");
if (name == NULL) {
return -1;
}
PyObject *filename = PyObject_GetAttrString(spec, "origin");
if (filename == NULL) {
Py_DECREF(name);
return -1;
}
/* We could also accommodate builtin modules here without much trouble. */
_Py_ext_module_origin origin = _Py_ext_module_origin_DYNAMIC;
int err = _Py_ext_module_loader_info_init(p_info, name, filename, origin);
Py_DECREF(name);
Py_DECREF(filename);
return err;
}
#endif /* HAVE_DYNAMIC_LOADING */
/********************************/
/* module init function results */