gh-157384: Accept non-ASCII names in PyImport_CreateModuleFromInitfunc()

The builtin loader info encoded the module name as ASCII, so a
non-ASCII spec name failed with UnicodeEncodeError even for multi-phase
init modules, which support such names when loaded dynamically.  Fall
back to UTF-8 and mark the name as non-ASCII, so that multi-phase init
works and single-phase init is rejected with the same error as for
dynamically loaded extensions.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HoZE3WdRWQshvinfBZ137N

Author: itamaro

Lib/test/test_embed.py

@@ -278,6 +278,9 @@ def test_create_module_from_initfunc(self):
                          "my_test_extension.exec_slot_ran='yes'\n"
                          "<module 'embedded_ext' (static-extension)>\n"
                          "embedded_ext.executed='yes'\n"
+                         "ascii(mp.__name__)=\"'m\\\\xf6dul_mp'\" mp.executed='yes'\n"
+                         "SystemError: 'initialization of m\\xf6dul_sp "
+                         "did not return PyModuleDef'\n"
                          )
 
     def test_inittab_submodule_multiphase(self):

Misc/NEWS.d/next/C_API/2026-09-12-18-37-22.gh-issue-157384.uFbdle.rst

@@ -0,0 +1,3 @@
+:c:func:`PyImport_CreateModuleFromInitfunc` now accepts non-ASCII module
+names for multi-phase init modules, instead of raising
+:exc:`UnicodeEncodeError`.

Programs/_testembed.c

@@ -2410,6 +2410,43 @@ static int test_repeated_init_and_inittab(void)
     return 0;
 }
 
+// Modules with non-ASCII names: multi-phase init is supported,
+// single-phase init is not.
+static PyModuleDef_Slot cmfi_nonascii_mp_slots[] = {
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
+    {0, NULL},
+};
+
+static PyModuleDef cmfi_nonascii_mp_def = {
+    PyModuleDef_HEAD_INIT,
+    .m_name = "nonascii_mp",
+    .m_size = 0,
+    .m_slots = cmfi_nonascii_mp_slots,
+};
+
+static PyObject*
+PyInit_cmfi_nonascii_mp(void)
+{
+    return PyModuleDef_Init(&cmfi_nonascii_mp_def);
+}
+
+static PyModuleDef cmfi_nonascii_sp_def = {
+    PyModuleDef_HEAD_INIT,
+    .m_name = "nonascii_sp",
+    .m_size = -1,
+};
+
+static PyObject*
+PyInit_cmfi_nonascii_sp(void)
+{
+    return PyModule_Create(&cmfi_nonascii_sp_def);
+}
+
+// "m\xf6dul_mp" and "m\xf6dul_sp" as UTF-8
+#define CMFI_NONASCII_MP_NAME "m\xc3\xb6" "dul_mp"
+#define CMFI_NONASCII_SP_NAME "m\xc3\xb6" "dul_sp"
+
 static PyObject*
 create_module(PyObject* self, PyObject* spec)
 {
@@ -2425,6 +2462,14 @@ create_module(PyObject* self, PyObject* spec)
         Py_DECREF(name);
         return PyImport_CreateModuleFromInitfunc(spec, PyInit_embedded_ext);
     }
+    if (PyUnicode_EqualToUTF8(name, CMFI_NONASCII_MP_NAME)) {
+        Py_DECREF(name);
+        return PyImport_CreateModuleFromInitfunc(spec, PyInit_cmfi_nonascii_mp);
+    }
+    if (PyUnicode_EqualToUTF8(name, CMFI_NONASCII_SP_NAME)) {
+        Py_DECREF(name);
+        return PyImport_CreateModuleFromInitfunc(spec, PyInit_cmfi_nonascii_sp);
+    }
     PyErr_Format(PyExc_LookupError, "static module %R not found", name);
     Py_DECREF(name);
     return NULL;
@@ -2472,6 +2517,11 @@ test_create_module_from_initfunc(void)
         L"import embedded_ext;"
         L"print(embedded_ext);"
         L"print(f'{embedded_ext.executed=}');"
+        // Non-ASCII names: multi-phase init works, single-phase init doesn't
+        L"import importlib;"
+        L"mp = importlib.import_module('m\\xf6dul_mp');"
+        L"print(f'{ascii(mp.__name__)=} {mp.executed=}');"
+        L"try_import('m\\xf6dul_sp');"
     };
     PyConfig config;
     if (PyImport_AppendInittab("create_static_module",
@@ -2491,7 +2541,8 @@ test_create_module_from_initfunc(void)
         "   _ORIGIN = \"static-extension\"\n"
         "   @classmethod\n"
         "   def find_spec(cls, fullname, path, target=None):\n"
-        "       if fullname in {'my_test_extension', 'embedded_ext'}:\n"
+        "       if fullname in {'my_test_extension', 'embedded_ext',\n"
+        "                       'm\\xf6dul_mp', 'm\\xf6dul_sp'}:\n"
         "           return spec_from_loader(fullname, cls, origin=cls._ORIGIN)\n"
         "       return None\n"
         "   @staticmethod\n"
@@ -2502,6 +2553,13 @@ test_create_module_from_initfunc(void)
         "       create_static_module.exec_module(module)\n"
         "       module.executed = 'yes'\n"
         "sys.meta_path.append(StaticExtensionImporter)\n"
+        "def try_import(name):\n"
+        "   try:\n"
+        "       importlib.import_module(name)\n"
+        "   except SystemError as exc:\n"
+        "       print(f'SystemError: {ascii(str(exc))}')\n"
+        "   else:\n"
+        "       print(f'no SystemError for {ascii(name)}!')\n"
     );
     if (result < 0) {
         fprintf(stderr, "PyRun_SimpleString() failed\n");

Python/importdl.c

@@ -158,9 +158,22 @@ _Py_ext_module_loader_info_init_for_builtin(
     assert(PyUnicode_Check(name));
     assert(PyUnicode_GetLength(name) > 0);
 
+    /* The encoded name is only used for error messages, so unlike
+     * get_encoded_name() we keep the full dotted name.  Non-ASCII names
+     * are allowed, but only for multi-phase init modules; hook_prefixes
+     * records which case we are in. */
+    const struct hook_prefixes *hook_prefixes = &ascii_only_prefixes;
     PyObject *name_encoded = PyUnicode_AsEncodedString(name, "ascii", NULL);
     if (name_encoded == NULL) {
-        return -1;
+        if (!PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) {
+            return -1;
+        }
+        PyErr_Clear();
+        name_encoded = PyUnicode_AsUTF8String(name);
+        if (name_encoded == NULL) {
+            return -1;
+        }
+        hook_prefixes = &nonascii_prefixes;
     }
 
     *info = (struct _Py_ext_module_loader_info){
@@ -169,7 +182,7 @@ _Py_ext_module_loader_info_init_for_builtin(
         /* We won't need filename. */
         .path=name,
         .origin=_Py_ext_module_origin_BUILTIN,
-        .hook_prefixes=&ascii_only_prefixes,
+        .hook_prefixes=hook_prefixes,
         .newcontext=NULL,
     };
     return 0;