gh-152132: Test Py_CompileString() in test_capi.test_run (#153470)

PyRun functions now raises ValueError if the start argument is
invalid.

* Add Modules/_testlimitedcapi/run.c.
* Rename _Py_CompileStringObjectWithModule() to
  _Py_CompileStringObject().

Include/cpython/pythonrun.h

@@ -40,6 +40,11 @@ PyAPI_FUNC(PyObject *) PyRun_FileExFlags(
     PyCompilerFlags *flags);
 
 
+PyAPI_FUNC(PyObject *) Py_CompileStringFlags(
+    const char *str,
+    const char *filename,
+    int start,
+    PyCompilerFlags *flags);
 PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(
     const char *str,
     const char *filename,       /* decoded from the filesystem encoding */

Include/internal/pycore_pythonrun.h

@@ -28,7 +28,7 @@ extern const char* _Py_SourceAsString(
     PyCompilerFlags *cf,
     PyObject **cmd_copy);
 
-extern PyObject * _Py_CompileStringObjectWithModule(
+extern PyObject * _Py_CompileString(
     const char *str,
     PyObject *filename, int start,
     PyCompilerFlags *flags, int optimize,

Lib/test/test_capi/test_run.py

@@ -1,20 +1,27 @@
+import ast
 import os
 import sys
 import tempfile
+import textwrap
 import unittest
 from collections import UserDict
 from test import support
 from test.support import import_helper
 from test.support.os_helper import unlink, TESTFN, TESTFN_ASCII, TESTFN_UNDECODABLE
 
+_testcapi = import_helper.import_module('_testcapi')
+_testlimitedcapi = import_helper.import_module('_testlimitedcapi')
+
 
 NULL = None
-_testcapi = import_helper.import_module('_testcapi')
 Py_single_input = _testcapi.Py_single_input
 Py_file_input = _testcapi.Py_file_input
 Py_eval_input = _testcapi.Py_eval_input
+INVALID_START = Py_single_input - 1
 STDIN = '<stdin>'
 STDERR_FD = 2
+PyCF_ONLY_AST = _testcapi.PyCF_ONLY_AST
+PyCF_IGNORE_COOKIE = _testcapi.PyCF_IGNORE_COOKIE
 
 # Code raising a SyntaxError
 SYNTAX_ERROR = 'True = 1'
@@ -68,6 +75,9 @@ class CAPITest(unittest.TestCase):
             run(b'raise ValueError("BUG")', {})
         self.assertEqual(str(cm.exception), 'BUG')
 
+        with self.assertRaises(ValueError):
+            func(b'x = 1', INVALID_START, {})
+
         self.assertIsNone(run(b'a\n', dict(a=1)))
         self.assertIsNone(run(b'a\n', dict(a=1), {}))
         self.assertIsNone(run(b'a\n', {}, dict(a=1)))
@@ -118,6 +128,9 @@ class CAPITest(unittest.TestCase):
             closeit = 1
             self.assertIsNone(run(dict(a=1), {}, closeit))
 
+        with self.assertRaises(ValueError):
+            func(filename, INVALID_START, {})
+
         self.assertRaises(NameError, run, {})
         self.assertRaises(NameError, run, {}, {})
         self.assertRaises(TypeError, run, dict(a=1), [])
@@ -357,6 +370,65 @@ class CAPITest(unittest.TestCase):
         # Test PyRun_SimpleStringFlags()
         self.check_run_simplestring(_testcapi.run_simplestringflags)
 
+    def check_compilestring(self, compilestring, has_flags, encode_filename=True):
+        filename_str = TESTFN
+        if encode_filename:
+            filename = os.fsencode(filename_str)
+        else:
+            filename = filename_str
+
+        def check_code(co, name, value):
+            ns = {}
+            exec(co, ns, ns)
+            self.assertEqual(ns[name], value)
+
+        co = compilestring(b'x = 1', filename, Py_file_input)
+        self.assertEqual(co.co_filename, filename_str)
+        check_code(co, 'x', 1)
+
+        if has_flags:
+            code = textwrap.dedent("""
+                # encoding: latin1
+                x = 'a\xe9'
+            """)
+            co = compilestring(code.encode(), filename, Py_file_input, PyCF_IGNORE_COOKIE)
+            self.assertEqual(co.co_filename, filename_str)
+            check_code(co, 'x', 'a\xe9')
+
+            co = compilestring(code.encode(), filename, Py_file_input)
+            self.assertEqual(co.co_filename, filename_str)
+            check_code(co, 'x', 'a\xc3\xa9')
+
+            tree = compilestring(b'x = 1', filename, Py_file_input, PyCF_ONLY_AST)
+            self.assertIsInstance(tree, ast.AST)
+
+        co = compilestring(b'raise ValueError("BUG")', filename, Py_file_input)
+        with self.assertRaises(ValueError):
+            exec(co, {})
+
+        with self.assertRaises(SyntaxError) as cm:
+            compilestring(SYNTAX_ERROR.encode(), filename, Py_file_input)
+
+        with self.assertRaises(ValueError):
+            compilestring(b'x = 1', filename, INVALID_START)
+
+    def test_compilestring(self):
+        # Test Py_CompileString()
+        self.check_compilestring(_testlimitedcapi.run_compilestring, False)
+
+    def test_compilestringflags(self):
+        # Test Py_CompileStringFlags()
+        self.check_compilestring(_testcapi.run_compilestringflags, True)
+
+    def test_compilestringexflags(self):
+        # Test Py_CompileStringExFlags()
+        self.check_compilestring(_testcapi.run_compilestringexflags, True)
+
+    def test_compilestringobject(self):
+        # Test Py_CompileStringObject()
+        self.check_compilestring(_testcapi.run_compilestringobject, True,
+                                 encode_filename=False)
+
 
 if __name__ == '__main__':
     unittest.main()

Modules/Setup.stdlib.in

@@ -174,7 +174,7 @@
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
 @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tuple.c
 @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c _testcapi/weakref.c
-@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c
+@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c
 @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c
 @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c
 

Modules/_testcapi/run.c

@@ -24,34 +24,16 @@
 #undef PyRun_InteractiveLoop
 
 
-// Some PyRun functions crash if start is invalid,
-// so validate the start argument.
-static int
-check_start(int start)
-{
-    if (start == Py_single_input || start == Py_file_input
-        || start == Py_eval_input || start == Py_func_type_input)
-    {
-        return 0;
-    }
-    PyErr_SetString(PyExc_ValueError, "invalid start argument");
-    return -1;
-}
-
-
 // Test PyRun_String()
 static PyObject*
 run_string(PyObject *mod, PyObject *args)
 {
     const char *str;
-    int start = 0;
+    int start;
     PyObject *globals = NULL;
     PyObject *locals = NULL;
 
-    if (!PyArg_ParseTuple(args, "y|iOO", &str, &start, &globals, &locals)) {
-        return NULL;
-    }
-    if (check_start(start) < 0) {
+    if (!PyArg_ParseTuple(args, "yi|OO", &str, &start, &globals, &locals)) {
         return NULL;
     }
     NULLABLE(globals);
@@ -79,9 +61,6 @@ run_stringflags(PyObject *mod, PyObject *args)
                           &cf_flags, &cf_feature_version)) {
         return NULL;
     }
-    if (check_start(start) < 0) {
-        return NULL;
-    }
     NULLABLE(globals);
     NULLABLE(locals);
     if (cf_flags || cf_feature_version) {
@@ -481,9 +460,6 @@ run_file(PyObject *mod, PyObject *args)
                           &globals, &locals)) {
         return NULL;
     }
-    if (check_start(start) < 0) {
-        return NULL;
-    }
     NULLABLE(globals);
     NULLABLE(locals);
 
@@ -518,9 +494,6 @@ run_fileex(PyObject *mod, PyObject *args)
                           &closeit)) {
         return NULL;
     }
-    if (check_start(start) < 0) {
-        return NULL;
-    }
     NULLABLE(globals);
     NULLABLE(locals);
 
@@ -560,9 +533,6 @@ run_fileflags(PyObject *mod, PyObject *args)
                           &cf_flags, &cf_feature_version)) {
         return NULL;
     }
-    if (check_start(start) < 0) {
-        return NULL;
-    }
     NULLABLE(globals);
     NULLABLE(locals);
     if (cf_flags || cf_feature_version) {
@@ -606,9 +576,6 @@ run_fileexflags(PyObject *mod, PyObject *args)
                           &closeit, &cf_flags, &cf_feature_version)) {
         return NULL;
     }
-    if (check_start(start) < 0) {
-        return NULL;
-    }
     NULLABLE(globals);
     NULLABLE(locals);
     if (cf_flags || cf_feature_version) {
@@ -733,6 +700,86 @@ run_interactiveloopflags(PyObject *mod, PyObject *args)
 }
 
 
+// Test Py_CompileStringFlags()
+static PyObject*
+run_compilestringflags(PyObject *mod, PyObject *args)
+{
+    const char *str;
+    const char *filename;
+    int start;
+    PyCompilerFlags flags = _PyCompilerFlags_INIT;
+    PyCompilerFlags *pflags = NULL;
+    int cf_flags = 0;
+    int cf_feature_version = 0;
+
+    if (!PyArg_ParseTuple(args, "yyi|ii", &str, &filename, &start,
+                          &cf_flags, &cf_feature_version)) {
+        return NULL;
+    }
+    if (cf_flags || cf_feature_version) {
+        flags.cf_flags = cf_flags;
+        flags.cf_feature_version = cf_feature_version;
+        pflags = &flags;
+    }
+
+    return Py_CompileStringFlags(str, filename, start, pflags);
+}
+
+
+// Test Py_CompileStringExFlags()
+static PyObject*
+run_compilestringexflags(PyObject *mod, PyObject *args)
+{
+    const char *str;
+    const char *filename;
+    int start;
+    PyCompilerFlags flags = _PyCompilerFlags_INIT;
+    PyCompilerFlags *pflags = NULL;
+    int cf_flags = 0;
+    int cf_feature_version = 0;
+    int optimize = -1;
+
+    if (!PyArg_ParseTuple(args, "yyi|iii", &str, &filename, &start,
+                          &cf_flags, &cf_feature_version, &optimize)) {
+        return NULL;
+    }
+    if (cf_flags || cf_feature_version) {
+        flags.cf_flags = cf_flags;
+        flags.cf_feature_version = cf_feature_version;
+        pflags = &flags;
+    }
+
+    return Py_CompileStringExFlags(str, filename, start, pflags, optimize);
+}
+
+
+// Test Py_CompileStringObject()
+static PyObject*
+run_compilestringobject(PyObject *mod, PyObject *args)
+{
+    const char *str;
+    PyObject *filename;
+    int start;
+    PyCompilerFlags flags = _PyCompilerFlags_INIT;
+    PyCompilerFlags *pflags = NULL;
+    int cf_flags = 0;
+    int cf_feature_version = 0;
+    int optimize = -1;
+
+    if (!PyArg_ParseTuple(args, "yOi|iii", &str, &filename, &start,
+                          &cf_flags, &cf_feature_version, &optimize)) {
+        return NULL;
+    }
+    if (cf_flags || cf_feature_version) {
+        flags.cf_flags = cf_flags;
+        flags.cf_feature_version = cf_feature_version;
+        pflags = &flags;
+    }
+
+    return Py_CompileStringObject(str, filename, start, pflags, optimize);
+}
+
+
 static PyMethodDef test_methods[] = {
     {"run_string", run_string, METH_VARARGS},
     {"run_stringflags", run_stringflags, METH_VARARGS},
@@ -754,6 +801,9 @@ static PyMethodDef test_methods[] = {
     {"run_simplestringflags", run_simplestringflags, METH_VARARGS},
     {"run_interactiveloop", run_interactiveloop, METH_VARARGS},
     {"run_interactiveloopflags", run_interactiveloopflags, METH_VARARGS},
+    {"run_compilestringflags", run_compilestringflags, METH_VARARGS},
+    {"run_compilestringexflags", run_compilestringexflags, METH_VARARGS},
+    {"run_compilestringobject", run_compilestringobject, METH_VARARGS},
     {NULL},
 };
 
@@ -763,5 +813,11 @@ _PyTestCapi_Init_Run(PyObject *mod)
     if (PyModule_AddFunctions(mod, test_methods) < 0) {
         return -1;
     }
+    if (PyModule_AddIntMacro(mod, PyCF_ONLY_AST) < 0) {
+        return -1;
+    }
+    if (PyModule_AddIntMacro(mod, PyCF_IGNORE_COOKIE) < 0) {
+        return -1;
+    }
     return 0;
 }

Modules/_testlimitedcapi.c

@@ -101,5 +101,8 @@ PyInit__testlimitedcapi(void)
     if (_PyTestLimitedCAPI_Init_Weakref(mod) < 0) {
         return NULL;
     }
+    if (_PyTestLimitedCAPI_Init_Run(mod) < 0) {
+        return NULL;
+    }
     return mod;
 }

Modules/_testlimitedcapi/parts.h

@@ -46,5 +46,6 @@ int _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *module);
 int _PyTestLimitedCAPI_Init_Version(PyObject *module);
 int _PyTestLimitedCAPI_Init_File(PyObject *module);
 int _PyTestLimitedCAPI_Init_Weakref(PyObject *module);
+int _PyTestLimitedCAPI_Init_Run(PyObject *module);
 
 #endif // Py_TESTLIMITEDCAPI_PARTS_H

Modules/_testlimitedcapi/run.c

@@ -0,0 +1,35 @@
+#include "parts.h"
+#include "util.h"
+
+
+// Test functions, not macros
+#undef Py_CompileString
+
+
+// Test Py_CompileString()
+static PyObject*
+run_compilestring(PyObject *mod, PyObject *args)
+{
+    const char *str;
+    const char *filename;
+    int start;
+
+    if (!PyArg_ParseTuple(args, "yyi", &str, &filename, &start)) {
+        return NULL;
+    }
+
+    return Py_CompileString(str, filename, start);
+}
+
+
+static PyMethodDef test_methods[] = {
+    {"run_compilestring", run_compilestring, METH_VARARGS},
+    {NULL},
+};
+
+int
+_PyTestLimitedCAPI_Init_Run(PyObject *m)
+{
+    return PyModule_AddFunctions(m, test_methods);
+}
+

PCbuild/_testlimitedcapi.vcxproj

@@ -118,6 +118,7 @@
     <ClCompile Include="..\Modules\_testlimitedcapi\version.c" />
     <ClCompile Include="..\Modules\_testlimitedcapi\file.c" />
     <ClCompile Include="..\Modules\_testlimitedcapi\weakref.c" />
+    <ClCompile Include="..\Modules\_testlimitedcapi\run.c" />
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="..\PC\python_nt.rc" />

PCbuild/_testlimitedcapi.vcxproj.filters

@@ -34,6 +34,7 @@
     <ClCompile Include="..\Modules\_testlimitedcapi\version.c" />
     <ClCompile Include="..\Modules\_testlimitedcapi\file.c" />
     <ClCompile Include="..\Modules\_testlimitedcapi\weakref.c" />
+    <ClCompile Include="..\Modules\_testlimitedcapi\run.c" />
     <ClCompile Include="..\Modules\_testlimitedcapi.c" />
   </ItemGroup>
   <ItemGroup>

Python/bltinmodule.c

@@ -960,9 +960,9 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename,
     tstate->suppress_co_const_immortalization++;
 #endif
 
-    result = _Py_CompileStringObjectWithModule(str, filename,
-                                               start[compile_mode], &cf,
-                                               optimize, modname);
+    result = _Py_CompileString(str, filename,
+                               start[compile_mode], &cf,
+                               optimize, modname);
 
 #ifdef Py_GIL_DISABLED
     tstate->suppress_co_const_immortalization--;

Python/pythonrun.c

@@ -1240,11 +1240,27 @@ void PyErr_DisplayException(PyObject *exc)
     PyErr_Display(NULL, exc, NULL);
 }
 
+static int
+check_start(int start)
+{
+    if (start == Py_single_input || start == Py_file_input
+        || start == Py_eval_input || start == Py_func_type_input)
+    {
+        return 0;
+    }
+    PyErr_SetString(PyExc_ValueError, "invalid start argument");
+    return -1;
+}
+
 static PyObject *
 _PyRun_String(const char *str, PyObject* name, int start,
               PyObject *globals, PyObject *locals, PyCompilerFlags *flags,
               int generate_new_source)
 {
+    if (check_start(start) < 0) {
+        return NULL;
+    }
+
     PyObject *ret = NULL;
     mod_ty mod;
     PyArena *arena;
@@ -1294,6 +1310,10 @@ static PyObject *
 _PyRun_File(FILE *fp, PyObject *filename, int start, PyObject *globals,
             PyObject *locals, int closeit, PyCompilerFlags *flags)
 {
+    if (check_start(start) < 0) {
+        return NULL;
+    }
+
     PyArena *arena = _PyArena_New();
     if (arena == NULL) {
         return NULL;
@@ -1538,14 +1558,17 @@ PyObject *
 Py_CompileStringObject(const char *str, PyObject *filename, int start,
                        PyCompilerFlags *flags, int optimize)
 {
-    return _Py_CompileStringObjectWithModule(str, filename, start,
-                                             flags, optimize, NULL);
+    return _Py_CompileString(str, filename, start, flags, optimize, NULL);
 }
 
 PyObject *
-_Py_CompileStringObjectWithModule(const char *str, PyObject *filename, int start,
-                       PyCompilerFlags *flags, int optimize, PyObject *module)
+_Py_CompileString(const char *str, PyObject *filename, int start,
+                  PyCompilerFlags *flags, int optimize, PyObject *module)
 {
+    if (check_start(start) < 0) {
+        return NULL;
+    }
+
     PyCodeObject *co;
     mod_ty mod;
     PyArena *arena = _PyArena_New();
@@ -1754,7 +1777,7 @@ Py_CompileString(const char *str, const char *p, int s)
 }
 
 #undef Py_CompileStringFlags
-PyAPI_FUNC(PyObject *)
+PyObject*
 Py_CompileStringFlags(const char *str, const char *p, int s,
                       PyCompilerFlags *flags)
 {