gh-108494: Argument Clinic partial supports of Limited C API (#108495)

Argument Clinic now has a partial support of the
Limited API:

* Add --limited option to clinic.c.
* Add '_testclinic_limited' extension which is built with
  the limited C API version 3.13.
* For now, hardcode in clinic.py that "_testclinic_limited.c" targets
  the limited C API.

Lib/test/test_clinic.py

@@ -13,6 +13,7 @@ import inspect
 import os.path
 import re
 import sys
+import types
 import unittest
 
 test_tools.skip_if_missing('clinic')
@@ -21,6 +22,13 @@ with test_tools.imports_under_tool('clinic'):
     from clinic import DSLParser
 
 
+def default_namespace():
+    ns = types.SimpleNamespace()
+    ns.force = False
+    ns.limited_capi = clinic.DEFAULT_LIMITED_CAPI
+    return ns
+
+
 def _make_clinic(*, filename='clinic_tests'):
     clang = clinic.CLanguage(None)
     c = clinic.Clinic(clang, filename=filename)
@@ -52,6 +60,11 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None,
     return cm.exception
 
 
+class MockClinic:
+    def __init__(self):
+        self.limited_capi = clinic.DEFAULT_LIMITED_CAPI
+
+
 class ClinicWholeFileTest(TestCase):
     maxDiff = None
 
@@ -691,8 +704,9 @@ class ParseFileUnitTest(TestCase):
         self, *, filename, expected_error, verify=True, output=None
     ):
         errmsg = re.escape(dedent(expected_error).strip())
+        ns = default_namespace()
         with self.assertRaisesRegex(clinic.ClinicError, errmsg):
-            clinic.parse_file(filename)
+            clinic.parse_file(filename, ns=ns)
 
     def test_parse_file_no_extension(self) -> None:
         self.expect_parsing_failure(
@@ -832,8 +846,9 @@ class ClinicBlockParserTest(TestCase):
 
         blocks = list(clinic.BlockParser(input, language))
         writer = clinic.BlockPrinter(language)
+        mock_clinic = MockClinic()
         for block in blocks:
-            writer.print_block(block)
+            writer.print_block(block, clinic=mock_clinic)
         output = writer.f.getvalue()
         assert output == input, "output != input!\n\noutput " + repr(output) + "\n\n input " + repr(input)
 
@@ -3508,6 +3523,27 @@ class ClinicFunctionalTest(unittest.TestCase):
         self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g")
 
 
+try:
+    import _testclinic_limited
+except ImportError:
+    _testclinic_limited = None
+
+@unittest.skipIf(_testclinic_limited is None, "_testclinic_limited is missing")
+class LimitedCAPIFunctionalTest(unittest.TestCase):
+    locals().update((name, getattr(_testclinic_limited, name))
+                    for name in dir(_testclinic_limited) if name.startswith('test_'))
+
+    def test_my_int_func(self):
+        with self.assertRaises(TypeError):
+            _testclinic_limited.my_int_func()
+        self.assertEqual(_testclinic_limited.my_int_func(3), 3)
+        with self.assertRaises(TypeError):
+            _testclinic_limited.my_int_func(1.0)
+        with self.assertRaises(TypeError):
+            _testclinic_limited.my_int_func("xyz")
+
+
+
 class PermutationTests(unittest.TestCase):
     """Test permutation support functions."""
 

Misc/NEWS.d/next/Tools-Demos/2023-08-25-22-40-12.gh-issue-108494.4RbDdu.rst

@@ -0,0 +1,2 @@
+:ref:`Argument Clinic <howto-clinic>` now has a partial support of the
+:ref:`Limited API <limited-c-api>`. Patch by Victor Stinner.

Modules/Setup.stdlib.in

@@ -161,6 +161,7 @@
 @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c
 @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c
 @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c
+@MODULE__TESTCLINIC_TRUE@_testclinic_limited _testclinic_limited.c
 
 # Some testing modules MUST be built as shared libraries.
 *shared*

Modules/_testclinic_limited.c

@@ -0,0 +1,69 @@
+// For now, only limited C API 3.13 is supported
+#define Py_LIMITED_API 0x030d0000
+
+/* Always enable assertions */
+#undef NDEBUG
+
+#include "Python.h"
+
+
+#include "clinic/_testclinic_limited.c.h"
+
+
+/*[clinic input]
+module  _testclinic_limited
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=dd408149a4fc0dbb]*/
+
+
+/*[clinic input]
+test_empty_function
+
+[clinic start generated code]*/
+
+static PyObject *
+test_empty_function_impl(PyObject *module)
+/*[clinic end generated code: output=0f8aeb3ddced55cb input=0dd7048651ad4ae4]*/
+{
+    Py_RETURN_NONE;
+}
+
+
+/*[clinic input]
+my_int_func -> int
+
+    arg: int
+    /
+
+[clinic start generated code]*/
+
+static int
+my_int_func_impl(PyObject *module, int arg)
+/*[clinic end generated code: output=761cd54582f10e4f input=16eb8bba71d82740]*/
+{
+    return arg;
+}
+
+
+static PyMethodDef tester_methods[] = {
+    TEST_EMPTY_FUNCTION_METHODDEF
+    MY_INT_FUNC_METHODDEF
+    {NULL, NULL}
+};
+
+static struct PyModuleDef _testclinic_module = {
+    PyModuleDef_HEAD_INIT,
+    .m_name = "_testclinic_limited",
+    .m_size = 0,
+    .m_methods = tester_methods,
+};
+
+PyMODINIT_FUNC
+PyInit__testclinic_limited(void)
+{
+    PyObject *m = PyModule_Create(&_testclinic_module);
+    if (m == NULL) {
+        return NULL;
+    }
+    return m;
+}

Modules/clinic/_testclinic_limited.c.h

@@ -0,0 +1,53 @@
+/*[clinic input]
+preserve
+[clinic start generated code]*/
+
+PyDoc_STRVAR(test_empty_function__doc__,
+"test_empty_function($module, /)\n"
+"--\n"
+"\n");
+
+#define TEST_EMPTY_FUNCTION_METHODDEF    \
+    {"test_empty_function", (PyCFunction)test_empty_function, METH_NOARGS, test_empty_function__doc__},
+
+static PyObject *
+test_empty_function_impl(PyObject *module);
+
+static PyObject *
+test_empty_function(PyObject *module, PyObject *Py_UNUSED(ignored))
+{
+    return test_empty_function_impl(module);
+}
+
+PyDoc_STRVAR(my_int_func__doc__,
+"my_int_func($module, arg, /)\n"
+"--\n"
+"\n");
+
+#define MY_INT_FUNC_METHODDEF    \
+    {"my_int_func", (PyCFunction)my_int_func, METH_O, my_int_func__doc__},
+
+static int
+my_int_func_impl(PyObject *module, int arg);
+
+static PyObject *
+my_int_func(PyObject *module, PyObject *arg_)
+{
+    PyObject *return_value = NULL;
+    int arg;
+    int _return_value;
+
+    arg = PyLong_AsInt(arg_);
+    if (arg == -1 && PyErr_Occurred()) {
+        goto exit;
+    }
+    _return_value = my_int_func_impl(module, arg);
+    if ((_return_value == -1) && PyErr_Occurred()) {
+        goto exit;
+    }
+    return_value = PyLong_FromLong((long)_return_value);
+
+exit:
+    return return_value;
+}
+/*[clinic end generated code: output=07e2e8ed6923cd16 input=a9049054013a1b77]*/

Tools/build/generate_stdlib_module_names.py

@@ -28,6 +28,7 @@ IGNORE = {
     '_testbuffer',
     '_testcapi',
     '_testclinic',
+    '_testclinic_limited',
     '_testconsole',
     '_testimportmultiple',
     '_testinternalcapi',

Tools/clinic/clinic.py

@@ -63,6 +63,7 @@ from typing import (
 
 version = '1'
 
+DEFAULT_LIMITED_CAPI = False
 NO_VARARG = "PY_SSIZE_T_MAX"
 CLINIC_PREFIX = "__clinic_"
 CLINIC_PREFIXED_ARGS = {
@@ -1360,7 +1361,21 @@ class CLanguage(Language):
                     vararg
                 )
                 nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0"
-            if not new_or_init:
+
+            if clinic.limited_capi:
+                # positional-or-keyword arguments
+                flags = "METH_VARARGS|METH_KEYWORDS"
+
+                parser_prototype = self.PARSER_PROTOTYPE_KEYWORD
+                parser_code = [normalize_snippet("""
+                    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "{format_units}:{name}", _keywords,
+                        {parse_arguments}))
+                        goto exit;
+                """, indent=4)]
+                argname_fmt = 'args[%d]'
+                declarations = ""
+
+            elif not new_or_init:
                 flags = "METH_FASTCALL|METH_KEYWORDS"
                 parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS
                 argname_fmt = 'args[%d]'
@@ -2111,7 +2126,8 @@ class BlockPrinter:
             self,
             block: Block,
             *,
-            core_includes: bool = False
+            core_includes: bool = False,
+            clinic: Clinic | None = None,
     ) -> None:
         input = block.input
         output = block.output
@@ -2140,7 +2156,11 @@ class BlockPrinter:
         write("\n")
 
         output = ''
-        if core_includes:
+        if clinic:
+            limited_capi = clinic.limited_capi
+        else:
+            limited_capi = DEFAULT_LIMITED_CAPI
+        if core_includes and not limited_capi:
             output += textwrap.dedent("""
                 #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
                 #  include "pycore_gc.h"            // PyGC_Head
@@ -2344,6 +2364,7 @@ impl_definition block
             *,
             filename: str,
             verify: bool = True,
+            limited_capi: bool = False,
     ) -> None:
         # maps strings to Parser objects.
         # (instantiated from the "parsers" global.)
@@ -2353,6 +2374,7 @@ impl_definition block
             fail("Custom printers are broken right now")
         self.printer = printer or BlockPrinter(language)
         self.verify = verify
+        self.limited_capi = limited_capi
         self.filename = filename
         self.modules: ModuleDict = {}
         self.classes: ClassDict = {}
@@ -2450,7 +2472,7 @@ impl_definition block
                     self.parsers[dsl_name] = parsers[dsl_name](self)
                 parser = self.parsers[dsl_name]
                 parser.parse(block)
-            printer.print_block(block)
+            printer.print_block(block, clinic=self)
 
         # these are destinations not buffers
         for name, destination in self.destinations.items():
@@ -2465,7 +2487,7 @@ impl_definition block
                     block.input = "dump " + name + "\n"
                     warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.")
                     printer.write("\n")
-                    printer.print_block(block)
+                    printer.print_block(block, clinic=self)
                     continue
 
                 if destination.type == 'file':
@@ -2490,7 +2512,7 @@ impl_definition block
 
                     block.input = 'preserve\n'
                     printer_2 = BlockPrinter(self.language)
-                    printer_2.print_block(block, core_includes=True)
+                    printer_2.print_block(block, core_includes=True, clinic=self)
                     write_file(destination.filename, printer_2.f.getvalue())
                     continue
 
@@ -2536,9 +2558,15 @@ impl_definition block
 def parse_file(
         filename: str,
         *,
-        verify: bool = True,
-        output: str | None = None
+        ns: argparse.Namespace,
+        output: str | None = None,
 ) -> None:
+    verify = not ns.force
+    limited_capi = ns.limited_capi
+    # XXX Temporary solution
+    if os.path.basename(filename) == '_testclinic_limited.c':
+        print(f"{filename} uses limited C API")
+        limited_capi = True
     if not output:
         output = filename
 
@@ -2560,7 +2588,10 @@ def parse_file(
         return
 
     assert isinstance(language, CLanguage)
-    clinic = Clinic(language, verify=verify, filename=filename)
+    clinic = Clinic(language,
+                    verify=verify,
+                    filename=filename,
+                    limited_capi=limited_capi)
     cooked = clinic.parse(raw)
 
     write_file(output, cooked)
@@ -5987,6 +6018,8 @@ For more information see https://docs.python.org/3/howto/clinic.html""")
     cmdline.add_argument("--exclude", type=str, action="append",
                          help=("a file to exclude in --make mode; "
                                "can be given multiple times"))
+    cmdline.add_argument("--limited", dest="limited_capi", action='store_true',
+                         help="use the Limited C API")
     cmdline.add_argument("filename", metavar="FILE", type=str, nargs="*",
                          help="the list of files to process")
     return cmdline
@@ -6077,7 +6110,7 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None:
                     continue
                 if ns.verbose:
                     print(path)
-                parse_file(path, verify=not ns.force)
+                parse_file(path, ns=ns)
         return
 
     if not ns.filename:
@@ -6089,7 +6122,7 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None:
     for filename in ns.filename:
         if ns.verbose:
             print(filename)
-        parse_file(filename, output=ns.output, verify=not ns.force)
+        parse_file(filename, output=ns.output, ns=ns)
 
 
 def main(argv: list[str] | None = None) -> NoReturn: