gh-132775: Expand the Capability of Interpreter.call() (gh-133484)

It now supports most callables, full args, and return values.

Author: ericsnowcurrently

Include/internal/pycore_crossinterp.h

@@ -317,7 +317,9 @@ typedef enum error_code {
     _PyXI_ERR_ALREADY_RUNNING = -4,
     _PyXI_ERR_MAIN_NS_FAILURE = -5,
     _PyXI_ERR_APPLY_NS_FAILURE = -6,
-    _PyXI_ERR_NOT_SHAREABLE = -7,
+    _PyXI_ERR_PRESERVE_FAILURE = -7,
+    _PyXI_ERR_EXC_PROPAGATION_FAILURE = -8,
+    _PyXI_ERR_NOT_SHAREABLE = -9,
 } _PyXI_errcode;
 
 
@@ -350,16 +352,33 @@ typedef struct xi_session _PyXI_session;
 PyAPI_FUNC(_PyXI_session *) _PyXI_NewSession(void);
 PyAPI_FUNC(void) _PyXI_FreeSession(_PyXI_session *);
 
+typedef struct {
+    PyObject *preserved;
+    PyObject *excinfo;
+    _PyXI_errcode errcode;
+} _PyXI_session_result;
+PyAPI_FUNC(void) _PyXI_ClearResult(_PyXI_session_result *);
+
 PyAPI_FUNC(int) _PyXI_Enter(
     _PyXI_session *session,
     PyInterpreterState *interp,
-    PyObject *nsupdates);
-PyAPI_FUNC(void) _PyXI_Exit(_PyXI_session *session);
-
-PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace(_PyXI_session *);
-
-PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session);
-PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session);
+    PyObject *nsupdates,
+    _PyXI_session_result *);
+PyAPI_FUNC(int) _PyXI_Exit(
+    _PyXI_session *,
+    _PyXI_errcode,
+    _PyXI_session_result *);
+
+PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace(
+    _PyXI_session *,
+    _PyXI_errcode *);
+
+PyAPI_FUNC(int) _PyXI_Preserve(
+    _PyXI_session *,
+    const char *,
+    PyObject *,
+    _PyXI_errcode *);
+PyAPI_FUNC(PyObject *) _PyXI_GetPreserved(_PyXI_session_result *, const char *);
 
 
 /*************/

Lib/test/_code_definitions.py

@@ -57,6 +57,15 @@ def spam_with_globals_and_builtins():
     print(res)
 
 
+def spam_full_args(a, b, /, c, d, *args, e, f, **kwargs):
+    return (a, b, c, d, e, f, args, kwargs)
+
+
+def spam_full_args_with_defaults(a=-1, b=-2, /, c=-3, d=-4, *args,
+                                 e=-5, f=-6, **kwargs):
+    return (a, b, c, d, e, f, args, kwargs)
+
+
 def spam_args_attrs_and_builtins(a, b, /, c, d, *args, e, f, **kwargs):
     if args.__len__() > 2:
         return None
@@ -67,6 +76,10 @@ def spam_returns_arg(x):
     return x
 
 
+def spam_raises():
+    raise Exception('spam!')
+
+
 def spam_with_inner_not_closure():
     def eggs():
         pass
@@ -177,8 +190,11 @@ def ham_C_closure(z):
     spam_minimal,
     spam_with_builtins,
     spam_with_globals_and_builtins,
+    spam_full_args,
+    spam_full_args_with_defaults,
     spam_args_attrs_and_builtins,
     spam_returns_arg,
+    spam_raises,
     spam_with_inner_not_closure,
     spam_with_inner_closure,
     spam_annotated,
@@ -219,8 +235,10 @@ def ham_C_closure(z):
     spam,
     spam_minimal,
     spam_with_builtins,
+    spam_full_args,
     spam_args_attrs_and_builtins,
     spam_returns_arg,
+    spam_raises,
     spam_annotated,
     spam_with_inner_not_closure,
     spam_with_inner_closure,
@@ -238,6 +256,7 @@ def ham_C_closure(z):
 STATELESS_CODE = [
     *STATELESS_FUNCTIONS,
     script_with_globals,
+    spam_full_args_with_defaults,
     spam_with_globals_and_builtins,
     spam_full,
 ]
@@ -248,6 +267,7 @@ def ham_C_closure(z):
     script_with_explicit_empty_return,
     spam_minimal,
     spam_with_builtins,
+    spam_raises,
     spam_with_inner_not_closure,
     spam_with_inner_closure,
 ]

Lib/test/support/interpreters/__init__.py

@@ -226,33 +226,32 @@ def exec(self, code, /):
         if excinfo is not None:
             raise ExecutionFailed(excinfo)
 
-    def call(self, callable, /):
-        """Call the object in the interpreter with given args/kwargs.
+    def _call(self, callable, args, kwargs):
+        res, excinfo = _interpreters.call(self._id, callable, args, kwargs, restrict=True)
+        if excinfo is not None:
+            raise ExecutionFailed(excinfo)
+        return res
 
-        Only functions that take no arguments and have no closure
-        are supported.
+    def call(self, callable, /, *args, **kwargs):
+        """Call the object in the interpreter with given args/kwargs.
 
-        The return value is discarded.
+        Nearly all callables, args, kwargs, and return values are
+        supported.  All "shareable" objects are supported, as are
+        "stateless" functions (meaning non-closures that do not use
+        any globals).  This method will fall back to pickle.
 
         If the callable raises an exception then the error display
-        (including full traceback) is send back between the interpreters
+        (including full traceback) is sent back between the interpreters
         and an ExecutionFailed exception is raised, much like what
         happens with Interpreter.exec().
         """
-        # XXX Support args and kwargs.
-        # XXX Support arbitrary callables.
-        # XXX Support returning the return value (e.g. via pickle).
-        excinfo = _interpreters.call(self._id, callable, restrict=True)
-        if excinfo is not None:
-            raise ExecutionFailed(excinfo)
+        return self._call(callable, args, kwargs)
 
-    def call_in_thread(self, callable, /):
+    def call_in_thread(self, callable, /, *args, **kwargs):
         """Return a new thread that calls the object in the interpreter.
 
         The return value and any raised exception are discarded.
         """
-        def task():
-            self.call(callable)
-        t = threading.Thread(target=task)
+        t = threading.Thread(target=self._call, args=(callable, args, kwargs))
         t.start()
         return t

Lib/test/test_code.py

@@ -701,6 +701,26 @@ def test_local_kinds(self):
                 'checks': CO_FAST_LOCAL,
                 'res': CO_FAST_LOCAL,
             },
+            defs.spam_full_args: {
+                'a': POSONLY,
+                'b': POSONLY,
+                'c': POSORKW,
+                'd': POSORKW,
+                'e': KWONLY,
+                'f': KWONLY,
+                'args': VARARGS,
+                'kwargs': VARKWARGS,
+            },
+            defs.spam_full_args_with_defaults: {
+                'a': POSONLY,
+                'b': POSONLY,
+                'c': POSORKW,
+                'd': POSORKW,
+                'e': KWONLY,
+                'f': KWONLY,
+                'args': VARARGS,
+                'kwargs': VARKWARGS,
+            },
             defs.spam_args_attrs_and_builtins: {
                 'a': POSONLY,
                 'b': POSONLY,
@@ -714,6 +734,7 @@ def test_local_kinds(self):
             defs.spam_returns_arg: {
                 'x': POSORKW,
             },
+            defs.spam_raises: {},
             defs.spam_with_inner_not_closure: {
                 'eggs': CO_FAST_LOCAL,
             },
@@ -934,6 +955,20 @@ def new_var_counts(*,
                 purelocals=5,
                 globalvars=6,
             ),
+            defs.spam_full_args: new_var_counts(
+                posonly=2,
+                posorkw=2,
+                kwonly=2,
+                varargs=1,
+                varkwargs=1,
+            ),
+            defs.spam_full_args_with_defaults: new_var_counts(
+                posonly=2,
+                posorkw=2,
+                kwonly=2,
+                varargs=1,
+                varkwargs=1,
+            ),
             defs.spam_args_attrs_and_builtins: new_var_counts(
                 posonly=2,
                 posorkw=2,
@@ -945,6 +980,9 @@ def new_var_counts(*,
             defs.spam_returns_arg: new_var_counts(
                 posorkw=1,
             ),
+            defs.spam_raises: new_var_counts(
+                globalvars=1,
+            ),
             defs.spam_with_inner_not_closure: new_var_counts(
                 purelocals=1,
             ),
@@ -1097,10 +1135,16 @@ def new_var_counts(*,
     def test_stateless(self):
         self.maxDiff = None
 
+        STATELESS_FUNCTIONS = [
+            *defs.STATELESS_FUNCTIONS,
+            # stateless with defaults
+            defs.spam_full_args_with_defaults,
+        ]
+
         for func in defs.STATELESS_CODE:
             with self.subTest((func, '(code)')):
                 _testinternalcapi.verify_stateless_code(func.__code__)
-        for func in defs.STATELESS_FUNCTIONS:
+        for func in STATELESS_FUNCTIONS:
             with self.subTest((func, '(func)')):
                 _testinternalcapi.verify_stateless_code(func)
 
@@ -1110,7 +1154,7 @@ def test_stateless(self):
                     with self.assertRaises(Exception):
                         _testinternalcapi.verify_stateless_code(func.__code__)
 
-            if func not in defs.STATELESS_FUNCTIONS:
+            if func not in STATELESS_FUNCTIONS:
                 with self.subTest((func, '(func)')):
                     with self.assertRaises(Exception):
                         _testinternalcapi.verify_stateless_code(func)