pythongh-99113: A Per-Interpreter GIL! (pythongh-104210)

This is the culmination of PEP 684 (and of my 8-year long multi-core Python project)!

Each subinterpreter may now be created with its own GIL (via Py_NewInterpreterFromConfig()).  If not so configured then the interpreter will share with the main interpreter--the status quo since subinterpreters were added decades ago.  The main interpreter always has its own GIL and subinterpreters from Py_NewInterpreter() will always share with the main interpreter.

Include/internal/pycore_ceval.h

@@ -21,8 +21,7 @@ struct _ceval_runtime_state;
 
 
 extern void _Py_FinishPendingCalls(PyThreadState *tstate);
-extern void _PyEval_InitRuntimeState(struct _ceval_runtime_state *);
-extern void _PyEval_InitState(struct _ceval_state *, PyThread_type_lock);
+extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock);
 extern void _PyEval_FiniState(struct _ceval_state *ceval);
 PyAPI_FUNC(void) _PyEval_SignalReceived(PyInterpreterState *interp);
 PyAPI_FUNC(int) _PyEval_AddPendingCall(

Include/internal/pycore_ceval_state.h

@@ -49,9 +49,6 @@ struct _ceval_runtime_state {
        the main thread of the main interpreter can handle signals: see
        _Py_ThreadCanHandleSignals(). */
     _Py_atomic_int signals_pending;
-
-    /* This is (only) used indirectly through PyInterpreterState.ceval.gil. */
-    struct _gil_runtime_state gil;
 };
 
 #ifdef PY_HAVE_PERF_TRAMPOLINE

Include/internal/pycore_interp.h

@@ -178,6 +178,9 @@ struct _is {
        basis.  Also see _PyRuntimeState regarding the various mutex fields.
        */
 
+    /* The per-interpreter GIL, which might not be used. */
+    struct _gil_runtime_state _gil;
+
     /* the initial PyInterpreterState.threads.head */
     PyThreadState _initial_thread;
 };

Include/internal/pycore_runtime.h

@@ -32,8 +32,6 @@ struct _getargs_runtime_state {
     struct _PyArg_Parser *static_parsers;
 };
 
-/* ceval state */
-
 /* GIL state */
 
 struct _gilstate_runtime_state {

Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst

@@ -0,0 +1,6 @@
+The GIL is now (optionally) per-interpreter.  This is the fundamental change
+for PEP 684.  This is all made possible by virtue of the isolated state of
+each interpreter in the process.  The behavior of the main interpreter
+remains unchanged.  Likewise, interpreters created using
+``Py_NewInterpreter()`` are not affected.  To get an interpreter with its
+own GIL, call ``Py_NewInterpreterFromConfig()``.

Python/ceval_gil.c

@@ -464,17 +464,15 @@ take_gil(PyThreadState *tstate)
 
 void _PyEval_SetSwitchInterval(unsigned long microseconds)
 {
-    /* XXX per-interpreter GIL */
-    PyInterpreterState *interp = _PyInterpreterState_Main();
+    PyInterpreterState *interp = _PyInterpreterState_Get();
     struct _gil_runtime_state *gil = interp->ceval.gil;
     assert(gil != NULL);
     gil->interval = microseconds;
 }
 
 unsigned long _PyEval_GetSwitchInterval(void)
 {
-    /* XXX per-interpreter GIL */
-    PyInterpreterState *interp = _PyInterpreterState_Main();
+    PyInterpreterState *interp = _PyInterpreterState_Get();
     struct _gil_runtime_state *gil = interp->ceval.gil;
     assert(gil != NULL);
     return gil->interval;
@@ -484,7 +482,9 @@ unsigned long _PyEval_GetSwitchInterval(void)
 int
 _PyEval_ThreadsInitialized(void)
 {
-    /* XXX per-interpreter GIL */
+    /* XXX This is only needed for an assert in PyGILState_Ensure(),
+     * which currently does not work with subinterpreters.
+     * Thus we only use the main interpreter. */
     PyInterpreterState *interp = _PyInterpreterState_Main();
     if (interp == NULL) {
         return 0;
@@ -532,27 +532,16 @@ _PyEval_InitGIL(PyThreadState *tstate, int own_gil)
     assert(tstate->interp->ceval.gil == NULL);
     int locked;
     if (!own_gil) {
+        /* The interpreter will share the main interpreter's instead. */
         PyInterpreterState *main_interp = _PyInterpreterState_Main();
         assert(tstate->interp != main_interp);
         struct _gil_runtime_state *gil = main_interp->ceval.gil;
         init_shared_gil(tstate->interp, gil);
         locked = current_thread_holds_gil(gil, tstate);
     }
-    /* XXX per-interpreter GIL */
-    else if (!_Py_IsMainInterpreter(tstate->interp)) {
-        /* Currently, the GIL is shared by all interpreters,
-           and only the main interpreter is responsible to create
-           and destroy it. */
-        struct _gil_runtime_state *main_gil = _PyInterpreterState_Main()->ceval.gil;
-        init_shared_gil(tstate->interp, main_gil);
-        // XXX For now we lie.
-        tstate->interp->ceval.own_gil = 1;
-        locked = current_thread_holds_gil(main_gil, tstate);
-    }
     else {
         PyThread_init_thread();
-        // XXX per-interpreter GIL: switch to interp->_gil.
-        init_own_gil(tstate->interp, &tstate->interp->runtime->ceval.gil);
+        init_own_gil(tstate->interp, &tstate->interp->_gil);
         locked = 0;
     }
     if (!locked) {
@@ -565,32 +554,22 @@ _PyEval_InitGIL(PyThreadState *tstate, int own_gil)
 void
 _PyEval_FiniGIL(PyInterpreterState *interp)
 {
-    if (interp->ceval.gil == NULL) {
+    struct _gil_runtime_state *gil = interp->ceval.gil;
+    if (gil == NULL) {
         /* It was already finalized (or hasn't been initialized yet). */
         assert(!interp->ceval.own_gil);
         return;
     }
     else if (!interp->ceval.own_gil) {
 #ifdef Py_DEBUG
         PyInterpreterState *main_interp = _PyInterpreterState_Main();
-        assert(interp != main_interp);
+        assert(main_interp != NULL && interp != main_interp);
         assert(interp->ceval.gil == main_interp->ceval.gil);
 #endif
         interp->ceval.gil = NULL;
         return;
     }
 
-    /* XXX per-interpreter GIL */
-    struct _gil_runtime_state *gil = &interp->runtime->ceval.gil;
-    if (!_Py_IsMainInterpreter(interp)) {
-        /* Currently, the GIL is shared by all interpreters,
-           and only the main interpreter is responsible to create
-           and destroy it. */
-        assert(interp->ceval.gil == gil);
-        interp->ceval.gil = NULL;
-        return;
-    }
-
     if (!gil_created(gil)) {
         /* First Py_InitializeFromConfig() call: the GIL doesn't exist
            yet: do nothing. */
@@ -974,21 +953,13 @@ Py_MakePendingCalls(void)
     return 0;
 }
 
-/* The interpreter's recursion limit */
-
 void
-_PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval)
+_PyEval_InitState(PyInterpreterState *interp, PyThread_type_lock pending_lock)
 {
-    /* XXX per-interpreter GIL */
-    _gil_initialize(&ceval->gil);
-}
+    _gil_initialize(&interp->_gil);
 
-void
-_PyEval_InitState(struct _ceval_state *ceval, PyThread_type_lock pending_lock)
-{
-    struct _pending_calls *pending = &ceval->pending;
+    struct _pending_calls *pending = &interp->ceval.pending;
     assert(pending->lock == NULL);
-
     pending->lock = pending_lock;
 }
 

Python/pystate.c

@@ -425,8 +425,6 @@ init_runtime(_PyRuntimeState *runtime,
     runtime->open_code_userdata = open_code_userdata;
     runtime->audit_hook_head = audit_hook_head;
 
-    _PyEval_InitRuntimeState(&runtime->ceval);
-
     PyPreConfig_InitPythonConfig(&runtime->preconfig);
 
     PyThread_type_lock *lockptrs[NUMLOCKS] = {
@@ -682,7 +680,7 @@ init_interpreter(PyInterpreterState *interp,
         memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp));
     }
 
-    _PyEval_InitState(&interp->ceval, pending_lock);
+    _PyEval_InitState(interp, pending_lock);
     _PyGC_InitState(&interp->gc);
     PyConfig_InitPythonConfig(&interp->config);
     _PyType_InitCache(interp);