bpo-40514: Add --with-experimental-isolated-subinterpreters (GH-19926)

Add --with-experimental-isolated-subinterpreters build option to
configure: better isolate subinterpreters, experimental build mode.

When used, force the usage of the libc malloc() memory allocator,
since pymalloc relies on the unique global interpreter lock (GIL).

Misc/NEWS.d/next/Build/2020-05-05-15-39-11.bpo-40514.bZZmuS.rst

@@ -0,0 +1,2 @@
+Add ``--with-experimental-isolated-subinterpreters`` build option to
+``configure``: better isolate subinterpreters, experimental build mode.

Python/preconfig.c

@@ -291,7 +291,17 @@ _PyPreConfig_InitCompatConfig(PyPreConfig *config)
     config->coerce_c_locale_warn = 0;
 
     config->dev_mode = -1;
+#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+    /* bpo-40512: pymalloc is not compatible with subinterpreters,
+       force usage of libc malloc() which is thread-safe. */
+#ifdef Py_DEBUG
+    config->allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG;
+#else
+    config->allocator = PYMEM_ALLOCATOR_MALLOC;
+#endif
+#else
     config->allocator = PYMEM_ALLOCATOR_NOT_SET;
+#endif
 #ifdef MS_WINDOWS
     config->legacy_windows_fs_encoding = -1;
 #endif

configure

@@ -845,6 +845,7 @@ with_computed_gotos
 with_ensurepip
 with_openssl
 with_ssl_default_suites
+with_experimental_isolated_subinterpreters
 '
       ac_precious_vars='build_alias
 host_alias
@@ -1575,6 +1576,9 @@ Optional Packages:
                           leave OpenSSL's defaults untouched, STRING: use a
                           custom string, PROTOCOL_SSLv2 ignores the setting,
                           see Doc/library/ssl.rst
+  --with-experimental-isolated-subinterpreters
+                          better isolate subinterpreters, experimental build
+                          mode (default is no)
 
 Some influential environment variables:
   MACHDEP     name for machine-dependent library files
@@ -17489,6 +17493,30 @@ $as_echo "#define PY_SSL_DEFAULT_CIPHERS 1" >>confdefs.h
 fi
 
 
+# --with-experimental-isolated-subinterpreters
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-experimental-isolated-subinterpreters" >&5
+$as_echo_n "checking for --with-experimental-isolated-subinterpreters... " >&6; }
+
+# Check whether --with-experimental-isolated-subinterpreters was given.
+if test "${with_experimental_isolated_subinterpreters+set}" = set; then :
+  withval=$with_experimental_isolated_subinterpreters;
+if test "$withval" != no
+then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; };
+  $as_echo "#define EXPERIMENTAL_ISOLATED_SUBINTERPRETERS 1" >>confdefs.h
+
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; };
+fi
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
 
 # generate output files
 ac_config_files="$ac_config_files Makefile.pre Misc/python.pc Misc/python-embed.pc Misc/python-config.sh"

configure.ac

@@ -5717,6 +5717,23 @@ AC_MSG_RESULT(python)
 AC_DEFINE(PY_SSL_DEFAULT_CIPHERS, 1)
 ])
 
+# --with-experimental-isolated-subinterpreters
+AH_TEMPLATE(EXPERIMENTAL_ISOLATED_SUBINTERPRETERS,
+            [Better isolate subinterpreters, experimental build mode.])
+AC_MSG_CHECKING(for --with-experimental-isolated-subinterpreters)
+AC_ARG_WITH(experimental-isolated-subinterpreters,
+  AS_HELP_STRING([--with-experimental-isolated-subinterpreters],
+                 [better isolate subinterpreters, experimental build mode (default is no)]),
+[
+if test "$withval" != no
+then
+  AC_MSG_RESULT(yes);
+  AC_DEFINE(EXPERIMENTAL_ISOLATED_SUBINTERPRETERS)
+else
+  AC_MSG_RESULT(no);
+fi],
+[AC_MSG_RESULT(no)])
+
 
 # generate output files
 AC_CONFIG_FILES(Makefile.pre Misc/python.pc Misc/python-embed.pc Misc/python-config.sh)

pyconfig.h.in

@@ -38,6 +38,9 @@
 /* Define if --enable-ipv6 is specified */
 #undef ENABLE_IPV6
 
+/* Better isolate subinterpreters, experimental build mode. */
+#undef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+
 /* Define to 1 if your system stores words within floats with the most
    significant word first */
 #undef FLOAT_WORDS_BIGENDIAN