bpo-41116: Ensure system supplied libraries are found on macOS 11 (GH-23301)

On macOS system provided libraries are in a shared library cache
and not at their usual location. This PR teaches distutils to search
in the SDK, even if there was no "-sysroot" argument in
the compiler flags.

Lib/_osx_support.py

@@ -52,7 +52,7 @@ def _find_executable(executable, path=None):
         return executable
 
 
-def _read_output(commandstring):
+def _read_output(commandstring, capture_stderr=False):
     """Output from successful command execution or None"""
     # Similar to os.popen(commandstring, "r").read(),
     # but without actually using os.popen because that
@@ -67,7 +67,10 @@ def _read_output(commandstring):
             os.getpid(),), "w+b")
 
     with contextlib.closing(fp) as fp:
-        cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
+        if capture_stderr:
+            cmd = "%s >'%s' 2>&1" % (commandstring, fp.name)
+        else:
+            cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
         return fp.read().decode('utf-8').strip() if not os.system(cmd) else None
 
 
@@ -145,6 +148,33 @@ def _save_modified_value(_config_vars, cv, newvalue):
         _config_vars[_INITPRE + cv] = oldvalue
     _config_vars[cv] = newvalue
 
+
+_cache_default_sysroot = None
+def _default_sysroot(cc):
+    """ Returns the root of the default SDK for this system, or '/' """
+    global _cache_default_sysroot
+
+    if _cache_default_sysroot is not None:
+        return _cache_default_sysroot
+   
+    contents = _read_output('%s -c -E -v - </dev/null' % (cc,), True)
+    in_incdirs = False   
+    for line in contents.splitlines():
+        if line.startswith("#include <...>"):
+            in_incdirs = True
+        elif line.startswith("End of search list"):
+            in_incdirs = False
+        elif in_incdirs:
+            line = line.strip()
+            if line == '/usr/include':
+                _cache_default_sysroot = '/'
+            elif line.endswith(".sdk/usr/include"):
+                _cache_default_sysroot = line[:-12]
+    if _cache_default_sysroot is None:
+        _cache_default_sysroot = '/'
+
+    return _cache_default_sysroot
+
 def _supports_universal_builds():
     """Returns True if universal builds are supported on this system"""
     # As an approximation, we assume that if we are running on 10.4 or above,

Lib/distutils/unixccompiler.py

@@ -290,7 +290,7 @@ class UnixCCompiler(CCompiler):
             cflags = sysconfig.get_config_var('CFLAGS')
             m = re.search(r'-isysroot\s*(\S+)', cflags)
             if m is None:
-                sysroot = '/'
+                sysroot = _osx_support._default_sysroot(sysconfig.get_config_var('CC'))
             else:
                 sysroot = m.group(1)
 

Misc/NEWS.d/next/macOS/2020-11-15-16-43-45.bpo-41116.oCkbrF.rst

@@ -0,0 +1 @@
+Ensure distutils.unixxcompiler.find_library_file can find system provided libraries on macOS 11.

setup.py

@@ -9,6 +9,7 @@ import re
 import sys
 import sysconfig
 from glob import glob, escape
+import _osx_support
 
 
 try:
@@ -176,34 +177,10 @@ def macosx_sdk_root():
     m = re.search(r'-isysroot\s*(\S+)', cflags)
     if m is not None:
         MACOS_SDK_ROOT = m.group(1)
-        MACOS_SDK_SPECIFIED = MACOS_SDK_ROOT != '/'
     else:
-        MACOS_SDK_ROOT = '/'
+        MACOS_SDK_ROOT = _osx_support._default_sysroot(
-        MACOS_SDK_SPECIFIED = False
+            sysconfig.get_config_var('CC'))
-        cc = sysconfig.get_config_var('CC')
+    MACOS_SDK_SPECIFIED = MACOS_SDK_ROOT != '/'
-        tmpfile = '/tmp/setup_sdk_root.%d' % os.getpid()
-        try:
-            os.unlink(tmpfile)
-        except:
-            pass
-        ret = run_command('%s -E -v - </dev/null 2>%s 1>/dev/null' % (cc, tmpfile))
-        in_incdirs = False
-        try:
-            if ret == 0:
-                with open(tmpfile) as fp:
-                    for line in fp.readlines():
-                        if line.startswith("#include <...>"):
-                            in_incdirs = True
-                        elif line.startswith("End of search list"):
-                            in_incdirs = False
-                        elif in_incdirs:
-                            line = line.strip()
-                            if line == '/usr/include':
-                                MACOS_SDK_ROOT = '/'
-                            elif line.endswith(".sdk/usr/include"):
-                                MACOS_SDK_ROOT = line[:-12]
-        finally:
-            os.unlink(tmpfile)
 
     return MACOS_SDK_ROOT