gh-114099 - Add iOS framework loading machinery. (GH-116454)

Co-authored-by: Malcolm Smith <smith@chaquo.com>
Co-authored-by: Eric Snow <ericsnowcurrently@gmail.com>

.gitignore

@@ -69,7 +69,7 @@ Lib/test/data/*
 /_bootstrap_python
 /Makefile
 /Makefile.pre
-iOSTestbed.*
+/iOSTestbed.*
 iOS/Frameworks/
 iOS/Resources/Info.plist
 iOS/testbed/build

Doc/library/importlib.rst

@@ -1241,6 +1241,69 @@ find and load modules.
     and how the module's :attr:`__file__` is populated.
 
 
+.. class:: AppleFrameworkLoader(name, path)
+
+   A specialization of :class:`importlib.machinery.ExtensionFileLoader` that
+   is able to load extension modules in Framework format.
+
+   For compatibility with the iOS App Store, *all* binary modules in an iOS app
+   must be dynamic libraries, contained in a framework with appropriate
+   metadata, stored in the ``Frameworks`` folder of the packaged app. There can
+   be only a single binary per framework, and there can be no executable binary
+   material outside the Frameworks folder.
+
+   To accomodate this requirement, when running on iOS, extension module
+   binaries are *not* packaged as ``.so`` files on ``sys.path``, but as
+   individual standalone frameworks. To discover those frameworks, this loader
+   is be registered against the ``.fwork`` file extension, with a ``.fwork``
+   file acting as a placeholder in the original location of the binary on
+   ``sys.path``. The ``.fwork`` file contains the path of the actual binary in
+   the ``Frameworks`` folder, relative to the app bundle. To allow for
+   resolving a framework-packaged binary back to the original location, the
+   framework is expected to contain a ``.origin`` file that contains the
+   location of the ``.fwork`` file, relative to the app bundle.
+
+   For example, consider the case of an import ``from foo.bar import _whiz``,
+   where ``_whiz`` is implemented with the binary module
+   ``sources/foo/bar/_whiz.abi3.so``, with ``sources`` being the location
+   registered on ``sys.path``, relative to the application bundle. This module
+   *must* be distributed as
+   ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz`` (creating the framework
+   name from the full import path of the module), with an ``Info.plist`` file
+   in the ``.framework`` directory identifying the binary as a framework. The
+   ``foo.bar._whiz`` module would be represented in the original location with
+   a ``sources/foo/bar/_whiz.abi3.fwork`` marker file, containing the path
+   ``Frameworks/foo.bar._whiz/foo.bar._whiz``. The framework would also contain
+   ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz.origin``, containing the
+   path to the ``.fwork`` file.
+
+   When a module is loaded with this loader, the ``__file__`` for the module
+   will report as the location of the ``.fwork`` file. This allows code to use
+   the ``__file__`` of a  module as an anchor for file system traveral.
+   However, the spec origin will reference the location of the *actual* binary
+   in the ``.framework`` folder.
+
+   The Xcode project building the app is responsible for converting any ``.so``
+   files from wherever they exist in the ``PYTHONPATH`` into frameworks in the
+   ``Frameworks`` folder (including stripping extensions from the module file,
+   the addition of framework metadata, and signing the resulting framework),
+   and creating the ``.fwork`` and ``.origin`` files. This will usually be done
+   with a build step in the Xcode project; see the iOS documentation for
+   details on how to construct this build step.
+
+   .. versionadded:: 3.13
+
+   .. availability:: iOS.
+
+   .. attribute:: name
+
+      Name of the module the loader supports.
+
+   .. attribute:: path
+
+      Path to the ``.fwork`` file for the extension module.
+
+
 :mod:`importlib.util` -- Utility code for importers
 ---------------------------------------------------
 

Doc/tools/extensions/pyspecific.py

@@ -133,7 +133,7 @@ class Availability(SphinxDirective):
     known_platforms = frozenset({
         "AIX", "Android", "BSD", "DragonFlyBSD", "Emscripten", "FreeBSD",
         "GNU/kFreeBSD", "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris",
-        "Unix", "VxWorks", "WASI", "Windows", "macOS",
+        "Unix", "VxWorks", "WASI", "Windows", "macOS", "iOS",
         # libc
         "BSD libc", "glibc", "musl",
         # POSIX platforms with pthreads

Lib/ctypes/__init__.py

@@ -348,6 +348,17 @@ class CDLL(object):
                  winmode=None):
         if name:
             name = _os.fspath(name)
+
+            # If the filename that has been provided is an iOS/tvOS/watchOS
+            # .fwork file, dereference the location to the true origin of the
+            # binary.
+            if name.endswith(".fwork"):
+                with open(name) as f:
+                    name = _os.path.join(
+                        _os.path.dirname(_sys.executable),
+                        f.read().strip()
+                    )
+
         self._name = name
         flags = self._func_flags_
         if use_errno:

Lib/ctypes/util.py

@@ -67,7 +67,7 @@ if os.name == "nt":
                 return fname
         return None
 
-elif os.name == "posix" and sys.platform == "darwin":
+elif os.name == "posix" and sys.platform in {"darwin", "ios", "tvos", "watchos"}:
     from ctypes.macholib.dyld import dyld_find as _dyld_find
     def find_library(name):
         possible = ['lib%s.dylib' % name,

Lib/importlib/_bootstrap_external.py

@@ -52,7 +52,7 @@ _pathseps_with_colon = {f':{s}' for s in path_separators}
 
 # Bootstrap-related code ######################################################
 _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win',
-_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin'
+_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin', 'ios', 'tvos', 'watchos'
 _CASE_INSENSITIVE_PLATFORMS =  (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY
                                 + _CASE_INSENSITIVE_PLATFORMS_STR_KEY)
 
@@ -1714,6 +1714,46 @@ class FileFinder:
         return f'FileFinder({self.path!r})'
 
 
+class AppleFrameworkLoader(ExtensionFileLoader):
+    """A loader for modules that have been packaged as frameworks for
+    compatibility with Apple's iOS App Store policies.
+    """
+    def create_module(self, spec):
+        # If the ModuleSpec has been created by the FileFinder, it will have
+        # been created with an origin pointing to the .fwork file. We need to
+        # redirect this to the location in the Frameworks folder, using the
+        # content of the .fwork file.
+        if spec.origin.endswith(".fwork"):
+            with _io.FileIO(spec.origin, 'r') as file:
+                framework_binary = file.read().decode().strip()
+            bundle_path = _path_split(sys.executable)[0]
+            spec.origin = _path_join(bundle_path, framework_binary)
+
+        # If the loader is created based on the spec for a loaded module, the
+        # path will be pointing at the Framework location. If this occurs,
+        # get the original .fwork location to use as the module's __file__.
+        if self.path.endswith(".fwork"):
+            path = self.path
+        else:
+            with _io.FileIO(self.path + ".origin", 'r') as file:
+                origin = file.read().decode().strip()
+                bundle_path = _path_split(sys.executable)[0]
+                path = _path_join(bundle_path, origin)
+
+        module = _bootstrap._call_with_frames_removed(_imp.create_dynamic, spec)
+
+        _bootstrap._verbose_message(
+            "Apple framework extension module {!r} loaded from {!r} (path {!r})",
+            spec.name,
+            spec.origin,
+            path,
+        )
+
+        # Ensure that the __file__ points at the .fwork location
+        module.__file__ = path
+
+        return module
+
 # Import setup ###############################################################
 
 def _fix_up_module(ns, name, pathname, cpathname=None):
@@ -1746,10 +1786,17 @@ def _get_supported_file_loaders():
 
     Each item is a tuple (loader, suffixes).
     """
-    extensions = ExtensionFileLoader, _imp.extension_suffixes()
+    if sys.platform in {"ios", "tvos", "watchos"}:
+        extension_loaders = [(AppleFrameworkLoader, [
+            suffix.replace(".so", ".fwork")
+            for suffix in _imp.extension_suffixes()
+        ])]
+    else:
+        extension_loaders = []
+    extension_loaders.append((ExtensionFileLoader, _imp.extension_suffixes()))
     source = SourceFileLoader, SOURCE_SUFFIXES
     bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
-    return [extensions, source, bytecode]
+    return extension_loaders + [source, bytecode]
 
 
 def _set_bootstrap_module(_bootstrap_module):

Lib/importlib/abc.py

@@ -180,7 +180,11 @@ class ExecutionLoader(InspectLoader):
         else:
             return self.source_to_code(source, path)
 
-_register(ExecutionLoader, machinery.ExtensionFileLoader)
+_register(
+    ExecutionLoader,
+    machinery.ExtensionFileLoader,
+    machinery.AppleFrameworkLoader,
+)
 
 
 class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader):

Lib/importlib/machinery.py

@@ -12,6 +12,7 @@ from ._bootstrap_external import FileFinder
 from ._bootstrap_external import SourceFileLoader
 from ._bootstrap_external import SourcelessFileLoader
 from ._bootstrap_external import ExtensionFileLoader
+from ._bootstrap_external import AppleFrameworkLoader
 from ._bootstrap_external import NamespaceLoader
 
 

Lib/inspect.py

@@ -954,6 +954,10 @@ def getsourcefile(object):
     elif any(filename.endswith(s) for s in
                  importlib.machinery.EXTENSION_SUFFIXES):
         return None
+    elif filename.endswith(".fwork"):
+        # Apple mobile framework markers are another type of non-source file
+        return None
+
     # return a filename found in the linecache even if it doesn't exist on disk
     if filename in linecache.cache:
         return filename
@@ -984,6 +988,7 @@ def getmodule(object, _filename=None):
         return object
     if hasattr(object, '__module__'):
         return sys.modules.get(object.__module__)
+
     # Try the filename to modulename cache
     if _filename is not None and _filename in modulesbyfile:
         return sys.modules.get(modulesbyfile[_filename])
@@ -1119,7 +1124,7 @@ def findsource(object):
         # Allow filenames in form of "<something>" to pass through.
         # `doctest` monkeypatches `linecache` module to enable
         # inspection, so let `linecache.getlines` to be called.
-        if not (file.startswith('<') and file.endswith('>')):
+        if (not (file.startswith('<') and file.endswith('>'))) or file.endswith('.fwork'):
             raise OSError('source code not available')
 
     module = getmodule(object, file)

Lib/modulefinder.py

@@ -72,7 +72,12 @@ def _find_module(name, path=None):
     if isinstance(spec.loader, importlib.machinery.SourceFileLoader):
         kind = _PY_SOURCE
 
-    elif isinstance(spec.loader, importlib.machinery.ExtensionFileLoader):
+    elif isinstance(
+        spec.loader, (
+            importlib.machinery.ExtensionFileLoader,
+            importlib.machinery.AppleFrameworkLoader,
+        )
+    ):
         kind = _C_EXTENSION
 
     elif isinstance(spec.loader, importlib.machinery.SourcelessFileLoader):

Lib/test/test_capi/test_misc.py

@@ -2056,6 +2056,13 @@ class SubinterpreterTest(unittest.TestCase):
         self.addCleanup(os.close, r)
         self.addCleanup(os.close, w)
 
+        # Apple extensions must be distributed as frameworks. This requires
+        # a specialist loader.
+        if support.is_apple_mobile:
+            loader = "AppleFrameworkLoader"
+        else:
+            loader = "ExtensionFileLoader"
+
         script = textwrap.dedent(f"""
             import importlib.machinery
             import importlib.util
@@ -2063,7 +2070,7 @@ class SubinterpreterTest(unittest.TestCase):
 
             fullname = '_test_module_state_shared'
             origin = importlib.util.find_spec('_testmultiphase').origin
-            loader = importlib.machinery.ExtensionFileLoader(fullname, origin)
+            loader = importlib.machinery.{loader}(fullname, origin)
             spec = importlib.util.spec_from_loader(fullname, loader)
             module = importlib.util.module_from_spec(spec)
             attr_id = str(id(module.Error)).encode()
@@ -2371,7 +2378,12 @@ class Test_ModuleStateAccess(unittest.TestCase):
     def setUp(self):
         fullname = '_testmultiphase_meth_state_access'  # XXX
         origin = importlib.util.find_spec('_testmultiphase').origin
-        loader = importlib.machinery.ExtensionFileLoader(fullname, origin)
+        # Apple extensions must be distributed as frameworks. This requires
+        # a specialist loader.
+        if support.is_apple_mobile:
+            loader = importlib.machinery.AppleFrameworkLoader(fullname, origin)
+        else:
+            loader = importlib.machinery.ExtensionFileLoader(fullname, origin)
         spec = importlib.util.spec_from_loader(fullname, loader)
         module = importlib.util.module_from_spec(spec)
         loader.exec_module(module)

Lib/test/test_import/__init__.py

@@ -5,7 +5,11 @@ import json
 import importlib.util
 from importlib._bootstrap_external import _get_sourcefile
 from importlib.machinery import (
-    BuiltinImporter, ExtensionFileLoader, FrozenImporter, SourceFileLoader,
+    AppleFrameworkLoader,
+    BuiltinImporter,
+    ExtensionFileLoader,
+    FrozenImporter,
+    SourceFileLoader,
 )
 import marshal
 import os
@@ -25,7 +29,7 @@ import _imp
 
 from test.support import os_helper
 from test.support import (
-    STDLIB_DIR, swap_attr, swap_item, cpython_only, is_emscripten,
+    STDLIB_DIR, swap_attr, swap_item, cpython_only, is_apple_mobile, is_emscripten,
     is_wasi, run_in_subinterp, run_in_subinterp_with_config, Py_TRACE_REFS)
 from test.support.import_helper import (
     forget, make_legacy_pyc, unlink, unload, ready_to_import,
@@ -66,6 +70,7 @@ def _require_loader(module, loader, skip):
     MODULE_KINDS = {
         BuiltinImporter: 'built-in',
         ExtensionFileLoader: 'extension',
+        AppleFrameworkLoader: 'framework extension',
         FrozenImporter: 'frozen',
         SourceFileLoader: 'pure Python',
     }
@@ -91,7 +96,12 @@ def require_builtin(module, *, skip=False):
     assert module.__spec__.origin == 'built-in', module.__spec__
 
 def require_extension(module, *, skip=False):
-    _require_loader(module, ExtensionFileLoader, skip)
+    # Apple extensions must be distributed as frameworks. This requires
+    # a specialist loader.
+    if is_apple_mobile:
+        _require_loader(module, AppleFrameworkLoader, skip)
+    else:
+        _require_loader(module, ExtensionFileLoader, skip)
 
 def require_frozen(module, *, skip=True):
     module = _require_loader(module, FrozenImporter, skip)
@@ -134,7 +144,8 @@ if _testsinglephase is not None:
         # it to its nominal state.
         sys.modules.pop('_testsinglephase', None)
         _orig._clear_globals()
-        _testinternalcapi.clear_extension('_testsinglephase', _orig.__file__)
+        origin = _orig.__spec__.origin
+        _testinternalcapi.clear_extension('_testsinglephase', origin)
         import _testsinglephase
 
 
@@ -360,7 +371,7 @@ class ImportTests(unittest.TestCase):
             self.assertEqual(cm.exception.path, _testcapi.__file__)
             self.assertRegex(
                 str(cm.exception),
-                r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|pyd)\)"
+                r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|fwork|pyd)\)"
             )
         else:
             self.assertEqual(
@@ -1689,6 +1700,14 @@ class SubinterpImportTests(unittest.TestCase):
             os.set_blocking(r, False)
         return (r, w)
 
+    def create_extension_loader(self, modname, filename):
+        # Apple extensions must be distributed as frameworks. This requires
+        # a specialist loader.
+        if is_apple_mobile:
+            return AppleFrameworkLoader(modname, filename)
+        else:
+            return ExtensionFileLoader(modname, filename)
+
     def import_script(self, name, fd, filename=None, check_override=None):
         override_text = ''
         if check_override is not None:
@@ -1697,12 +1716,19 @@ class SubinterpImportTests(unittest.TestCase):
                 _imp._override_multi_interp_extensions_check({check_override})
                 '''
         if filename:
+            # Apple extensions must be distributed as frameworks. This requires
+            # a specialist loader.
+            if is_apple_mobile:
+                loader = "AppleFrameworkLoader"
+            else:
+                loader = "ExtensionFileLoader"
+
             return textwrap.dedent(f'''
                 from importlib.util import spec_from_loader, module_from_spec
-                from importlib.machinery import ExtensionFileLoader
+                from importlib.machinery import {loader}
                 import os, sys
                 {override_text}
-                loader = ExtensionFileLoader({name!r}, {filename!r})
+                loader = {loader}({name!r}, {filename!r})
                 spec = spec_from_loader({name!r}, loader)
                 try:
                     module = module_from_spec(spec)
@@ -1883,7 +1909,7 @@ class SubinterpImportTests(unittest.TestCase):
     def test_multi_init_extension_non_isolated_compat(self):
         modname = '_test_non_isolated'
         filename = _testmultiphase.__file__
-        loader = ExtensionFileLoader(modname, filename)
+        loader = self.create_extension_loader(modname, filename)
         spec = importlib.util.spec_from_loader(modname, loader)
         module = importlib.util.module_from_spec(spec)
         loader.exec_module(module)
@@ -1901,7 +1927,7 @@ class SubinterpImportTests(unittest.TestCase):
     def test_multi_init_extension_per_interpreter_gil_compat(self):
         modname = '_test_shared_gil_only'
         filename = _testmultiphase.__file__
-        loader = ExtensionFileLoader(modname, filename)
+        loader = self.create_extension_loader(modname, filename)
         spec = importlib.util.spec_from_loader(modname, loader)
         module = importlib.util.module_from_spec(spec)
         loader.exec_module(module)
@@ -2034,10 +2060,25 @@ class SinglephaseInitTests(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         spec = importlib.util.find_spec(cls.NAME)
-        from importlib.machinery import ExtensionFileLoader
-        cls.FILE = spec.origin
         cls.LOADER = type(spec.loader)
-        assert cls.LOADER is ExtensionFileLoader
+
+        # Apple extensions must be distributed as frameworks. This requires
+        # a specialist loader, and we need to differentiate between the
+        # spec.origin and the original file location.
+        if is_apple_mobile:
+            assert cls.LOADER is AppleFrameworkLoader
+
+            cls.ORIGIN = spec.origin
+            with open(spec.origin + ".origin", "r") as f:
+                cls.FILE = os.path.join(
+                    os.path.dirname(sys.executable),
+                    f.read().strip()
+                )
+        else:
+            assert cls.LOADER is ExtensionFileLoader
+
+            cls.ORIGIN = spec.origin
+            cls.FILE = spec.origin
 
         # Start fresh.
         cls.clean_up()
@@ -2053,14 +2094,15 @@ class SinglephaseInitTests(unittest.TestCase):
     @classmethod
     def clean_up(cls):
         name = cls.NAME
-        filename = cls.FILE
         if name in sys.modules:
             if hasattr(sys.modules[name], '_clear_globals'):
-                assert sys.modules[name].__file__ == filename
+                assert sys.modules[name].__file__ == cls.FILE, \
+                    f"{sys.modules[name].__file__} != {cls.FILE}"
+
                 sys.modules[name]._clear_globals()
             del sys.modules[name]
         # Clear all internally cached data for the extension.
-        _testinternalcapi.clear_extension(name, filename)
+        _testinternalcapi.clear_extension(name, cls.ORIGIN)
 
     #########################
     # helpers
@@ -2068,7 +2110,7 @@ class SinglephaseInitTests(unittest.TestCase):
     def add_module_cleanup(self, name):
         def clean_up():
             # Clear all internally cached data for the extension.
-            _testinternalcapi.clear_extension(name, self.FILE)
+            _testinternalcapi.clear_extension(name, self.ORIGIN)
         self.addCleanup(clean_up)
 
     def _load_dynamic(self, name, path):
@@ -2091,7 +2133,7 @@ class SinglephaseInitTests(unittest.TestCase):
         except AttributeError:
             already_loaded = self.already_loaded = {}
         assert name not in already_loaded
-        mod = self._load_dynamic(name, self.FILE)
+        mod = self._load_dynamic(name, self.ORIGIN)
         self.assertNotIn(mod, already_loaded.values())
         already_loaded[name] = mod
         return types.SimpleNamespace(
@@ -2103,7 +2145,7 @@ class SinglephaseInitTests(unittest.TestCase):
     def re_load(self, name, mod):
         assert sys.modules[name] is mod
         assert mod.__dict__ == mod.__dict__
-        reloaded = self._load_dynamic(name, self.FILE)
+        reloaded = self._load_dynamic(name, self.ORIGIN)
         return types.SimpleNamespace(
             name=name,
             module=reloaded,
@@ -2129,7 +2171,7 @@ class SinglephaseInitTests(unittest.TestCase):
                 name = {self.NAME!r}
                 if name in sys.modules:
                     sys.modules.pop(name)._clear_globals()
-                _testinternalcapi.clear_extension(name, {self.FILE!r})
+                _testinternalcapi.clear_extension(name, {self.ORIGIN!r})
                 '''))
             _interpreters.destroy(interpid)
         self.addCleanup(clean_up)
@@ -2146,7 +2188,7 @@ class SinglephaseInitTests(unittest.TestCase):
             postcleanup = f'''
                 {import_}
                 mod._clear_globals()
-                _testinternalcapi.clear_extension(name, {self.FILE!r})
+                _testinternalcapi.clear_extension(name, {self.ORIGIN!r})
                 '''
 
         try:
@@ -2184,7 +2226,7 @@ class SinglephaseInitTests(unittest.TestCase):
         # mod.__name__  might not match, but the spec will.
         self.assertEqual(mod.__spec__.name, loaded.name)
         self.assertEqual(mod.__file__, self.FILE)
-        self.assertEqual(mod.__spec__.origin, self.FILE)
+        self.assertEqual(mod.__spec__.origin, self.ORIGIN)
         if not isolated:
             self.assertTrue(issubclass(mod.error, Exception))
         self.assertEqual(mod.int_const, 1969)
@@ -2578,7 +2620,7 @@ class SinglephaseInitTests(unittest.TestCase):
         # First, load in the main interpreter but then completely clear it.
         loaded_main = self.load(self.NAME)
         loaded_main.module._clear_globals()
-        _testinternalcapi.clear_extension(self.NAME, self.FILE)
+        _testinternalcapi.clear_extension(self.NAME, self.ORIGIN)
 
         # At this point:
         #  * alive in 0 interpreters

Lib/test/test_importlib/extension/test_finder.py

@@ -1,3 +1,4 @@
+from test.support import is_apple_mobile
 from test.test_importlib import abc, util
 
 machinery = util.import_importlib('importlib.machinery')
@@ -19,9 +20,27 @@ class FinderTests(abc.FinderTests):
             )
 
     def find_spec(self, fullname):
-        importer = self.machinery.FileFinder(util.EXTENSIONS.path,
+        if is_apple_mobile:
-                                            (self.machinery.ExtensionFileLoader,
+            # Apple mobile platforms require a specialist loader that uses
-                                             self.machinery.EXTENSION_SUFFIXES))
+            # .fwork files as placeholders for the true `.so` files.
+            loaders = [
+                (
+                    self.machinery.AppleFrameworkLoader,
+                    [
+                        ext.replace(".so", ".fwork")
+                        for ext in self.machinery.EXTENSION_SUFFIXES
+                    ]
+                )
+            ]
+        else:
+            loaders = [
+                (
+                    self.machinery.ExtensionFileLoader,
+                    self.machinery.EXTENSION_SUFFIXES
+                )
+            ]
+
+        importer = self.machinery.FileFinder(util.EXTENSIONS.path, *loaders)
 
         return importer.find_spec(fullname)
 

Lib/test/test_importlib/extension/test_loader.py

@@ -1,3 +1,4 @@
+from test.support import is_apple_mobile
 from test.test_importlib import abc, util
 
 machinery = util.import_importlib('importlib.machinery')
@@ -23,8 +24,15 @@ class LoaderTests:
             raise unittest.SkipTest(
                 f"{util.EXTENSIONS.name} is a builtin module"
             )
-        self.loader = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name,
+
-                                                         util.EXTENSIONS.file_path)
+        # Apple extensions must be distributed as frameworks. This requires
+        # a specialist loader.
+        if is_apple_mobile:
+            self.LoaderClass = self.machinery.AppleFrameworkLoader
+        else:
+            self.LoaderClass = self.machinery.ExtensionFileLoader
+
+        self.loader = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path)
 
     def load_module(self, fullname):
         with warnings.catch_warnings():
@@ -32,13 +40,11 @@ class LoaderTests:
             return self.loader.load_module(fullname)
 
     def test_equality(self):
-        other = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name,
+        other = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path)
-                                                   util.EXTENSIONS.file_path)
         self.assertEqual(self.loader, other)
 
     def test_inequality(self):
-        other = self.machinery.ExtensionFileLoader('_' + util.EXTENSIONS.name,
+        other = self.LoaderClass('_' + util.EXTENSIONS.name, util.EXTENSIONS.file_path)
-                                                   util.EXTENSIONS.file_path)
         self.assertNotEqual(self.loader, other)
 
     def test_load_module_API(self):
@@ -58,8 +64,7 @@ class LoaderTests:
                                 ('__package__', '')]:
                 self.assertEqual(getattr(module, attr), value)
             self.assertIn(util.EXTENSIONS.name, sys.modules)
-            self.assertIsInstance(module.__loader__,
+            self.assertIsInstance(module.__loader__, self.LoaderClass)
-                                  self.machinery.ExtensionFileLoader)
 
     # No extension module as __init__ available for testing.
     test_package = None
@@ -86,7 +91,7 @@ class LoaderTests:
         self.assertFalse(self.loader.is_package(util.EXTENSIONS.name))
         for suffix in self.machinery.EXTENSION_SUFFIXES:
             path = os.path.join('some', 'path', 'pkg', '__init__' + suffix)
-            loader = self.machinery.ExtensionFileLoader('pkg', path)
+            loader = self.LoaderClass('pkg', path)
             self.assertTrue(loader.is_package('pkg'))
 
 
@@ -101,6 +106,14 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests):
     def setUp(self):
         if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS:
             raise unittest.SkipTest("Requires dynamic loading support.")
+
+        # Apple extensions must be distributed as frameworks. This requires
+        # a specialist loader.
+        if is_apple_mobile:
+            self.LoaderClass = self.machinery.AppleFrameworkLoader
+        else:
+            self.LoaderClass = self.machinery.ExtensionFileLoader
+
         self.name = '_testsinglephase'
         if self.name in sys.builtin_module_names:
             raise unittest.SkipTest(
@@ -109,8 +122,8 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests):
         finder = self.machinery.FileFinder(None)
         self.spec = importlib.util.find_spec(self.name)
         assert self.spec
-        self.loader = self.machinery.ExtensionFileLoader(
+
-            self.name, self.spec.origin)
+        self.loader = self.LoaderClass(self.name, self.spec.origin)
 
     def load_module(self):
         with warnings.catch_warnings():
@@ -120,7 +133,7 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests):
     def load_module_by_name(self, fullname):
         # Load a module from the test extension by name.
         origin = self.spec.origin
-        loader = self.machinery.ExtensionFileLoader(fullname, origin)
+        loader = self.LoaderClass(fullname, origin)
         spec = importlib.util.spec_from_loader(fullname, loader)
         module = importlib.util.module_from_spec(spec)
         loader.exec_module(module)
@@ -137,8 +150,7 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests):
             with self.assertRaises(AttributeError):
                 module.__path__
             self.assertIs(module, sys.modules[self.name])
-            self.assertIsInstance(module.__loader__,
+            self.assertIsInstance(module.__loader__, self.LoaderClass)
-                                  self.machinery.ExtensionFileLoader)
 
     # No extension module as __init__ available for testing.
     test_package = None
@@ -182,6 +194,14 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests):
     def setUp(self):
         if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS:
             raise unittest.SkipTest("Requires dynamic loading support.")
+
+        # Apple extensions must be distributed as frameworks. This requires
+        # a specialist loader.
+        if is_apple_mobile:
+            self.LoaderClass = self.machinery.AppleFrameworkLoader
+        else:
+            self.LoaderClass = self.machinery.ExtensionFileLoader
+
         self.name = '_testmultiphase'
         if self.name in sys.builtin_module_names:
             raise unittest.SkipTest(
@@ -190,8 +210,7 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests):
         finder = self.machinery.FileFinder(None)
         self.spec = importlib.util.find_spec(self.name)
         assert self.spec
-        self.loader = self.machinery.ExtensionFileLoader(
+        self.loader = self.LoaderClass(self.name, self.spec.origin)
-            self.name, self.spec.origin)
 
     def load_module(self):
         # Load the module from the test extension.
@@ -202,7 +221,7 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests):
     def load_module_by_name(self, fullname):
         # Load a module from the test extension by name.
         origin = self.spec.origin
-        loader = self.machinery.ExtensionFileLoader(fullname, origin)
+        loader = self.LoaderClass(fullname, origin)
         spec = importlib.util.spec_from_loader(fullname, loader)
         module = importlib.util.module_from_spec(spec)
         loader.exec_module(module)
@@ -228,8 +247,7 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests):
             with self.assertRaises(AttributeError):
                 module.__path__
             self.assertIs(module, sys.modules[self.name])
-            self.assertIsInstance(module.__loader__,
+            self.assertIsInstance(module.__loader__, self.LoaderClass)
-                                  self.machinery.ExtensionFileLoader)
 
     def test_functionality(self):
         # Test basic functionality of stuff defined in an extension module.

Lib/test/test_importlib/test_util.py

@@ -707,13 +707,20 @@ class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase):
 
     @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module")
     def test_incomplete_multi_phase_init_module(self):
+        # Apple extensions must be distributed as frameworks. This requires
+        # a specialist loader.
+        if support.is_apple_mobile:
+            loader = "AppleFrameworkLoader"
+        else:
+            loader = "ExtensionFileLoader"
+
         prescript = textwrap.dedent(f'''
             from importlib.util import spec_from_loader, module_from_spec
-            from importlib.machinery import ExtensionFileLoader
+            from importlib.machinery import {loader}
 
             name = '_test_shared_gil_only'
             filename = {_testmultiphase.__file__!r}
-            loader = ExtensionFileLoader(name, filename)
+            loader = {loader}(name, filename)
             spec = spec_from_loader(name, loader)
 
             ''')

Lib/test/test_importlib/util.py

@@ -8,6 +8,7 @@ import os
 import os.path
 from test import support
 from test.support import import_helper
+from test.support import is_apple_mobile
 from test.support import os_helper
 import unittest
 import sys
@@ -43,6 +44,11 @@ else:
         global EXTENSIONS
         for path in sys.path:
             for ext in machinery.EXTENSION_SUFFIXES:
+                # Apple mobile platforms mechanically load .so files,
+                # but the findable files are labelled .fwork
+                if is_apple_mobile:
+                    ext = ext.replace(".so", ".fwork")
+
                 filename = EXTENSIONS.name + ext
                 file_path = os.path.join(path, filename)
                 if os.path.exists(file_path):

Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst

@@ -0,0 +1,2 @@
+Added a Loader that can discover extension modules in an iOS-style Frameworks
+folder.

configure

@@ -12740,7 +12740,6 @@ if test -z "$SHLIB_SUFFIX"; then
 		esac
 		;;
 	CYGWIN*)   SHLIB_SUFFIX=.dll;;
-	iOS)   SHLIB_SUFFIX=.dylib;;
 	*)	   SHLIB_SUFFIX=.so;;
 	esac
 fi

configure.ac

@@ -3285,7 +3285,6 @@ if test -z "$SHLIB_SUFFIX"; then
 		esac
 		;;
 	CYGWIN*)   SHLIB_SUFFIX=.dll;;
-	iOS)   SHLIB_SUFFIX=.dylib;;
 	*)	   SHLIB_SUFFIX=.so;;
 	esac
 fi

iOS/Resources/Info.plist.in

@@ -17,7 +17,7 @@
 	<key>CFBundlePackageType</key>
 	<string>FMWK</string>
 	<key>CFBundleShortVersionString</key>
-	<string>%VERSION%</string>
+	<string>@VERSION@</string>
 	<key>CFBundleLongVersionString</key>
 	<string>%VERSION%, (c) 2001-2024 Python Software Foundation.</string>
 	<key>CFBundleSignature</key>

iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj

@@ -273,7 +273,7 @@
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 			shellPath = /bin/sh;
-			shellScript = "set -e\n\ninstall_dylib () {\n    INSTALL_BASE=$1\n    FULL_DYLIB=$2\n\n    # The name of the .dylib file\n    DYLIB=$(basename \"$FULL_DYLIB\")\n    # The name of the .dylib file, relative to the install base\n    RELATIVE_DYLIB=${FULL_DYLIB#$CODESIGNING_FOLDER_PATH/$INSTALL_BASE/}\n    # The full dotted name of the binary module, constructed from the file path.\n    FULL_MODULE_NAME=$(echo $RELATIVE_DYLIB | cut -d \".\" -f 1 | tr \"/\" \".\"); \n    # A bundle identifier; not actually used, but required by Xcode framework packaging\n    FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n    # The name of the framework folder.\n    FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n    # If the framework folder doesn't exist, create it.\n    if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n        echo \"Creating framework for $RELATIVE_DYLIB\" \n        mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n        cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n        plutil -replace CFBundleExecutable -string \"$DYLIB\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n        plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"        \n    fi\n    \n    echo \"Installing binary for $RELATIVE_DYLIB\" \n    mv \"$FULL_DYLIB\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library dylibs...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.dylib\" | while read FULL_DYLIB; do\n    install_dylib python/lib/$PYTHON_VER/lib-dynload \"$FULL_DYLIB\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n";
+			shellScript = "set -e\n\ninstall_dylib () {\n    INSTALL_BASE=$1\n    FULL_EXT=$2\n\n    # The name of the extension file\n    EXT=$(basename \"$FULL_EXT\")\n    # The location of the extension file, relative to the bundle\n    RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n    # The path to the extension file, relative to the install base\n    PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n    # The full dotted name of the extension module, constructed from the file path.\n    FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n    # A bundle identifier; not actually used, but required by Xcode framework packaging\n    FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n    # The name of the framework folder.\n    FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n    # If the framework folder doesn't exist, create it.\n    if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n        echo \"Creating framework for $RELATIVE_EXT\" \n        mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n        cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n        plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n        plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n    fi\n    \n    echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n    mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n    # Create a placeholder .fwork file where the .so was\n    echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n    # Create a back reference to the .so file location in the framework\n    echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\"             \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n    install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n";
 		};
 /* End PBXShellScriptBuildPhase section */
 

iOS/testbed/iOSTestbedTests/iOSTestbedTests.m

@@ -15,7 +15,7 @@
     const char *argv[] = {
         "iOSTestbed", // argv[0] is the process that is running.
         "-uall",  // Enable all resources
-        "-v",  // run in verbose mode so we get test failure information
+        "-W",  // Display test output on failure
         // To run a subset of tests, add the test names below; e.g.,
         // "test_os",
         // "test_sys",