From ddbeb2f3e02a510c5784ffd74c5e09e8c70b5881 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 18 Jun 2020 14:53:19 +0200 Subject: [PATCH] bpo-38377: Add support.skip_if_broken_multiprocessing_synchronize() (GH-20944) On Linux, skip tests using multiprocessing if the current user cannot create a file in /dev/shm/ directory. Add the skip_if_broken_multiprocessing_synchronize() function to the test.support module. --- Doc/library/test.rst | 8 +++++++ Lib/test/_test_multiprocessing.py | 2 +- Lib/test/support/__init__.py | 22 +++++++++++++++++++ Lib/test/test_asyncio/test_events.py | 4 ++-- Lib/test/test_concurrent_futures.py | 2 +- Lib/test/test_logging.py | 8 +++---- .../test_multiprocessing_main_handling.py | 2 +- Lib/test/test_venv.py | 8 ++++--- .../2020-06-17-18-00-21.bpo-38377.jfg4TH.rst | 4 ++++ 9 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2020-06-17-18-00-21.bpo-38377.jfg4TH.rst diff --git a/Doc/library/test.rst b/Doc/library/test.rst index b39b601fb64..cd05ef07b4a 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -919,6 +919,14 @@ The :mod:`test.support` module defines the following functions: .. versionadded:: 3.6 +.. function:: skip_if_broken_multiprocessing_synchronize() + + Skip tests if the :mod:`multiprocessing.synchronize` module is missing, if + there is no available semaphore implementation, or if creating a lock raises + an :exc:`OSError`. + + .. versionadded:: 3.10 + The :mod:`test.support` module defines the following classes: diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index d01a6680e40..444e234509c 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -34,7 +34,7 @@ from test.support import threading_helper # Skip tests if _multiprocessing wasn't built. _multiprocessing = test.support.import_module('_multiprocessing') # Skip tests if sem_open implementation is broken. -test.support.import_module('multiprocessing.synchronize') +support.skip_if_broken_multiprocessing_synchronize() import threading import multiprocessing.connection diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index da63d9281b1..d9dbdc13008 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1957,3 +1957,25 @@ def wait_process(pid, *, exitcode, timeout=None): # sanity check: it should not fail in practice if pid2 != pid: raise AssertionError(f"pid {pid2} != pid {pid}") + +def skip_if_broken_multiprocessing_synchronize(): + """ + Skip tests if the multiprocessing.synchronize module is missing, if there + is no available semaphore implementation, or if creating a lock raises an + OSError. + """ + + # Skip tests if the _multiprocessing extension is missing. + import_module('_multiprocessing') + + # Skip tests if there is no available semaphore implementation: + # multiprocessing.synchronize requires _multiprocessing.SemLock. + synchronize = import_module('multiprocessing.synchronize') + + try: + # bpo-38377: On Linux, creating a semaphore is the current user + # does not have the permission to create a file in /dev/shm. + # Create a semaphore to check permissions. + synchronize.Lock(ctx=None) + except OSError as exc: + raise unittest.SkipTest(f"broken multiprocessing SemLock: {exc!r}") diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index e7324d2e481..ef9d6fc48e9 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2673,10 +2673,10 @@ class GetEventLoopTestsMixin: if sys.platform != 'win32': def test_get_event_loop_new_process(self): - # Issue bpo-32126: The multiprocessing module used by + # bpo-32126: The multiprocessing module used by # ProcessPoolExecutor is not functional when the # multiprocessing.synchronize module cannot be imported. - support.import_module('multiprocessing.synchronize') + support.skip_if_broken_multiprocessing_synchronize() async def main(): pool = concurrent.futures.ProcessPoolExecutor() diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 0ed75e6098a..7da967ea6ce 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -4,7 +4,7 @@ from test.support import threading_helper # Skip tests if _multiprocessing wasn't built. support.import_module('_multiprocessing') # Skip tests if sem_open implementation is broken. -support.import_module('multiprocessing.synchronize') +support.skip_if_broken_multiprocessing_synchronize() from test.support import hashlib_helper from test.support.script_helper import assert_python_ok diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 275ce2e45f1..e719d264a91 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3630,9 +3630,9 @@ if hasattr(logging.handlers, 'QueueListener'): @patch.object(logging.handlers.QueueListener, 'handle') def test_handle_called_with_mp_queue(self, mock_handle): - # Issue 28668: The multiprocessing (mp) module is not functional + # bpo-28668: The multiprocessing (mp) module is not functional # when the mp.synchronize module cannot be imported. - support.import_module('multiprocessing.synchronize') + support.skip_if_broken_multiprocessing_synchronize() for i in range(self.repeat): log_queue = multiprocessing.Queue() self.setup_and_log(log_queue, '%s_%s' % (self.id(), i)) @@ -3656,9 +3656,9 @@ if hasattr(logging.handlers, 'QueueListener'): indicates that messages were not registered on the queue until _after_ the QueueListener stopped. """ - # Issue 28668: The multiprocessing (mp) module is not functional + # bpo-28668: The multiprocessing (mp) module is not functional # when the mp.synchronize module cannot be imported. - support.import_module('multiprocessing.synchronize') + support.skip_if_broken_multiprocessing_synchronize() for i in range(self.repeat): queue = multiprocessing.Queue() self.setup_and_log(queue, '%s_%s' %(self.id(), i)) diff --git a/Lib/test/test_multiprocessing_main_handling.py b/Lib/test/test_multiprocessing_main_handling.py index b6abfcc7e28..be1ff10e03a 100644 --- a/Lib/test/test_multiprocessing_main_handling.py +++ b/Lib/test/test_multiprocessing_main_handling.py @@ -23,7 +23,7 @@ import multiprocessing AVAILABLE_START_METHODS = set(multiprocessing.get_all_start_methods()) # Issue #22332: Skip tests if sem_open implementation is broken. -support.import_module('multiprocessing.synchronize') +support.skip_if_broken_multiprocessing_synchronize() verbose = support.verbose diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index ef6d7bd5ad7..d3191ed7b99 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -16,7 +16,8 @@ import sys import tempfile from test.support import (captured_stdout, captured_stderr, requires_zlib, can_symlink, EnvironmentVarGuard, rmtree, - import_module) + import_module, + skip_if_broken_multiprocessing_synchronize) import unittest import venv from unittest.mock import patch @@ -357,10 +358,11 @@ class BasicTest(BaseTest): """ Test that the multiprocessing is able to spawn. """ - # Issue bpo-36342: Instantiation of a Pool object imports the + # bpo-36342: Instantiation of a Pool object imports the # multiprocessing.synchronize module. Skip the test if this module # cannot be imported. - import_module('multiprocessing.synchronize') + skip_if_broken_multiprocessing_synchronize() + rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) envpy = os.path.join(os.path.realpath(self.env_dir), diff --git a/Misc/NEWS.d/next/Tests/2020-06-17-18-00-21.bpo-38377.jfg4TH.rst b/Misc/NEWS.d/next/Tests/2020-06-17-18-00-21.bpo-38377.jfg4TH.rst new file mode 100644 index 00000000000..11a30761d36 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2020-06-17-18-00-21.bpo-38377.jfg4TH.rst @@ -0,0 +1,4 @@ +On Linux, skip tests using multiprocessing if the current user cannot create +a file in ``/dev/shm/`` directory. Add the +:func:`~test.support.skip_if_broken_multiprocessing_synchronize` function to +the :mod:`test.support` module.