mirrors/cpython
0
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2024-11-23 09:54:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

bpo-42955: Add sys.modules_names (GH-24238)

Browse Source 
Add sys.module_names, containing the list of the standard library
module names.
This commit is contained in:
Victor Stinner 2021-01-25 13:24:42 +01:00 committed by GitHub
parent 879986d8a9
commit db584bdad3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 305 additions and 275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Doc/library/sys.rst
View File

@ -153,10 +153,12 @@ always available.
.. data:: builtin_module_names
A tuple of strings giving the names of all modules that are compiled into this
A tuple of strings containing the names of all modules that are compiled into this
Python interpreter. (This information is not available in any other way ---
``modules.keys()`` only lists the imported modules.)
See also the :attr:`sys.module_names` list.
.. function:: call_tracing(func, args)
@ -1060,6 +1062,24 @@ always available.
This is still called as a fallback if a :data:`meta_path` entry doesn't
have a :meth:`~importlib.abc.MetaPathFinder.find_spec` method.
.. data:: module_names
A frozenset of strings containing the names of standard library modules.
It is the same on all platforms. Modules which are not available on
some platforms and modules disabled at Python build are also listed.
All module kinds are listed: pure Python, built-in, frozen and extension
modules. Test modules are excluded.
For packages, only sub-packages are listed, not sub-modules. For example,
``concurrent`` package and ``concurrent.futures`` sub-package are listed,
but not ``concurrent.futures.base`` sub-module.
See also the :attr:`sys.builtin_module_names` list.
.. versionchanged:: 3.10
.. data:: modules
This is a dictionary that maps module names to modules which have already been

4
Doc/whatsnew/3.10.rst
View File

@ -396,6 +396,10 @@ Add :data:`sys.orig_argv` attribute: the list of the original command line
arguments passed to the Python executable.
(Contributed by Victor Stinner in :issue:`23427`.)
Add :data:`sys.module_names`, containing the list of the standard library
module names.
(Contributed by Victor Stinner in :issue:`42955`.)
threading
---------

17
Lib/test/test_capi.py
View File

@ -569,12 +569,23 @@ class CAPITest(unittest.TestCase):
self.assertEqual(len(modules), total)
def test_fatal_error(self):
# By default, stdlib extension modules are ignored,
# but not test modules.
expected = ('_testcapi',)
not_expected = ('sys', 'builtins', '_imp', '_thread', '_weakref',
'_io', 'marshal', '_signal', '_abc')
code = 'import _testcapi; _testcapi.fatal_error(b"MESSAGE")'
not_expected = ('sys',)
code = 'import _testcapi, sys; _testcapi.fatal_error(b"MESSAGE")'
self.check_fatal_error(code, expected, not_expected)
# Mark _testcapi as stdlib module, but not sys
expected = ('sys',)
not_expected = ('_testcapi',)
code = textwrap.dedent('''
import _testcapi, sys
sys.module_names = frozenset({"_testcapi"})
_testcapi.fatal_error(b"MESSAGE")
''')
self.check_fatal_error(code, expected)
class TestPendingCalls(unittest.TestCase):

8
Lib/test/test_faulthandler.py
View File

@ -334,8 +334,9 @@ class FaultHandlerTests(unittest.TestCase):
def test_dump_ext_modules(self):
code = """
import faulthandler
# _testcapi is a test module and not considered as a stdlib module
import _testcapi
import sys
# Don't filter stdlib module names
sys.module_names = frozenset()
faulthandler.enable()
faulthandler._sigsegv()
"""
@ -346,7 +347,8 @@ class FaultHandlerTests(unittest.TestCase):
if not match:
self.fail(f"Cannot find 'Extension modules:' in {stderr!r}")
modules = set(match.group(1).strip().split(', '))
self.assertIn('_testcapi', modules)
for name in ('sys', 'faulthandler'):
self.assertIn(name, modules)
def test_is_enabled(self):
orig_stderr = sys.stderr

5
Lib/test/test_sys.py
View File

@ -986,6 +986,11 @@ class SysModuleTest(unittest.TestCase):
self.assertEqual(proc.stdout.rstrip().splitlines(), expected,
proc)
def test_module_names(self):
self.assertIsInstance(sys.module_names, frozenset)
for name in sys.module_names:
self.assertIsInstance(name, str)
@test.support.cpython_only
class UnraisableHookTest(unittest.TestCase):

2
Misc/NEWS.d/next/Library/2021-01-18-11-59-46.bpo-42955.CSWLC9.rst Normal file
View File

@ -0,0 +1,2 @@
Add :data:`sys.module_names`, containing the list of the standard library
module names. Patch by Victor Stinner.

295
Python/module_names.h
View File

@ -1,63 +1,123 @@
// Auto-generated by Tools/scripts/generate_module_names.py.
// List used to create sys.module_names.
static const char* _Py_module_names[] = {
// Built-in modules
"__future__",
"_abc",
"_aix_support",
"_ast",
"_asyncio",
"_bisect",
"_blake2",
"_bootsubprocess",
"_bz2",
"_codecs",
"_codecs_cn",
"_codecs_hk",
"_codecs_iso2022",
"_codecs_jp",
"_codecs_kr",
"_codecs_tw",
"_collections",
"_collections_abc",
"_compat_pickle",
"_compression",
"_contextvars",
"_crypt",
"_csv",
"_ctypes",
"_curses",
"_curses_panel",
"_datetime",
"_dbm",
"_decimal",
"_elementtree",
"_functools",
"_gdbm",
"_hashlib",
"_heapq",
"_imp",
"_io",
"_json",
"_locale",
"_lsprof",
"_lzma",
"_markupbase",
"_md5",
"_msi",
"_multibytecodec",
"_multiprocessing",
"_opcode",
"_operator",
"_osx_support",
"_pickle",
"_posixshmem",
"_posixsubprocess",
"_py_abc",
"_pydecimal",
"_pyio",
"_queue",
"_random",
"_sha1",
"_sha256",
"_sha3",
"_sha512",
"_signal",
"_sitebuiltins",
"_socket",
"_sqlite3",
"_sre",
"_ssl",
"_stat",
"_statistics",
"_string",
"_strptime",
"_struct",
"_symtable",
"_thread",
"_threading_local",
"_tkinter",
"_tracemalloc",
"_uuid",
"_warnings",
"_weakref",
"atexit",
"builtins",
"errno",
"faulthandler",
"gc",
"itertools",
"marshal",
"posix",
"pwd",
"sys",
"time",
// Pure Python modules (Lib/*.py)
"__future__",
"_weakrefset",
"_winapi",
"_xxsubinterpreters",
"_zoneinfo",
"abc",
"aifc",
"antigravity",
"argparse",
"array",
"ast",
"asynchat",
"asyncio",
"asyncore",
"atexit",
"audioop",
"base64",
"bdb",
"binascii",
"binhex",
"bisect",
"builtins",
"bz2",
"cProfile",
"calendar",
"cgi",
"cgitb",
"chunk",
"cmath",
"cmd",
"code",
"codecs",
"codeop",
"collections",
"colorsys",
"compileall",
"concurrent",
"concurrent.futures",
"configparser",
"contextlib",
"contextvars",
@ -65,45 +125,80 @@ static const char* _Py_module_names[] = {
"copyreg",
"crypt",
"csv",
"ctypes",
"ctypes.macholib",
"curses",
"dataclasses",
"datetime",
"dbm",
"decimal",
"difflib",
"dis",
"distutils",
"distutils.command",
"doctest",
"email",
"email.mime",
"encodings",
"ensurepip",
"ensurepip._bundled",
"enum",
"errno",
"faulthandler",
"fcntl",
"filecmp",
"fileinput",
"fnmatch",
"fractions",
"ftplib",
"functools",
"gc",
"genericpath",
"getopt",
"getpass",
"gettext",
"glob",
"graphlib",
"grp",
"gzip",
"hashlib",
"heapq",
"hmac",
"html",
"http",
"idlelib",
"imaplib",
"imghdr",
"imp",
"importlib",
"inspect",
"io",
"ipaddress",
"itertools",
"json",
"keyword",
"lib2to3",
"lib2to3.fixes",
"lib2to3.pgen2",
"linecache",
"locale",
"logging",
"lzma",
"mailbox",
"mailcap",
"marshal",
"math",
"mimetypes",
"mmap",
"modulefinder",
"msilib",
"msvcrt",
"multiprocessing",
"multiprocessing.dummy",
"netrc",
"nis",
"nntplib",
"nt",
"ntpath",
"nturl2path",
"numbers",
@ -111,6 +206,7 @@ static const char* _Py_module_names[] = {
"operator",
"optparse",
"os",
"ossaudiodev",
"pathlib",
"pdb",
"pickle",
@ -120,23 +216,30 @@ static const char* _Py_module_names[] = {
"platform",
"plistlib",
"poplib",
"posix",
"posixpath",
"pprint",
"profile",
"pstats",
"pty",
"pwd",
"py_compile",
"pyclbr",
"pydoc",
"pydoc_data",
"pyexpat",
"queue",
"quopri",
"random",
"re",
"readline",
"reprlib",
"resource",
"rlcompleter",
"runpy",
"sched",
"secrets",
"select",
"selectors",
"shelve",
"shlex",
@ -148,6 +251,8 @@ static const char* _Py_module_names[] = {
"sndhdr",
"socket",
"socketserver",
"spwd",
"sqlite3",
"sre_compile",
"sre_constants",
"sre_parse",
@ -160,15 +265,20 @@ static const char* _Py_module_names[] = {
"subprocess",
"sunau",
"symtable",
"sys",
"sysconfig",
"syslog",
"tabnanny",
"tarfile",
"telnetlib",
"tempfile",
"termios",
"textwrap",
"this",
"threading",
"time",
"timeit",
"tkinter",
"token",
"tokenize",
"trace",
@ -176,161 +286,32 @@ static const char* _Py_module_names[] = {
"tracemalloc",
"tty",
"turtle",
"turtledemo",
"types",
"typing",
"unicodedata",
"unittest",
"urllib",
"uu",
"uuid",
"venv",
"warnings",
"wave",
"weakref",
"webbrowser",
"xdrlib",
"zipapp",
"zipfile",
"zipimport",
// Packages and sub-packages
"asyncio",
"collections",
"concurrent",
"concurrent.futures",
"ctypes",
"ctypes.macholib",
"curses",
"dbm",
"distutils",
"distutils.command",
"email",
"email.mime",
"encodings",
"ensurepip",
"ensurepip._bundled",
"html",
"http",
"idlelib",
"importlib",
"json",
"lib2to3",
"lib2to3.fixes",
"lib2to3.pgen2",
"logging",
"msilib",
"multiprocessing",
"multiprocessing.dummy",
"pydoc_data",
"sqlite3",
"tkinter",
"turtledemo",
"unittest",
"urllib",
"venv",
"winreg",
"winsound",
"wsgiref",
"xdrlib",
"xml",
"xml.dom",
"xml.etree",
"xml.parsers",
"xml.sax",
"xmlrpc",
"zoneinfo",
// Extension modules built by setup.py
"_asyncio",
"_bisect",
"_blake2",
"_bz2",
"_codecs_cn",
"_codecs_hk",
"_codecs_iso2022",
"_codecs_jp",
"_codecs_kr",
"_codecs_tw",
"_contextvars",
"_crypt",
"_csv",
"_ctypes",
"_curses",
"_curses_panel",
"_datetime",
"_dbm",
"_decimal",
"_elementtree",
"_gdbm",
"_hashlib",
"_heapq",
"_json",
"_lsprof",
"_lzma",
"_md5",
"_multibytecodec",
"_multiprocessing",
"_opcode",
"_pickle",
"_posixshmem",
"_posixsubprocess",
"_queue",
"_random",
"_sha1",
"_sha256",
"_sha3",
"_sha512",
"_socket",
"_sqlite3",
"_ssl",
"_statistics",
"_struct",
"_tkinter",
"_uuid",
"_xxsubinterpreters",
"_zoneinfo",
"array",
"audioop",
"binascii",
"cmath",
"fcntl",
"grp",
"math",
"mmap",
"nis",
"ossaudiodev",
"pyexpat",
"readline",
"resource",
"select",
"spwd",
"syslog",
"termios",
"unicodedata",
"zipapp",
"zipfile",
"zipimport",
"zlib",
// Built-in and extension modules built by Modules/Setup
"_abc",
"_codecs",
"_collections",
"_functools",
"_io",
"_locale",
"_operator",
"_signal",
"_sre",
"_stat",
"_symtable",
"_thread",
"_tracemalloc",
"_weakref",
"atexit",
"errno",
"faulthandler",
"itertools",
"posix",
"pwd",
"time",
// Windows extension modules
"_msi",
"_winapi",
"msvcrt",
"nt",
"winreg",
"winsound",
"zoneinfo",
};

59
Python/pylifecycle.c
View File

@ -18,8 +18,6 @@
#include "pycore_sysmodule.h" // _PySys_ClearAuditHooks()
#include "pycore_traceback.h" // _Py_DumpTracebackThreads()
#include "module_names.h" // _Py_module_names
#include <locale.h> // setlocale()
#ifdef HAVE_SIGNAL_H
@ -2499,7 +2497,7 @@ fatal_error_exit(int status)
// Dump the list of extension modules of sys.modules, excluding stdlib modules
// (_Py_module_names), into fd file descriptor.
// (sys.module_names), into fd file descriptor.
//
// This function is called by a signal handler in faulthandler: avoid memory
// allocations and keep the implementation simple. For example, the list is not
@ -2515,10 +2513,31 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp)
return;
}
Py_ssize_t pos;
PyObject *key, *value;
// Avoid PyDict_GetItemString() which calls PyUnicode_FromString(),
// memory cannot be allocated on the heap in a signal handler.
// Iterate on the dict instead.
PyObject *module_names = NULL;
pos = 0;
while (PyDict_Next(interp->sysdict, &pos, &key, &value)) {
if (PyUnicode_Check(key)
&& PyUnicode_CompareWithASCIIString(key, "module_names") == 0) {
module_names = value;
break;
}
}
// If we failed to get sys.module_names or it's not a frozenset,
// don't exclude stdlib modules.
if (module_names != NULL && !PyFrozenSet_Check(module_names)) {
module_names = NULL;
}
// List extensions
int header = 1;
Py_ssize_t count = 0;
Py_ssize_t pos = 0;
PyObject *key, *value;
pos = 0;
while (PyDict_Next(modules, &pos, &key, &value)) {
if (!PyUnicode_Check(key)) {
continue;
@ -2526,22 +2545,26 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp)
if (!_PyModule_IsExtension(value)) {
continue;
}
// Check if it is a stdlib extension module.
// Use the module name from the sys.modules key,
// don't attempt to get the module object name.
const Py_ssize_t names_len = Py_ARRAY_LENGTH(_Py_module_names);
int is_stdlib_mod = 0;
for (Py_ssize_t i=0; i < names_len; i++) {
const char *name = _Py_module_names[i];
if (PyUnicode_CompareWithASCIIString(key, name) == 0) {
is_stdlib_mod = 1;
break;
if (module_names != NULL) {
int is_stdlib_ext = 0;
Py_ssize_t i;
PyObject *item;
Py_hash_t hash;
for (i=0; _PySet_NextEntry(module_names, &i, &item, &hash); ) {
if (PyUnicode_Check(item)
&& PyUnicode_Compare(key, item) == 0)
{
is_stdlib_ext = 1;
break;
}
}
if (is_stdlib_ext) {
// Ignore stdlib extension
continue;
}
}
if (is_stdlib_mod) {
// Ignore stdlib extension module.
continue;
}
if (header) {

64
Python/sysmodule.c
View File

@ -29,6 +29,7 @@ Data members:
#include "frameobject.h" // PyFrame_GetBack()
#include "pydtrace.h"
#include "osdefs.h" // DELIM
#include "module_names.h" // _Py_module_names
#include <locale.h>
#ifdef MS_WINDOWS
@ -2020,33 +2021,63 @@ static PyMethodDef sys_methods[] = {
{NULL, NULL} /* sentinel */
};
static PyObject *
list_builtin_module_names(void)
{
PyObject *list = PyList_New(0);
int i;
if (list == NULL)
if (list == NULL) {
return NULL;
for (i = 0; PyImport_Inittab[i].name != NULL; i++) {
PyObject *name = PyUnicode_FromString(
PyImport_Inittab[i].name);
if (name == NULL)
break;
PyList_Append(list, name);
}
for (Py_ssize_t i = 0; PyImport_Inittab[i].name != NULL; i++) {
PyObject *name = PyUnicode_FromString(PyImport_Inittab[i].name);
if (name == NULL) {
goto error;
}
if (PyList_Append(list, name) < 0) {
Py_DECREF(name);
goto error;
}
Py_DECREF(name);
}
if (PyList_Sort(list) != 0) {
Py_DECREF(list);
list = NULL;
goto error;
}
if (list) {
PyObject *v = PyList_AsTuple(list);
Py_DECREF(list);
list = v;
}
return list;
PyObject *tuple = PyList_AsTuple(list);
Py_DECREF(list);
return tuple;
error:
Py_DECREF(list);
return NULL;
}
static PyObject *
list_module_names(void)
{
Py_ssize_t len = Py_ARRAY_LENGTH(_Py_module_names);
PyObject *names = PyTuple_New(len);
if (names == NULL) {
return NULL;
}
for (Py_ssize_t i = 0; i < len; i++) {
PyObject *name = PyUnicode_FromString(_Py_module_names[i]);
if (name == NULL) {
Py_DECREF(names);
return NULL;
}
PyTuple_SET_ITEM(names, i, name);
}
PyObject *set = PyObject_CallFunction((PyObject *)&PyFrozenSet_Type,
"(O)", names);
Py_DECREF(names);
return set;
}
/* Pre-initialization support for sys.warnoptions and sys._xoptions
*
* Modern internal code paths:
@ -2753,6 +2784,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict)
SET_SYS("hash_info", get_hash_info(tstate));
SET_SYS("maxunicode", PyLong_FromLong(0x10FFFF));
SET_SYS("builtin_module_names", list_builtin_module_names());
SET_SYS("module_names", list_module_names());
#if PY_BIG_ENDIAN
SET_SYS_FROM_STRING("byteorder", "big");
#else

104
Tools/scripts/generate_module_names.py
View File

@ -17,27 +17,6 @@ IGNORE = {
'__pycache__',
'site-packages',
# Helper modules of public modules.
# For example, sysconfig uses _osx_support.
'_aix_support',
'_collections_abc',
'_compat_pickle',
'_compression',
'_markupbase',
'_osx_support',
'_sitebuiltins',
'_strptime',
'_threading_local',
'_weakrefset',
# Used to bootstrap setup.py
'_bootsubprocess',
# pure Python implementation
'_py_abc',
'_pydecimal',
'_pyio',
# test modules
'__phello__.foo',
'_ctypes_test',
@ -69,40 +48,20 @@ WINDOWS_MODULES = (
)
def write_comment(fp, comment):
print(f"// {comment}", file=fp)
def write_modules(fp, names):
for name in sorted(names):
if name in IGNORE:
continue
print(f'"{name}",', file=fp)
print(file=fp)
def list_builtin_modules(fp):
write_comment(fp, "Built-in modules")
write_modules(fp, sys.builtin_module_names)
# Pure Python modules (Lib/*.py)
def list_python_modules(fp):
write_comment(fp, "Pure Python modules (Lib/*.py)")
names = []
def list_python_modules(names):
for filename in os.listdir(STDLIB_PATH):
if not filename.endswith(".py"):
continue
name = filename.removesuffix(".py")
names.append(name)
write_modules(fp, names)
names.add(name)
def _list_sub_packages(path, names, parent=None):
for name in os.listdir(path):
package_path = os.path.join(path, name)
if name in IGNORE:
continue
package_path = os.path.join(path, name)
if not os.path.isdir(package_path):
continue
if not any(package_file.endswith(".py")
@ -114,40 +73,28 @@ def _list_sub_packages(path, names, parent=None):
qualname = name
if qualname in IGNORE:
continue
names.append(qualname)
names.add(qualname)
_list_sub_packages(package_path, names, qualname)
# Packages and sub-packages
def list_packages(fp):
write_comment(fp, "Packages and sub-packages")
names = []
def list_packages(names):
_list_sub_packages(STDLIB_PATH, names)
write_modules(fp, names)
# Windows extensions
def list_windows_extensions(fp):
write_comment(fp, "Windows extension modules")
write_modules(fp, WINDOWS_MODULES)
# Extension modules built by setup.py
def list_setup(fp):
def list_setup_extensions(names):
cmd = [sys.executable, SETUP_PY, "-q", "build", "--list-module-names"]
output = subprocess.check_output(cmd)
output = output.decode("utf8")
names = output.splitlines()
write_comment(fp, "Extension modules built by setup.py")
write_modules(fp, names)
extensions = output.splitlines()
names |= set(extensions)
# Built-in and extension modules built by Modules/Setup
def list_modules_setup(fp):
def list_modules_setup_extensions(names):
assign_var = re.compile("^[A-Z]+=")
names = []
with open(MODULES_SETUP, encoding="utf-8") as modules_fp:
for line in modules_fp:
# Strip comment
@ -165,25 +112,26 @@ def list_modules_setup(fp):
continue
# "errno errnomodule.c" => write "errno"
name = parts[0]
names.append(name)
write_comment(fp, "Built-in and extension modules built by Modules/Setup")
write_modules(fp, names)
names.add(name)
def list_modules(fp):
def list_modules():
names = set(sys.builtin_module_names) | set(WINDOWS_MODULES)
list_modules_setup_extensions(names)
list_setup_extensions(names)
list_packages(names)
list_python_modules(names)
names -= set(IGNORE)
return names
def write_modules(fp, names):
print("// Auto-generated by Tools/scripts/generate_module_names.py.", file=fp)
print("// List used to create sys.module_names.", file=fp)
print(file=fp)
print("static const char* _Py_module_names[] = {", file=fp)
print(file=fp)
list_builtin_modules(fp)
list_python_modules(fp)
list_packages(fp)
list_setup(fp)
list_modules_setup(fp)
list_windows_extensions(fp)
for name in sorted(names):
print(f'"{name}",', file=fp)
print("};", file=fp)
@ -193,7 +141,9 @@ def main():
file=sys.stderr)
sys.exit(1)
list_modules(sys.stdout)
fp = sys.stdout
names = list_modules()
write_modules(fp, names)
if __name__ == "__main__":