bpo-40280: Address more test failures on Emscripten (GH-31050)

Co-authored-by: Brett Cannon <brett@python.org>
This commit is contained in:
Christian Heimes 2022-02-05 21:52:01 +02:00 committed by GitHub
parent 9d4161a60c
commit 96b344c2f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 227 additions and 49 deletions

View File

@ -1278,6 +1278,8 @@ def reap_children():
# Need os.waitpid(-1, os.WNOHANG): Windows is not supported # Need os.waitpid(-1, os.WNOHANG): Windows is not supported
if not (hasattr(os, 'waitpid') and hasattr(os, 'WNOHANG')): if not (hasattr(os, 'waitpid') and hasattr(os, 'WNOHANG')):
return return
elif not has_subprocess_support:
return
# Reap all our dead child processes so we don't leave zombies around. # Reap all our dead child processes so we don't leave zombies around.
# These hog resources and might be causing some of the buildbots to die. # These hog resources and might be causing some of the buildbots to die.

View File

@ -502,7 +502,7 @@ class FakePath:
def fd_count(): def fd_count():
"""Count the number of open file descriptors. """Count the number of open file descriptors.
""" """
if sys.platform.startswith(('linux', 'freebsd')): if sys.platform.startswith(('linux', 'freebsd', 'emscripten')):
try: try:
names = os.listdir("/proc/self/fd") names = os.listdir("/proc/self/fd")
# Subtract one because listdir() internally opens a file # Subtract one because listdir() internally opens a file

View File

@ -393,6 +393,7 @@ class BuiltinTest(unittest.TestCase):
msg=f"source={source} mode={mode}") msg=f"source={source} mode={mode}")
@unittest.skipIf(support.is_emscripten, "socket.accept is broken")
def test_compile_top_level_await(self): def test_compile_top_level_await(self):
"""Test whether code some top level await can be compiled. """Test whether code some top level await can be compiled.
@ -1213,6 +1214,7 @@ class BuiltinTest(unittest.TestCase):
os.environ.clear() os.environ.clear()
os.environ.update(old_environ) os.environ.update(old_environ)
@support.requires_subprocess()
def test_open_non_inheritable(self): def test_open_non_inheritable(self):
fileobj = open(__file__, encoding="utf-8") fileobj = open(__file__, encoding="utf-8")
with fileobj: with fileobj:

View File

@ -611,6 +611,7 @@ class CAPITest(unittest.TestCase):
self.assertNotIn(name, modules) self.assertNotIn(name, modules)
self.assertEqual(len(modules), total) self.assertEqual(len(modules), total)
@support.requires_subprocess()
def test_fatal_error(self): def test_fatal_error(self):
# By default, stdlib extension modules are ignored, # By default, stdlib extension modules are ignored,
# but not test modules. # but not test modules.
@ -880,6 +881,7 @@ class Test_testinternalcapi(unittest.TestCase):
if name.startswith('test_')) if name.startswith('test_'))
@support.requires_subprocess()
class PyMemDebugTests(unittest.TestCase): class PyMemDebugTests(unittest.TestCase):
PYTHONMALLOC = 'debug' PYTHONMALLOC = 'debug'
# '0x04c06e0' or '04C06E0' # '0x04c06e0' or '04C06E0'

View File

@ -19,6 +19,9 @@ try:
except ImportError: except ImportError:
_testcapi = None _testcapi = None
if not support.has_subprocess_support:
raise unittest.SkipTest("test module requires subprocess")
TIMEOUT = 0.5 TIMEOUT = 0.5
MS_WINDOWS = (os.name == 'nt') MS_WINDOWS = (os.name == 'nt')

View File

@ -9,7 +9,7 @@ from array import array
from weakref import proxy from weakref import proxy
from functools import wraps from functools import wraps
from test.support import cpython_only, swap_attr, gc_collect from test.support import cpython_only, swap_attr, gc_collect, is_emscripten
from test.support.os_helper import (TESTFN, TESTFN_UNICODE, make_bad_fd) from test.support.os_helper import (TESTFN, TESTFN_UNICODE, make_bad_fd)
from test.support.warnings_helper import check_warnings from test.support.warnings_helper import check_warnings
from collections import UserList from collections import UserList
@ -373,7 +373,7 @@ class OtherFileTests:
self.assertEqual(f.isatty(), False) self.assertEqual(f.isatty(), False)
f.close() f.close()
if sys.platform != "win32": if sys.platform != "win32" and not is_emscripten:
try: try:
f = self.FileIO("/dev/tty", "a") f = self.FileIO("/dev/tty", "a")
except OSError: except OSError:

View File

@ -24,14 +24,20 @@ from filecmp import dircmp
from fileinput import FileInput from fileinput import FileInput
from itertools import chain from itertools import chain
from http.cookies import Morsel from http.cookies import Morsel
try:
from multiprocessing.managers import ValueProxy from multiprocessing.managers import ValueProxy
from multiprocessing.pool import ApplyResult from multiprocessing.pool import ApplyResult
from multiprocessing.queues import SimpleQueue as MPSimpleQueue
except ImportError:
# _multiprocessing module is optional
ValueProxy = None
ApplyResult = None
MPSimpleQueue = None
try: try:
from multiprocessing.shared_memory import ShareableList from multiprocessing.shared_memory import ShareableList
except ImportError: except ImportError:
# multiprocessing.shared_memory is not available on e.g. Android # multiprocessing.shared_memory is not available on e.g. Android
ShareableList = None ShareableList = None
from multiprocessing.queues import SimpleQueue as MPSimpleQueue
from os import DirEntry from os import DirEntry
from re import Pattern, Match from re import Pattern, Match
from types import GenericAlias, MappingProxyType, AsyncGeneratorType from types import GenericAlias, MappingProxyType, AsyncGeneratorType
@ -79,13 +85,14 @@ class BaseTest(unittest.TestCase):
Queue, SimpleQueue, Queue, SimpleQueue,
_AssertRaisesContext, _AssertRaisesContext,
SplitResult, ParseResult, SplitResult, ParseResult,
ValueProxy, ApplyResult,
WeakSet, ReferenceType, ref, WeakSet, ReferenceType, ref,
ShareableList, MPSimpleQueue, ShareableList,
Future, _WorkItem, Future, _WorkItem,
Morsel] Morsel]
if ctypes is not None: if ctypes is not None:
generic_types.extend((ctypes.Array, ctypes.LibraryLoader)) generic_types.extend((ctypes.Array, ctypes.LibraryLoader))
if ValueProxy is not None:
generic_types.extend((ValueProxy, ApplyResult, MPSimpleQueue))
def test_subscriptable(self): def test_subscriptable(self):
for t in self.generic_types: for t in self.generic_types:

View File

@ -28,6 +28,9 @@ class GetpassGetuserTest(unittest.TestCase):
getpass.getuser() getpass.getuser()
except ImportError: # in case there's no pwd module except ImportError: # in case there's no pwd module
pass pass
except KeyError:
# current user has no pwd entry
pass
self.assertEqual( self.assertEqual(
environ.get.call_args_list, environ.get.call_args_list,
[mock.call(x) for x in ('LOGNAME', 'USER', 'LNAME', 'USERNAME')]) [mock.call(x) for x in ('LOGNAME', 'USER', 'LNAME', 'USERNAME')])

View File

@ -788,6 +788,7 @@ class TestBuggyCases(GetSourceBase):
self.assertSourceEqual(mod2.cls213, 218, 222) self.assertSourceEqual(mod2.cls213, 218, 222)
self.assertSourceEqual(mod2.cls213().func219(), 220, 221) self.assertSourceEqual(mod2.cls213().func219(), 220, 221)
@unittest.skipIf(support.is_emscripten, "socket.accept is broken")
def test_nested_class_definition_inside_async_function(self): def test_nested_class_definition_inside_async_function(self):
import asyncio import asyncio
self.addCleanup(asyncio.set_event_loop_policy, None) self.addCleanup(asyncio.set_event_loop_policy, None)

View File

@ -5,7 +5,8 @@ from textwrap import dedent
import unittest import unittest
import time import time
import _xxsubinterpreters as _interpreters from test.support import import_helper
_interpreters = import_helper.import_module('_xxsubinterpreters')
from test.support import interpreters from test.support import interpreters

View File

@ -76,6 +76,10 @@ def _default_chunk_size():
with open(__file__, "r", encoding="latin-1") as f: with open(__file__, "r", encoding="latin-1") as f:
return f._CHUNK_SIZE return f._CHUNK_SIZE
requires_alarm = unittest.skipUnless(
hasattr(signal, "alarm"), "test requires signal.alarm()"
)
class MockRawIOWithoutRead: class MockRawIOWithoutRead:
"""A RawIO implementation without read(), so as to exercise the default """A RawIO implementation without read(), so as to exercise the default
@ -4435,12 +4439,15 @@ class SignalsTest(unittest.TestCase):
if e.errno != errno.EBADF: if e.errno != errno.EBADF:
raise raise
@requires_alarm
def test_interrupted_write_unbuffered(self): def test_interrupted_write_unbuffered(self):
self.check_interrupted_write(b"xy", b"xy", mode="wb", buffering=0) self.check_interrupted_write(b"xy", b"xy", mode="wb", buffering=0)
@requires_alarm
def test_interrupted_write_buffered(self): def test_interrupted_write_buffered(self):
self.check_interrupted_write(b"xy", b"xy", mode="wb") self.check_interrupted_write(b"xy", b"xy", mode="wb")
@requires_alarm
def test_interrupted_write_text(self): def test_interrupted_write_text(self):
self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii") self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii")
@ -4472,9 +4479,11 @@ class SignalsTest(unittest.TestCase):
wio.close() wio.close()
os.close(r) os.close(r)
@requires_alarm
def test_reentrant_write_buffered(self): def test_reentrant_write_buffered(self):
self.check_reentrant_write(b"xy", mode="wb") self.check_reentrant_write(b"xy", mode="wb")
@requires_alarm
def test_reentrant_write_text(self): def test_reentrant_write_text(self):
self.check_reentrant_write("xy", mode="w", encoding="ascii") self.check_reentrant_write("xy", mode="w", encoding="ascii")
@ -4502,10 +4511,12 @@ class SignalsTest(unittest.TestCase):
os.close(w) os.close(w)
os.close(r) os.close(r)
@requires_alarm
def test_interrupted_read_retry_buffered(self): def test_interrupted_read_retry_buffered(self):
self.check_interrupted_read_retry(lambda x: x.decode('latin1'), self.check_interrupted_read_retry(lambda x: x.decode('latin1'),
mode="rb") mode="rb")
@requires_alarm
def test_interrupted_read_retry_text(self): def test_interrupted_read_retry_text(self):
self.check_interrupted_read_retry(lambda x: x, self.check_interrupted_read_retry(lambda x: x,
mode="r", encoding="latin1") mode="r", encoding="latin1")
@ -4578,9 +4589,11 @@ class SignalsTest(unittest.TestCase):
if e.errno != errno.EBADF: if e.errno != errno.EBADF:
raise raise
@requires_alarm
def test_interrupted_write_retry_buffered(self): def test_interrupted_write_retry_buffered(self):
self.check_interrupted_write_retry(b"x", mode="wb") self.check_interrupted_write_retry(b"x", mode="wb")
@requires_alarm
def test_interrupted_write_retry_text(self): def test_interrupted_write_retry_text(self):
self.check_interrupted_write_retry("x", mode="w", encoding="latin1") self.check_interrupted_write_retry("x", mode="w", encoding="latin1")

View File

@ -992,6 +992,7 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol):
@unittest.skipUnless(unix_shell and os.path.exists(unix_shell), @unittest.skipUnless(unix_shell and os.path.exists(unix_shell),
'requires a shell') 'requires a shell')
@unittest.skipUnless(hasattr(os, 'popen'), "needs os.popen()") @unittest.skipUnless(hasattr(os, 'popen'), "needs os.popen()")
@support.requires_subprocess()
def test_update2(self): def test_update2(self):
os.environ.clear() os.environ.clear()
os.environ.update(HELLO="World") os.environ.update(HELLO="World")
@ -1002,6 +1003,7 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol):
@unittest.skipUnless(unix_shell and os.path.exists(unix_shell), @unittest.skipUnless(unix_shell and os.path.exists(unix_shell),
'requires a shell') 'requires a shell')
@unittest.skipUnless(hasattr(os, 'popen'), "needs os.popen()") @unittest.skipUnless(hasattr(os, 'popen'), "needs os.popen()")
@support.requires_subprocess()
def test_os_popen_iter(self): def test_os_popen_iter(self):
with os.popen("%s -c 'echo \"line1\nline2\nline3\"'" with os.popen("%s -c 'echo \"line1\nline2\nline3\"'"
% unix_shell) as popen: % unix_shell) as popen:
@ -1173,6 +1175,8 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol):
def _test_underlying_process_env(self, var, expected): def _test_underlying_process_env(self, var, expected):
if not (unix_shell and os.path.exists(unix_shell)): if not (unix_shell and os.path.exists(unix_shell)):
return return
elif not support.has_subprocess_support:
return
with os.popen(f"{unix_shell} -c 'echo ${var}'") as popen: with os.popen(f"{unix_shell} -c 'echo ${var}'") as popen:
value = popen.read().strip() value = popen.read().strip()

View File

@ -184,7 +184,7 @@ class PosixTester(unittest.TestCase):
posix.truncate(os_helper.TESTFN, 0) posix.truncate(os_helper.TESTFN, 0)
@unittest.skipUnless(getattr(os, 'execve', None) in os.supports_fd, "test needs execve() to support the fd parameter") @unittest.skipUnless(getattr(os, 'execve', None) in os.supports_fd, "test needs execve() to support the fd parameter")
@unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") @support.requires_fork()
def test_fexecve(self): def test_fexecve(self):
fp = os.open(sys.executable, os.O_RDONLY) fp = os.open(sys.executable, os.O_RDONLY)
try: try:
@ -199,7 +199,7 @@ class PosixTester(unittest.TestCase):
@unittest.skipUnless(hasattr(posix, 'waitid'), "test needs posix.waitid()") @unittest.skipUnless(hasattr(posix, 'waitid'), "test needs posix.waitid()")
@unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") @support.requires_fork()
def test_waitid(self): def test_waitid(self):
pid = os.fork() pid = os.fork()
if pid == 0: if pid == 0:
@ -209,7 +209,7 @@ class PosixTester(unittest.TestCase):
res = posix.waitid(posix.P_PID, pid, posix.WEXITED) res = posix.waitid(posix.P_PID, pid, posix.WEXITED)
self.assertEqual(pid, res.si_pid) self.assertEqual(pid, res.si_pid)
@unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") @support.requires_fork()
def test_register_at_fork(self): def test_register_at_fork(self):
with self.assertRaises(TypeError, msg="Positional args not allowed"): with self.assertRaises(TypeError, msg="Positional args not allowed"):
os.register_at_fork(lambda: None) os.register_at_fork(lambda: None)
@ -1056,6 +1056,7 @@ class PosixTester(unittest.TestCase):
@unittest.skipUnless(hasattr(os, 'getegid'), "test needs os.getegid()") @unittest.skipUnless(hasattr(os, 'getegid'), "test needs os.getegid()")
@unittest.skipUnless(hasattr(os, 'popen'), "test needs os.popen()") @unittest.skipUnless(hasattr(os, 'popen'), "test needs os.popen()")
@support.requires_subprocess()
def test_getgroups(self): def test_getgroups(self):
with os.popen('id -G 2>/dev/null') as idg: with os.popen('id -G 2>/dev/null') as idg:
groups = idg.read().strip() groups = idg.read().strip()
@ -1481,7 +1482,7 @@ class TestPosixDirFd(unittest.TestCase):
self.addCleanup(posix.unlink, fullname) self.addCleanup(posix.unlink, fullname)
raise raise
@unittest.skipUnless(os.mkfifo in os.supports_dir_fd, "test needs dir_fd support in os.mkfifo()") @unittest.skipUnless(hasattr(os, 'mkfifo') and os.mkfifo in os.supports_dir_fd, "test needs dir_fd support in os.mkfifo()")
def test_mkfifo_dir_fd(self): def test_mkfifo_dir_fd(self):
with self.prepare() as (dir_fd, name, fullname): with self.prepare() as (dir_fd, name, fullname):
try: try:

View File

@ -69,7 +69,7 @@ class PwdTest(unittest.TestCase):
allnames = list(bynames.keys()) allnames = list(bynames.keys())
namei = 0 namei = 0
fakename = allnames[namei] fakename = allnames[namei] if allnames else "invaliduser"
while fakename in bynames: while fakename in bynames:
chars = list(fakename) chars = list(fakename)
for i in range(len(chars)): for i in range(len(chars)):

View File

@ -12,7 +12,7 @@ import traceback
from xml.parsers import expat from xml.parsers import expat
from xml.parsers.expat import errors from xml.parsers.expat import errors
from test.support import sortdict from test.support import sortdict, is_emscripten
class SetAttributeTest(unittest.TestCase): class SetAttributeTest(unittest.TestCase):
@ -466,7 +466,10 @@ class HandlerExceptionTest(unittest.TestCase):
"pyexpat.c", "StartElement") "pyexpat.c", "StartElement")
self.check_traceback_entry(entries[2], self.check_traceback_entry(entries[2],
"test_pyexpat.py", "StartElementHandler") "test_pyexpat.py", "StartElementHandler")
if sysconfig.is_python_build() and not (sys.platform == 'win32' and platform.machine() == 'ARM'): if (sysconfig.is_python_build()
and not (sys.platform == 'win32' and platform.machine() == 'ARM')
and not is_emscripten
):
self.assertIn('call_with_frame("StartElement"', entries[1][3]) self.assertIn('call_with_frame("StartElement"', entries[1][3])

View File

@ -98,6 +98,7 @@ class ResourceTest(unittest.TestCase):
except (OverflowError, ValueError): except (OverflowError, ValueError):
pass pass
@unittest.skipUnless(hasattr(resource, "getrusage"), "needs getrusage")
def test_getrusage(self): def test_getrusage(self):
self.assertRaises(TypeError, resource.getrusage) self.assertRaises(TypeError, resource.getrusage)
self.assertRaises(TypeError, resource.getrusage, 42, 42) self.assertRaises(TypeError, resource.getrusage, 42, 42)

View File

@ -22,7 +22,9 @@ from random import randint, random, randbytes
from test.support import script_helper from test.support import script_helper
from test.support import (findfile, requires_zlib, requires_bz2, from test.support import (findfile, requires_zlib, requires_bz2,
requires_lzma, captured_stdout, requires_subprocess) requires_lzma, captured_stdout, requires_subprocess)
from test.support.os_helper import TESTFN, unlink, rmtree, temp_dir, temp_cwd from test.support.os_helper import (
TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count
)
TESTFN2 = TESTFN + "2" TESTFN2 = TESTFN + "2"
@ -2539,14 +2541,14 @@ class TestsWithMultipleOpens(unittest.TestCase):
def test_many_opens(self): def test_many_opens(self):
# Verify that read() and open() promptly close the file descriptor, # Verify that read() and open() promptly close the file descriptor,
# and don't rely on the garbage collector to free resources. # and don't rely on the garbage collector to free resources.
startcount = fd_count()
self.make_test_archive(TESTFN2) self.make_test_archive(TESTFN2)
with zipfile.ZipFile(TESTFN2, mode="r") as zipf: with zipfile.ZipFile(TESTFN2, mode="r") as zipf:
for x in range(100): for x in range(100):
zipf.read('ones') zipf.read('ones')
with zipf.open('ones') as zopen1: with zipf.open('ones') as zopen1:
pass pass
with open(os.devnull, "rb") as f: self.assertEqual(startcount, fd_count())
self.assertLess(f.fileno(), 100)
def test_write_while_reading(self): def test_write_while_reading(self):
with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_DEFLATED) as zipf: with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_DEFLATED) as zipf:

View File

@ -0,0 +1,9 @@
Fix wasm32-emscripten test failures and platform issues.
- Disable syscalls that are not supported or don't work, e.g.
wait, getrusage, prlimit, mkfifo, mknod, setres[gu]id, setgroups.
- Use fd_count to cound open fds.
- Add more checks for subprocess and fork.
- Add workarounds for missing _multiprocessing and failing socket.accept().
- Enable bzip2.
- Disable large file support.
- Disable signal.alarm.

View File

@ -2,6 +2,8 @@
preserve preserve
[clinic start generated code]*/ [clinic start generated code]*/
#if defined(HAVE_GETRUSAGE)
PyDoc_STRVAR(resource_getrusage__doc__, PyDoc_STRVAR(resource_getrusage__doc__,
"getrusage($module, who, /)\n" "getrusage($module, who, /)\n"
"--\n" "--\n"
@ -29,6 +31,8 @@ exit:
return return_value; return return_value;
} }
#endif /* defined(HAVE_GETRUSAGE) */
PyDoc_STRVAR(resource_getrlimit__doc__, PyDoc_STRVAR(resource_getrlimit__doc__,
"getrlimit($module, resource, /)\n" "getrlimit($module, resource, /)\n"
"--\n" "--\n"
@ -160,7 +164,11 @@ exit:
return return_value; return return_value;
} }
#ifndef RESOURCE_GETRUSAGE_METHODDEF
#define RESOURCE_GETRUSAGE_METHODDEF
#endif /* !defined(RESOURCE_GETRUSAGE_METHODDEF) */
#ifndef RESOURCE_PRLIMIT_METHODDEF #ifndef RESOURCE_PRLIMIT_METHODDEF
#define RESOURCE_PRLIMIT_METHODDEF #define RESOURCE_PRLIMIT_METHODDEF
#endif /* !defined(RESOURCE_PRLIMIT_METHODDEF) */ #endif /* !defined(RESOURCE_PRLIMIT_METHODDEF) */
/*[clinic end generated code: output=ad190fb33d647d1e input=a9049054013a1b77]*/ /*[clinic end generated code: output=9ce1886c129eb2f3 input=a9049054013a1b77]*/

View File

@ -881,7 +881,7 @@ fail:
#define _PyLong_FromDev PyLong_FromLongLong #define _PyLong_FromDev PyLong_FromLongLong
#if defined(HAVE_MKNOD) && defined(HAVE_MAKEDEV) #if (defined(HAVE_MKNOD) && defined(HAVE_MAKEDEV)) || defined(HAVE_DEVICE_MACROS)
static int static int
_Py_Dev_Converter(PyObject *obj, void *p) _Py_Dev_Converter(PyObject *obj, void *p)
{ {
@ -890,7 +890,7 @@ _Py_Dev_Converter(PyObject *obj, void *p)
return 0; return 0;
return 1; return 1;
} }
#endif /* HAVE_MKNOD && HAVE_MAKEDEV */ #endif /* (HAVE_MKNOD && HAVE_MAKEDEV) || HAVE_DEVICE_MACROS */
#ifdef AT_FDCWD #ifdef AT_FDCWD

View File

@ -78,6 +78,7 @@ get_resource_state(PyObject *module)
static struct PyModuleDef resourcemodule; static struct PyModuleDef resourcemodule;
#ifdef HAVE_GETRUSAGE
/*[clinic input] /*[clinic input]
resource.getrusage resource.getrusage
@ -134,6 +135,7 @@ resource_getrusage_impl(PyObject *module, int who)
return result; return result;
} }
#endif
static int static int
py2rlimit(PyObject *limits, struct rlimit *rl_out) py2rlimit(PyObject *limits, struct rlimit *rl_out)

View File

@ -1282,7 +1282,7 @@ _PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info)
#endif #endif
/* getrusage(RUSAGE_SELF) */ /* getrusage(RUSAGE_SELF) */
#if defined(HAVE_SYS_RESOURCE_H) #if defined(HAVE_SYS_RESOURCE_H) && defined(HAVE_GETRUSAGE)
struct rusage ru; struct rusage ru;
if (getrusage(RUSAGE_SELF, &ru) == 0) { if (getrusage(RUSAGE_SELF, &ru) == 0) {

View File

@ -1,7 +1,11 @@
# Python WebAssembly (WASM) build # Python WebAssembly (WASM) build
**WARNING: WASM support is highly experimental! Lots of features are not working yet.**
This directory contains configuration and helpers to facilitate cross This directory contains configuration and helpers to facilitate cross
compilation of CPython to WebAssembly (WASM). compilation of CPython to WebAssembly (WASM). For now we support
*wasm32-emscripten* builds for modern browser and for *Node.js*. It's not
possible to build for *wasm32-wasi* out-of-the-box yet.
## wasm32-emscripten build ## wasm32-emscripten build
@ -22,16 +26,14 @@ popd
### Fetch and build additional emscripten ports ### Fetch and build additional emscripten ports
```shell ```shell
embuilder build zlib embuilder build zlib bzip2
``` ```
### Cross compile to wasm32-emscripten ### Cross compile to wasm32-emscripten for browser
For browser:
```shell ```shell
mkdir -p builddir/emscripten mkdir -p builddir/emscripten-browser
pushd builddir/emscripten pushd builddir/emscripten-browser
CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \ CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
emconfigure ../../configure -C \ emconfigure ../../configure -C \
@ -41,11 +43,27 @@ CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
--with-build-python=$(pwd)/../build/python --with-build-python=$(pwd)/../build/python
emmake make -j$(nproc) emmake make -j$(nproc)
popd
``` ```
For node: Serve `python.html` with a local webserver and open the file in a browser.
```shell
emrun builddir/emscripten-browser/python.html
```
or
```shell
python3 -m http.server
```
### Cross compile to wasm32-emscripten for node
``` ```
mkdir -p builddir/emscripten-node
pushd builddir/emscripten-node
CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \ CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
emconfigure ../../configure -C \ emconfigure ../../configure -C \
--host=wasm32-unknown-emscripten \ --host=wasm32-unknown-emscripten \
@ -54,18 +72,70 @@ CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
--with-build-python=$(pwd)/../build/python --with-build-python=$(pwd)/../build/python
emmake make -j$(nproc) emmake make -j$(nproc)
popd
``` ```
### Test in browser ```
node --experimental-wasm-threads --experimental-wasm-bulk-memory builddir/emscripten-node/python.js
Serve `python.html` with a local webserver and open the file in a browser.
```shell
emrun python.html
``` ```
or ## wasm32-emscripten limitations and issues
```shell - Most stdlib modules with a dependency on external libraries are missing:
python3 -m http.server ``ctypes``, ``readline``, ``sqlite3``, ``ssl``, and more.
``` - Shared extension modules are not implemented yet. All extension modules
are statically linked into the main binary.
- Processes are not supported. System calls like fork, popen, and subprocess
fail with ``ENOSYS`` or ``ENOSUP``.
- Blocking sockets are not available and non-blocking sockets don't work
correctly, e.g. ``socket.accept`` crashes the runtime. ``gethostbyname``
does not resolve to a real IP address. IPv6 is not available.
- The ``select`` module is limited. ``select.select()`` crashes the runtime
due to lack of exectfd support.
- The ``*at`` variants of functions (e.g. ``openat``) are not available.
The ``dir_fd`` argument of *os* module functions can't be used.
- Signal support is limited. ``signal.alarm``, ``itimer``, ``sigaction``
are not available or do not work correctly. ``SIGTERM`` exits the runtime.
- Most user, group, and permission related function and modules are not
supported or don't work as expected, e.g.``pwd`` module, ``grp`` module,
``os.setgroups``, ``os.chown``, and so on.
- Offset and iovec I/O functions (e.g. ``os.pread``, ``os.preadv``) are not
available.
- ``os.mknod`` and ``os.mkfifo``
[don't work](https://github.com/emscripten-core/emscripten/issues/16158)
and are disabled.
- Large file support crashes the runtime and is disabled.
- ``mmap`` module is unstable. flush (``msync``) can crash the runtime.
- Resource-related functions like ``os.nice`` and most functions of the
``resource`` module are not available.
- Some time and datetime features are broken. ``strftime`` and ``strptime``
have known bugs, e.g.
[%% quoting](https://github.com/emscripten-core/emscripten/issues/16155),
[%U off-by-one](https://github.com/emscripten-core/emscripten/issues/16156).
Extended glibc formatting features are not available.
- ``locales`` module is affected by musl libc issues,
[bpo-46390](https://bugs.python.org/issue46390).
- ``uuid`` module is affected by
[memory leak](https://github.com/emscripten-core/emscripten/issues/16081)
and crasher in Emscripten's ``freeaddrinfo``,
- Recursive ``glob`` leaks file descriptors.
- Python's object allocator ``obmalloc`` is disabled by default.
- ``ensurepip`` is not available.
### wasm32-emscripten in browsers
- The bundled stdlib is limited. Network-related modules,
distutils, multiprocessing, dbm, tests and similar modules
are not shipped. All other modules are bundled as pre-compiled
``pyc`` files.
- Threading is not supported.
### wasm32-emscripten in node
Node builds use ``NODERAWFS``, ``USE_PTHREADS`` and ``PROXY_TO_PTHREAD``
linker options.
- Node RawFS allows direct access to the host file system.
- pthread support requires WASM threads and SharedArrayBuffer (bulk memory).
The runtime keeps a pool of web workers around. Each web worker uses
several file descriptors (eventfd, epoll, pipe).

View File

@ -27,9 +27,6 @@ ac_cv_func_prlimit=no
# unsupported syscall, https://github.com/emscripten-core/emscripten/issues/13393 # unsupported syscall, https://github.com/emscripten-core/emscripten/issues/13393
ac_cv_func_shutdown=no ac_cv_func_shutdown=no
# breaks build, see https://github.com/ethanhs/python-wasm/issues/16
ac_cv_lib_bz2_BZ2_bzCompress=no
# clock_nanosleep() causes time.sleep() to sleep forever. # clock_nanosleep() causes time.sleep() to sleep forever.
# nanosleep() works correctly # nanosleep() works correctly
ac_cv_func_clock_nanosleep=no ac_cv_func_clock_nanosleep=no
@ -66,6 +63,11 @@ ac_cv_func_pwritev=no
ac_cv_func_pipe2=no ac_cv_func_pipe2=no
ac_cv_func_nice=no ac_cv_func_nice=no
ac_cv_func_setitimer=no ac_cv_func_setitimer=no
# unsupported syscall: __syscall_prlimit64
ac_cv_func_prlimit=no
# unsupported syscall: __syscall_getrusage
ac_cv_func_getrusage=no
ac_cv_func_posix_fallocate=no
# Syscalls that resulted in a segfault # Syscalls that resulted in a segfault
ac_cv_func_utimensat=no ac_cv_func_utimensat=no
@ -78,6 +80,20 @@ ac_cv_header_sys_ioctl_h=no
ac_cv_func_openpty=no ac_cv_func_openpty=no
ac_cv_func_forkpty=no ac_cv_func_forkpty=no
# mkfifo and mknod are broken, create regular file
ac_cv_func_mkfifo=no
ac_cv_func_mkfifoat=no
ac_cv_func_mknod=no
ac_cv_func_mknodat=no
# always fails with permission error
ac_cv_func_setgroups=no
ac_cv_func_setresuid=no
ac_cv_func_setresgid=no
# alarm signal is not delivered, may need a callback into the event loop?
ac_cv_func_alarm=no
# To use dlopen, you need to use Emscripten's linking support, # To use dlopen, you need to use Emscripten's linking support,
# see https://github.com/emscripten-core/emscripten/wiki/Linking) # see https://emscripten.org/docs/compiling/Dynamic-Linking.html
ac_cv_func_dlopen=no ac_cv_func_dlopen=no

18
configure generated vendored
View File

@ -9844,14 +9844,30 @@ _ACEOF
$as_echo_n "checking whether to enable large file support... " >&6; } $as_echo_n "checking whether to enable large file support... " >&6; }
if test "$ac_cv_sizeof_off_t" -gt "$ac_cv_sizeof_long" -a \ if test "$ac_cv_sizeof_off_t" -gt "$ac_cv_sizeof_long" -a \
"$ac_cv_sizeof_long_long" -ge "$ac_cv_sizeof_off_t"; then "$ac_cv_sizeof_long_long" -ge "$ac_cv_sizeof_off_t"; then
have_largefile_support="yes"
else
have_largefile_support="no"
fi
case $ac_sys_system in #(
Emscripten) :
have_largefile_support="no"
;; #(
*) :
;;
esac
if test "x$have_largefile_support" = xyes; then :
$as_echo "#define HAVE_LARGEFILE_SUPPORT 1" >>confdefs.h $as_echo "#define HAVE_LARGEFILE_SUPPORT 1" >>confdefs.h
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; } $as_echo "yes" >&6; }
else else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; } $as_echo "no" >&6; }
fi fi
# The cast to long int works around a bug in the HP C Compiler # The cast to long int works around a bug in the HP C Compiler
@ -13751,7 +13767,7 @@ for ac_func in \
gai_strerror getegid getentropy geteuid getgid getgrgid getgrgid_r \ gai_strerror getegid getentropy geteuid getgid getgrgid getgrgid_r \
getgrnam_r getgrouplist getgroups getitimer getloadavg getlogin \ getgrnam_r getgrouplist getgroups getitimer getloadavg getlogin \
getpeername getpgid getpid getppid getpriority _getpty \ getpeername getpgid getpid getppid getpriority _getpty \
getpwent getpwnam_r getpwuid_r getresgid getresuid getsid getspent \ getpwent getpwnam_r getpwuid_r getresgid getresuid getrusage getsid getspent \
getspnam getuid getwd if_nameindex initgroups kill killpg lchown linkat \ getspnam getuid getwd if_nameindex initgroups kill killpg lchown linkat \
lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \ lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \
mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \

View File

@ -2554,15 +2554,24 @@ AC_CHECK_SIZEOF(off_t, [], [
AC_MSG_CHECKING(whether to enable large file support) AC_MSG_CHECKING(whether to enable large file support)
if test "$ac_cv_sizeof_off_t" -gt "$ac_cv_sizeof_long" -a \ if test "$ac_cv_sizeof_off_t" -gt "$ac_cv_sizeof_long" -a \
"$ac_cv_sizeof_long_long" -ge "$ac_cv_sizeof_off_t"; then "$ac_cv_sizeof_long_long" -ge "$ac_cv_sizeof_off_t"; then
have_largefile_support="yes"
else
have_largefile_support="no"
fi
dnl LFS does not work with Emscripten 3.1
AS_CASE([$ac_sys_system],
[Emscripten], [have_largefile_support="no"]
)
AS_VAR_IF([have_largefile_support], [yes], [
AC_DEFINE(HAVE_LARGEFILE_SUPPORT, 1, AC_DEFINE(HAVE_LARGEFILE_SUPPORT, 1,
[Defined to enable large file support when an off_t is bigger than a long [Defined to enable large file support when an off_t is bigger than a long
and long long is at least as big as an off_t. You may need and long long is at least as big as an off_t. You may need
to add some flags for configuration and compilation to enable this mode. to add some flags for configuration and compilation to enable this mode.
(For Solaris and Linux, the necessary defines are already defined.)]) (For Solaris and Linux, the necessary defines are already defined.)])
AC_MSG_RESULT(yes) AC_MSG_RESULT(yes)
else ], [
AC_MSG_RESULT(no) AC_MSG_RESULT(no)
fi ])
AC_CHECK_SIZEOF(time_t, [], [ AC_CHECK_SIZEOF(time_t, [], [
#ifdef HAVE_SYS_TYPES_H #ifdef HAVE_SYS_TYPES_H
@ -4144,7 +4153,7 @@ AC_CHECK_FUNCS([ \
gai_strerror getegid getentropy geteuid getgid getgrgid getgrgid_r \ gai_strerror getegid getentropy geteuid getgid getgrgid getgrgid_r \
getgrnam_r getgrouplist getgroups getitimer getloadavg getlogin \ getgrnam_r getgrouplist getgroups getitimer getloadavg getlogin \
getpeername getpgid getpid getppid getpriority _getpty \ getpeername getpgid getpid getppid getpriority _getpty \
getpwent getpwnam_r getpwuid_r getresgid getresuid getsid getspent \ getpwent getpwnam_r getpwuid_r getresgid getresuid getrusage getsid getspent \
getspnam getuid getwd if_nameindex initgroups kill killpg lchown linkat \ getspnam getuid getwd if_nameindex initgroups kill killpg lchown linkat \
lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \ lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \
mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \

View File

@ -537,6 +537,9 @@
/* Define to 1 if you have the `getresuid' function. */ /* Define to 1 if you have the `getresuid' function. */
#undef HAVE_GETRESUID #undef HAVE_GETRESUID
/* Define to 1 if you have the `getrusage' function. */
#undef HAVE_GETRUSAGE
/* Define to 1 if you have the `getsid' function. */ /* Define to 1 if you have the `getsid' function. */
#undef HAVE_GETSID #undef HAVE_GETSID