Add benchmarks for importing just source w/o writing bytecode, importing source

while writing bytecode, and importing bytecode with source existing (don't care
about sourceless imports).

Lib/importlib/test/benchmark.py

@@ -1,14 +1,15 @@
-"""Benchmark some basic import use-cases."""
-# XXX
-#    - from source
-#        + sys.dont_write_bytecode = True
-#        + sys.dont_write_bytecode = False
-#    - from bytecode
-#    - extensions
+"""Benchmark some basic import use-cases.
+
+The assumption is made that this benchmark is run in a fresh interpreter and
+thus has no external changes made to import-related attributes in sys.
+
+"""
 from . import util
 from .source import util as source_util
 import imp
 import importlib
+import os
+import py_compile
 import sys
 import timeit
 
@@ -33,7 +34,7 @@ def bench(name, cleanup=lambda: None, *, seconds=1, repeat=3):
                 count -= 1
         yield count // seconds
 
-def from_cache(repeat):
+def from_cache(seconds, repeat):
     """sys.modules"""
     name = '<benchmark import>'
     module = imp.new_module(name)
@@ -41,32 +42,74 @@ def from_cache(repeat):
     module.__package__ = ''
     with util.uncache(name):
         sys.modules[name] = module
-        for result in bench(name, repeat=repeat):
+        for result in bench(name, repeat=repeat, seconds=seconds):
             yield result
 
 
-def builtin_mod(repeat):
+def builtin_mod(seconds, repeat):
     """Built-in module"""
     name = 'errno'
     if name in sys.modules:
         del sys.modules[name]
     # Relying on built-in importer being implicit.
-    for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat):
+    for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
+                        seconds=seconds):
         yield result
 
 
-def main(import_, *, repeat=3):
+def source_wo_bytecode(seconds, repeat):
+    """Source w/o bytecode"""
+    sys.dont_write_bytecode = True
+    try:
+        name = '__importlib_test_benchmark__'
+        # Clears out sys.modules and puts an entry at the front of sys.path.
+        with source_util.create_modules(name) as mapping:
+            assert not os.path.exists(imp.cache_from_source(mapping[name]))
+            for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
+                                seconds=seconds):
+                yield result
+    finally:
+        sys.dont_write_bytecode = False
+
+
+def source_writing_bytecode(seconds, repeat):
+    """Source writing bytecode"""
+    assert not sys.dont_write_bytecode
+    name = '__importlib_test_benchmark__'
+    with source_util.create_modules(name) as mapping:
+        def cleanup():
+            sys.modules.pop(name)
+            os.unlink(imp.cache_from_source(mapping[name]))
+        for result in bench(name, cleanup, repeat=repeat, seconds=seconds):
+            assert not os.path.exists(imp.cache_from_source(mapping[name]))
+            yield result
+
+
+def source_using_bytecode(seconds, repeat):
+    """Bytecode w/ source"""
+    name = '__importlib_test_benchmark__'
+    with source_util.create_modules(name) as mapping:
+        py_compile.compile(mapping[name])
+        assert os.path.exists(imp.cache_from_source(mapping[name]))
+        for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
+                            seconds=seconds):
+            yield result
+
+
+def main(import_):
     __builtins__.__import__ = import_
-    benchmarks = from_cache, builtin_mod
+    benchmarks = (from_cache, builtin_mod, source_using_bytecode,
+                  source_wo_bytecode, source_writing_bytecode,)
     print("Measuring imports/second\n")
     for benchmark in benchmarks:
         print(benchmark.__doc__, "[", end=' ')
         sys.stdout.flush()
         results = []
-        for result in benchmark(repeat):
+        for result in benchmark(seconds=1, repeat=3):
             results.append(result)
             print(result, end=' ')
             sys.stdout.flush()
+        assert not sys.dont_write_bytecode
         print("]", "best is", max(results))