Issue #13959: Re-implement imp.cache_from_source() in Lib/imp.py.

Lib/imp.py

@@ -10,11 +10,10 @@ from _imp import (lock_held, acquire_lock, release_lock, reload,
                   load_dynamic, get_frozen_object, is_frozen_package,
                   init_builtin, init_frozen, is_builtin, is_frozen,
                   _fix_co_filename)
-# Can (probably) move to importlib
-from _imp import (get_tag, get_suffixes, cache_from_source,
-                  source_from_cache)
 # Could move out of _imp, but not worth the code
 from _imp import get_magic
+# Can (probably) move to importlib
+from _imp import (get_tag, get_suffixes, source_from_cache)
 # Should be re-implemented here (and mostly deprecated)
 from _imp import (find_module, NullImporter,
                   SEARCH_ERROR, PY_SOURCE, PY_COMPILED, C_EXTENSION,
@@ -22,6 +21,7 @@ from _imp import (find_module, NullImporter,
                   PY_CODERESOURCE, IMP_HOOK)
 
 from importlib._bootstrap import _new_module as new_module
+from importlib._bootstrap import _cache_from_source as cache_from_source
 
 from importlib import _bootstrap
 import os

Lib/importlib/_bootstrap.py

@@ -178,6 +178,31 @@ def _new_module(name):
 
 # Finder/loader utility code ##################################################
 
+PYCACHE = '__pycache__'
+
+DEBUG_BYTECODE_SUFFIX = '.pyc'
+OPT_BYTECODE_SUFFIX = '.pyo'
+BYTECODE_SUFFIX = DEBUG_BYTECODE_SUFFIX if __debug__ else OPT_BYTECODE_SUFFIX
+
+def _cache_from_source(path, debug_override=None):
+    """Given the path to a .py file, return the path to its .pyc/.pyo file.
+
+    The .py file does not need to exist; this simply returns the path to the
+    .pyc/.pyo file calculated as if the .py file were imported.  The extension
+    will be .pyc unless __debug__ is not defined, then it will be .pyo.
+
+    If debug_override is not None, then it must be a boolean and is taken as
+    the value of __debug__ instead.
+
+    """
+    debug = __debug__ if debug_override is None else debug_override
+    suffix = DEBUG_BYTECODE_SUFFIX if debug else OPT_BYTECODE_SUFFIX
+    head, tail = _path_split(path)
+    base_filename, sep, _ = tail.partition('.')
+    filename = '{}{}{}{}'.format(base_filename, sep, _imp.get_tag(), suffix)
+    return _path_join(head, PYCACHE, filename)
+
+
 def verbose_message(message, *args):
     """Print the message to stderr if -v/PYTHONVERBOSE is turned on."""
     if sys.flags.verbose:
@@ -452,7 +477,7 @@ class _LoaderBasics:
         code_object = self.get_code(name)
         module.__file__ = self.get_filename(name)
         if not sourceless:
-            module.__cached__ = _imp.cache_from_source(module.__file__)
+            module.__cached__ = _cache_from_source(module.__file__)
         else:
             module.__cached__ = module.__file__
         module.__package__ = name
@@ -515,7 +540,7 @@ class SourceLoader(_LoaderBasics):
 
         """
         source_path = self.get_filename(fullname)
-        bytecode_path = _imp.cache_from_source(source_path)
+        bytecode_path = _cache_from_source(source_path)
         source_mtime = None
         if bytecode_path is not None:
             try:
@@ -554,9 +579,6 @@ class SourceLoader(_LoaderBasics):
         verbose_message('code object from {}', source_path)
         if (not sys.dont_write_bytecode and bytecode_path is not None and
             source_mtime is not None):
-            # If e.g. Jython ever implements imp.cache_from_source to have
-            # their own cached file format, this block of code will most likely
-            # throw an exception.
             data = bytearray(_MAGIC_NUMBER)
             data.extend(_w_long(source_mtime))
             data.extend(_w_long(len(source_bytes)))

Python/import.c

@@ -783,7 +783,6 @@ remove_module(PyObject *name)
 
 static PyObject * get_sourcefile(PyObject *filename);
 static PyObject *make_source_pathname(PyObject *pathname);
-static PyObject* make_compiled_pathname(PyObject *pathname, int debug);
 
 /* Execute a code object in a module and return the module object
  * WITH INCREMENTED REFERENCE COUNT.  If an error occurs, name is
@@ -924,71 +923,6 @@ rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end)
     return found;
 }
 
-/* Given a pathname for a Python source file, fill a buffer with the
-   pathname for the corresponding compiled file.  Return the pathname
-   for the compiled file, or NULL if there's no space in the buffer.
-   Doesn't set an exception.
-
-   foo.py -> __pycache__/foo.<tag>.pyc
-
-   pathstr is assumed to be "ready".
-*/
-
-static PyObject*
-make_compiled_pathname(PyObject *pathstr, int debug)
-{
-    PyObject *result;
-    Py_ssize_t fname, ext, len, i, pos, taglen;
-    Py_ssize_t pycache_len = sizeof(CACHEDIR) - 1;
-    int kind;
-    void *data;
-    Py_UCS4 lastsep;
-
-    /* Compute the output string size. */
-    len = PyUnicode_GET_LENGTH(pathstr);
-    /* If there is no separator, this returns -1, so
-       fname will be 0. */
-    fname = rightmost_sep_obj(pathstr, 0, len) + 1;
-    /* Windows: re-use the last separator character (/ or \\) when
-       appending the __pycache__ path. */
-    if (fname > 0)
-        lastsep = PyUnicode_READ_CHAR(pathstr, fname -1);
-    else
-        lastsep = SEP;
-    ext = fname - 1;
-    for(i = fname; i < len; i++)
-        if (PyUnicode_READ_CHAR(pathstr, i) == '.')
-            ext = i + 1;
-    if (ext < fname)
-        /* No dot in filename; use entire filename */
-        ext = len;
-
-    /* result = pathstr[:fname] + "__pycache__" + SEP +
-                pathstr[fname:ext] + tag + ".py[co]" */
-    taglen = strlen(pyc_tag);
-    result = PyUnicode_New(ext + pycache_len + 1 + taglen + 4,
-                           PyUnicode_MAX_CHAR_VALUE(pathstr));
-    if (!result)
-        return NULL;
-    kind = PyUnicode_KIND(result);
-    data = PyUnicode_DATA(result);
-    PyUnicode_CopyCharacters(result, 0, pathstr, 0, fname);
-    pos = fname;
-    for (i = 0; i < pycache_len; i++)
-        PyUnicode_WRITE(kind, data, pos++, CACHEDIR[i]);
-    PyUnicode_WRITE(kind, data, pos++, lastsep);
-    PyUnicode_CopyCharacters(result, pos, pathstr,
-                             fname, ext - fname);
-    pos += ext - fname;
-    for (i = 0; pyc_tag[i]; i++)
-        PyUnicode_WRITE(kind, data, pos++, pyc_tag[i]);
-    PyUnicode_WRITE(kind, data, pos++, '.');
-    PyUnicode_WRITE(kind, data, pos++, 'p');
-    PyUnicode_WRITE(kind, data, pos++, 'y');
-    PyUnicode_WRITE(kind, data, pos++, debug ? 'c' : 'o');
-    return result;
-}
-
 
 /* Given a pathname to a Python byte compiled file, return the path to the
    source file, if the path matches the PEP 3147 format.  This does not check
@@ -2991,49 +2925,6 @@ PyDoc_STRVAR(doc_reload,
 \n\
 Reload the module.  The module must have been successfully imported before.");
 
-static PyObject *
-imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws)
-{
-    static char *kwlist[] = {"path", "debug_override", NULL};
-
-    PyObject *pathname, *cpathname;
-    PyObject *debug_override = NULL;
-    int debug = !Py_OptimizeFlag;
-
-    if (!PyArg_ParseTupleAndKeywords(
-                args, kws, "O&|O", kwlist,
-                PyUnicode_FSDecoder, &pathname, &debug_override))
-        return NULL;
-
-    if (debug_override != NULL &&
-        (debug = PyObject_IsTrue(debug_override)) < 0) {
-        Py_DECREF(pathname);
-        return NULL;
-    }
-
-    if (PyUnicode_READY(pathname) < 0)
-        return NULL;
-
-    cpathname = make_compiled_pathname(pathname, debug);
-    Py_DECREF(pathname);
-
-    if (cpathname == NULL) {
-        PyErr_Format(PyExc_SystemError, "path buffer too short");
-        return NULL;
-    }
-    return cpathname;
-}
-
-PyDoc_STRVAR(doc_cache_from_source,
-"cache_from_source(path, [debug_override]) -> path\n\
-Given the path to a .py file, return the path to its .pyc/.pyo file.\n\
-\n\
-The .py file does not need to exist; this simply returns the path to the\n\
-.pyc/.pyo file calculated as if the .py file were imported.  The extension\n\
-will be .pyc unless __debug__ is not defined, then it will be .pyo.\n\
-\n\
-If debug_override is not None, then it must be a boolean and is taken as\n\
-the value of __debug__ instead.");
 
 static PyObject *
 imp_source_from_cache(PyObject *self, PyObject *args, PyObject *kws)
@@ -3116,8 +3007,6 @@ static PyMethodDef imp_methods[] = {
     {"acquire_lock", imp_acquire_lock, METH_NOARGS,  doc_acquire_lock},
     {"release_lock", imp_release_lock, METH_NOARGS,  doc_release_lock},
     {"reload",       imp_reload,       METH_O,       doc_reload},
-    {"cache_from_source", (PyCFunction)imp_cache_from_source,
-     METH_VARARGS | METH_KEYWORDS, doc_cache_from_source},
     {"source_from_cache", (PyCFunction)imp_source_from_cache,
      METH_VARARGS | METH_KEYWORDS, doc_source_from_cache},
     /* The rest are obsolete */