mirror of
https://github.com/python/cpython.git
synced 2024-12-19 06:44:47 +08:00
e96ec68101
a read1() method), and add an undocumented *write_through* parameter to mandate unbuffered writes.
2035 lines
68 KiB
Python
2035 lines
68 KiB
Python
"""
|
|
Python implementation of the io module.
|
|
"""
|
|
|
|
import os
|
|
import abc
|
|
import codecs
|
|
import warnings
|
|
# Import _thread instead of threading to reduce startup cost
|
|
try:
|
|
from _thread import allocate_lock as Lock
|
|
except ImportError:
|
|
from _dummy_thread import allocate_lock as Lock
|
|
|
|
import io
|
|
from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END)
|
|
from errno import EINTR
|
|
|
|
# open() uses st_blksize whenever we can
|
|
DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes
|
|
|
|
# NOTE: Base classes defined here are registered with the "official" ABCs
|
|
# defined in io.py. We don't use real inheritance though, because we don't
|
|
# want to inherit the C implementations.
|
|
|
|
|
|
class BlockingIOError(IOError):
|
|
|
|
"""Exception raised when I/O would block on a non-blocking I/O stream."""
|
|
|
|
def __init__(self, errno, strerror, characters_written=0):
|
|
super().__init__(errno, strerror)
|
|
if not isinstance(characters_written, int):
|
|
raise TypeError("characters_written must be a integer")
|
|
self.characters_written = characters_written
|
|
|
|
|
|
def open(file, mode="r", buffering=-1, encoding=None, errors=None,
|
|
newline=None, closefd=True):
|
|
|
|
r"""Open file and return a stream. Raise IOError upon failure.
|
|
|
|
file is either a text or byte string giving the name (and the path
|
|
if the file isn't in the current working directory) of the file to
|
|
be opened or an integer file descriptor of the file to be
|
|
wrapped. (If a file descriptor is given, it is closed when the
|
|
returned I/O object is closed, unless closefd is set to False.)
|
|
|
|
mode is an optional string that specifies the mode in which the file
|
|
is opened. It defaults to 'r' which means open for reading in text
|
|
mode. Other common values are 'w' for writing (truncating the file if
|
|
it already exists), and 'a' for appending (which on some Unix systems,
|
|
means that all writes append to the end of the file regardless of the
|
|
current seek position). In text mode, if encoding is not specified the
|
|
encoding used is platform dependent. (For reading and writing raw
|
|
bytes use binary mode and leave encoding unspecified.) The available
|
|
modes are:
|
|
|
|
========= ===============================================================
|
|
Character Meaning
|
|
--------- ---------------------------------------------------------------
|
|
'r' open for reading (default)
|
|
'w' open for writing, truncating the file first
|
|
'a' open for writing, appending to the end of the file if it exists
|
|
'b' binary mode
|
|
't' text mode (default)
|
|
'+' open a disk file for updating (reading and writing)
|
|
'U' universal newline mode (for backwards compatibility; unneeded
|
|
for new code)
|
|
========= ===============================================================
|
|
|
|
The default mode is 'rt' (open for reading text). For binary random
|
|
access, the mode 'w+b' opens and truncates the file to 0 bytes, while
|
|
'r+b' opens the file without truncation.
|
|
|
|
Python distinguishes between files opened in binary and text modes,
|
|
even when the underlying operating system doesn't. Files opened in
|
|
binary mode (appending 'b' to the mode argument) return contents as
|
|
bytes objects without any decoding. In text mode (the default, or when
|
|
't' is appended to the mode argument), the contents of the file are
|
|
returned as strings, the bytes having been first decoded using a
|
|
platform-dependent encoding or using the specified encoding if given.
|
|
|
|
buffering is an optional integer used to set the buffering policy.
|
|
Pass 0 to switch buffering off (only allowed in binary mode), 1 to select
|
|
line buffering (only usable in text mode), and an integer > 1 to indicate
|
|
the size of a fixed-size chunk buffer. When no buffering argument is
|
|
given, the default buffering policy works as follows:
|
|
|
|
* Binary files are buffered in fixed-size chunks; the size of the buffer
|
|
is chosen using a heuristic trying to determine the underlying device's
|
|
"block size" and falling back on `io.DEFAULT_BUFFER_SIZE`.
|
|
On many systems, the buffer will typically be 4096 or 8192 bytes long.
|
|
|
|
* "Interactive" text files (files for which isatty() returns True)
|
|
use line buffering. Other text files use the policy described above
|
|
for binary files.
|
|
|
|
encoding is the str name of the encoding used to decode or encode the
|
|
file. This should only be used in text mode. The default encoding is
|
|
platform dependent, but any encoding supported by Python can be
|
|
passed. See the codecs module for the list of supported encodings.
|
|
|
|
errors is an optional string that specifies how encoding errors are to
|
|
be handled---this argument should not be used in binary mode. Pass
|
|
'strict' to raise a ValueError exception if there is an encoding error
|
|
(the default of None has the same effect), or pass 'ignore' to ignore
|
|
errors. (Note that ignoring encoding errors can lead to data loss.)
|
|
See the documentation for codecs.register for a list of the permitted
|
|
encoding error strings.
|
|
|
|
newline is a string controlling how universal newlines works (it only
|
|
applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It works
|
|
as follows:
|
|
|
|
* On input, if newline is None, universal newlines mode is
|
|
enabled. Lines in the input can end in '\n', '\r', or '\r\n', and
|
|
these are translated into '\n' before being returned to the
|
|
caller. If it is '', universal newline mode is enabled, but line
|
|
endings are returned to the caller untranslated. If it has any of
|
|
the other legal values, input lines are only terminated by the given
|
|
string, and the line ending is returned to the caller untranslated.
|
|
|
|
* On output, if newline is None, any '\n' characters written are
|
|
translated to the system default line separator, os.linesep. If
|
|
newline is '', no translation takes place. If newline is any of the
|
|
other legal values, any '\n' characters written are translated to
|
|
the given string.
|
|
|
|
closedfd is a bool. If closefd is False, the underlying file descriptor will
|
|
be kept open when the file is closed. This does not work when a file name is
|
|
given and must be True in that case.
|
|
|
|
open() returns a file object whose type depends on the mode, and
|
|
through which the standard file operations such as reading and writing
|
|
are performed. When open() is used to open a file in a text mode ('w',
|
|
'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open
|
|
a file in a binary mode, the returned class varies: in read binary
|
|
mode, it returns a BufferedReader; in write binary and append binary
|
|
modes, it returns a BufferedWriter, and in read/write mode, it returns
|
|
a BufferedRandom.
|
|
|
|
It is also possible to use a string or bytearray as a file for both
|
|
reading and writing. For strings StringIO can be used like a file
|
|
opened in a text mode, and for bytes a BytesIO can be used like a file
|
|
opened in a binary mode.
|
|
"""
|
|
if not isinstance(file, (str, bytes, int)):
|
|
raise TypeError("invalid file: %r" % file)
|
|
if not isinstance(mode, str):
|
|
raise TypeError("invalid mode: %r" % mode)
|
|
if not isinstance(buffering, int):
|
|
raise TypeError("invalid buffering: %r" % buffering)
|
|
if encoding is not None and not isinstance(encoding, str):
|
|
raise TypeError("invalid encoding: %r" % encoding)
|
|
if errors is not None and not isinstance(errors, str):
|
|
raise TypeError("invalid errors: %r" % errors)
|
|
modes = set(mode)
|
|
if modes - set("arwb+tU") or len(mode) > len(modes):
|
|
raise ValueError("invalid mode: %r" % mode)
|
|
reading = "r" in modes
|
|
writing = "w" in modes
|
|
appending = "a" in modes
|
|
updating = "+" in modes
|
|
text = "t" in modes
|
|
binary = "b" in modes
|
|
if "U" in modes:
|
|
if writing or appending:
|
|
raise ValueError("can't use U and writing mode at once")
|
|
reading = True
|
|
if text and binary:
|
|
raise ValueError("can't have text and binary mode at once")
|
|
if reading + writing + appending > 1:
|
|
raise ValueError("can't have read/write/append mode at once")
|
|
if not (reading or writing or appending):
|
|
raise ValueError("must have exactly one of read/write/append mode")
|
|
if binary and encoding is not None:
|
|
raise ValueError("binary mode doesn't take an encoding argument")
|
|
if binary and errors is not None:
|
|
raise ValueError("binary mode doesn't take an errors argument")
|
|
if binary and newline is not None:
|
|
raise ValueError("binary mode doesn't take a newline argument")
|
|
raw = FileIO(file,
|
|
(reading and "r" or "") +
|
|
(writing and "w" or "") +
|
|
(appending and "a" or "") +
|
|
(updating and "+" or ""),
|
|
closefd)
|
|
line_buffering = False
|
|
if buffering == 1 or buffering < 0 and raw.isatty():
|
|
buffering = -1
|
|
line_buffering = True
|
|
if buffering < 0:
|
|
buffering = DEFAULT_BUFFER_SIZE
|
|
try:
|
|
bs = os.fstat(raw.fileno()).st_blksize
|
|
except (os.error, AttributeError):
|
|
pass
|
|
else:
|
|
if bs > 1:
|
|
buffering = bs
|
|
if buffering < 0:
|
|
raise ValueError("invalid buffering size")
|
|
if buffering == 0:
|
|
if binary:
|
|
return raw
|
|
raise ValueError("can't have unbuffered text I/O")
|
|
if updating:
|
|
buffer = BufferedRandom(raw, buffering)
|
|
elif writing or appending:
|
|
buffer = BufferedWriter(raw, buffering)
|
|
elif reading:
|
|
buffer = BufferedReader(raw, buffering)
|
|
else:
|
|
raise ValueError("unknown mode: %r" % mode)
|
|
if binary:
|
|
return buffer
|
|
text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
|
|
text.mode = mode
|
|
return text
|
|
|
|
|
|
class DocDescriptor:
|
|
"""Helper for builtins.open.__doc__
|
|
"""
|
|
def __get__(self, obj, typ):
|
|
return (
|
|
"open(file, mode='r', buffering=-1, encoding=None, "
|
|
"errors=None, newline=None, closefd=True)\n\n" +
|
|
open.__doc__)
|
|
|
|
class OpenWrapper:
|
|
"""Wrapper for builtins.open
|
|
|
|
Trick so that open won't become a bound method when stored
|
|
as a class variable (as dbm.dumb does).
|
|
|
|
See initstdio() in Python/pythonrun.c.
|
|
"""
|
|
__doc__ = DocDescriptor()
|
|
|
|
def __new__(cls, *args, **kwargs):
|
|
return open(*args, **kwargs)
|
|
|
|
|
|
# In normal operation, both `UnsupportedOperation`s should be bound to the
|
|
# same object.
|
|
try:
|
|
UnsupportedOperation = io.UnsupportedOperation
|
|
except AttributeError:
|
|
class UnsupportedOperation(ValueError, IOError):
|
|
pass
|
|
|
|
|
|
class IOBase(metaclass=abc.ABCMeta):
|
|
|
|
"""The abstract base class for all I/O classes, acting on streams of
|
|
bytes. There is no public constructor.
|
|
|
|
This class provides dummy implementations for many methods that
|
|
derived classes can override selectively; the default implementations
|
|
represent a file that cannot be read, written or seeked.
|
|
|
|
Even though IOBase does not declare read, readinto, or write because
|
|
their signatures will vary, implementations and clients should
|
|
consider those methods part of the interface. Also, implementations
|
|
may raise UnsupportedOperation when operations they do not support are
|
|
called.
|
|
|
|
The basic type used for binary data read from or written to a file is
|
|
bytes. bytearrays are accepted too, and in some cases (such as
|
|
readinto) needed. Text I/O classes work with str data.
|
|
|
|
Note that calling any method (even inquiries) on a closed stream is
|
|
undefined. Implementations may raise IOError in this case.
|
|
|
|
IOBase (and its subclasses) support the iterator protocol, meaning
|
|
that an IOBase object can be iterated over yielding the lines in a
|
|
stream.
|
|
|
|
IOBase also supports the :keyword:`with` statement. In this example,
|
|
fp is closed after the suite of the with statement is complete:
|
|
|
|
with open('spam.txt', 'r') as fp:
|
|
fp.write('Spam and eggs!')
|
|
"""
|
|
|
|
### Internal ###
|
|
|
|
def _unsupported(self, name):
|
|
"""Internal: raise an IOError exception for unsupported operations."""
|
|
raise UnsupportedOperation("%s.%s() not supported" %
|
|
(self.__class__.__name__, name))
|
|
|
|
### Positioning ###
|
|
|
|
def seek(self, pos, whence=0):
|
|
"""Change stream position.
|
|
|
|
Change the stream position to byte offset offset. offset is
|
|
interpreted relative to the position indicated by whence. Values
|
|
for whence are ints:
|
|
|
|
* 0 -- start of stream (the default); offset should be zero or positive
|
|
* 1 -- current stream position; offset may be negative
|
|
* 2 -- end of stream; offset is usually negative
|
|
|
|
Return an int indicating the new absolute position.
|
|
"""
|
|
self._unsupported("seek")
|
|
|
|
def tell(self):
|
|
"""Return an int indicating the current stream position."""
|
|
return self.seek(0, 1)
|
|
|
|
def truncate(self, pos=None):
|
|
"""Truncate file to size bytes.
|
|
|
|
Size defaults to the current IO position as reported by tell(). Return
|
|
the new size.
|
|
"""
|
|
self._unsupported("truncate")
|
|
|
|
### Flush and close ###
|
|
|
|
def flush(self):
|
|
"""Flush write buffers, if applicable.
|
|
|
|
This is not implemented for read-only and non-blocking streams.
|
|
"""
|
|
self._checkClosed()
|
|
# XXX Should this return the number of bytes written???
|
|
|
|
__closed = False
|
|
|
|
def close(self):
|
|
"""Flush and close the IO object.
|
|
|
|
This method has no effect if the file is already closed.
|
|
"""
|
|
if not self.__closed:
|
|
self.flush()
|
|
self.__closed = True
|
|
|
|
def __del__(self):
|
|
"""Destructor. Calls close()."""
|
|
# The try/except block is in case this is called at program
|
|
# exit time, when it's possible that globals have already been
|
|
# deleted, and then the close() call might fail. Since
|
|
# there's nothing we can do about such failures and they annoy
|
|
# the end users, we suppress the traceback.
|
|
try:
|
|
self.close()
|
|
except:
|
|
pass
|
|
|
|
### Inquiries ###
|
|
|
|
def seekable(self):
|
|
"""Return a bool indicating whether object supports random access.
|
|
|
|
If False, seek(), tell() and truncate() will raise UnsupportedOperation.
|
|
This method may need to do a test seek().
|
|
"""
|
|
return False
|
|
|
|
def _checkSeekable(self, msg=None):
|
|
"""Internal: raise UnsupportedOperation if file is not seekable
|
|
"""
|
|
if not self.seekable():
|
|
raise UnsupportedOperation("File or stream is not seekable."
|
|
if msg is None else msg)
|
|
|
|
def readable(self):
|
|
"""Return a bool indicating whether object was opened for reading.
|
|
|
|
If False, read() will raise UnsupportedOperation.
|
|
"""
|
|
return False
|
|
|
|
def _checkReadable(self, msg=None):
|
|
"""Internal: raise UnsupportedOperation if file is not readable
|
|
"""
|
|
if not self.readable():
|
|
raise UnsupportedOperation("File or stream is not readable."
|
|
if msg is None else msg)
|
|
|
|
def writable(self):
|
|
"""Return a bool indicating whether object was opened for writing.
|
|
|
|
If False, write() and truncate() will raise UnsupportedOperation.
|
|
"""
|
|
return False
|
|
|
|
def _checkWritable(self, msg=None):
|
|
"""Internal: raise UnsupportedOperation if file is not writable
|
|
"""
|
|
if not self.writable():
|
|
raise UnsupportedOperation("File or stream is not writable."
|
|
if msg is None else msg)
|
|
|
|
@property
|
|
def closed(self):
|
|
"""closed: bool. True iff the file has been closed.
|
|
|
|
For backwards compatibility, this is a property, not a predicate.
|
|
"""
|
|
return self.__closed
|
|
|
|
def _checkClosed(self, msg=None):
|
|
"""Internal: raise an ValueError if file is closed
|
|
"""
|
|
if self.closed:
|
|
raise ValueError("I/O operation on closed file."
|
|
if msg is None else msg)
|
|
|
|
### Context manager ###
|
|
|
|
def __enter__(self): # That's a forward reference
|
|
"""Context management protocol. Returns self (an instance of IOBase)."""
|
|
self._checkClosed()
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
"""Context management protocol. Calls close()"""
|
|
self.close()
|
|
|
|
### Lower-level APIs ###
|
|
|
|
# XXX Should these be present even if unimplemented?
|
|
|
|
def fileno(self):
|
|
"""Returns underlying file descriptor (an int) if one exists.
|
|
|
|
An IOError is raised if the IO object does not use a file descriptor.
|
|
"""
|
|
self._unsupported("fileno")
|
|
|
|
def isatty(self):
|
|
"""Return a bool indicating whether this is an 'interactive' stream.
|
|
|
|
Return False if it can't be determined.
|
|
"""
|
|
self._checkClosed()
|
|
return False
|
|
|
|
### Readline[s] and writelines ###
|
|
|
|
def readline(self, limit=-1):
|
|
r"""Read and return a line of bytes from the stream.
|
|
|
|
If limit is specified, at most limit bytes will be read.
|
|
Limit should be an int.
|
|
|
|
The line terminator is always b'\n' for binary files; for text
|
|
files, the newlines argument to open can be used to select the line
|
|
terminator(s) recognized.
|
|
"""
|
|
# For backwards compatibility, a (slowish) readline().
|
|
if hasattr(self, "peek"):
|
|
def nreadahead():
|
|
readahead = self.peek(1)
|
|
if not readahead:
|
|
return 1
|
|
n = (readahead.find(b"\n") + 1) or len(readahead)
|
|
if limit >= 0:
|
|
n = min(n, limit)
|
|
return n
|
|
else:
|
|
def nreadahead():
|
|
return 1
|
|
if limit is None:
|
|
limit = -1
|
|
elif not isinstance(limit, int):
|
|
raise TypeError("limit must be an integer")
|
|
res = bytearray()
|
|
while limit < 0 or len(res) < limit:
|
|
b = self.read(nreadahead())
|
|
if not b:
|
|
break
|
|
res += b
|
|
if res.endswith(b"\n"):
|
|
break
|
|
return bytes(res)
|
|
|
|
def __iter__(self):
|
|
self._checkClosed()
|
|
return self
|
|
|
|
def __next__(self):
|
|
line = self.readline()
|
|
if not line:
|
|
raise StopIteration
|
|
return line
|
|
|
|
def readlines(self, hint=None):
|
|
"""Return a list of lines from the stream.
|
|
|
|
hint can be specified to control the number of lines read: no more
|
|
lines will be read if the total size (in bytes/characters) of all
|
|
lines so far exceeds hint.
|
|
"""
|
|
if hint is None or hint <= 0:
|
|
return list(self)
|
|
n = 0
|
|
lines = []
|
|
for line in self:
|
|
lines.append(line)
|
|
n += len(line)
|
|
if n >= hint:
|
|
break
|
|
return lines
|
|
|
|
def writelines(self, lines):
|
|
self._checkClosed()
|
|
for line in lines:
|
|
self.write(line)
|
|
|
|
io.IOBase.register(IOBase)
|
|
|
|
|
|
class RawIOBase(IOBase):
|
|
|
|
"""Base class for raw binary I/O."""
|
|
|
|
# The read() method is implemented by calling readinto(); derived
|
|
# classes that want to support read() only need to implement
|
|
# readinto() as a primitive operation. In general, readinto() can be
|
|
# more efficient than read().
|
|
|
|
# (It would be tempting to also provide an implementation of
|
|
# readinto() in terms of read(), in case the latter is a more suitable
|
|
# primitive operation, but that would lead to nasty recursion in case
|
|
# a subclass doesn't implement either.)
|
|
|
|
def read(self, n=-1):
|
|
"""Read and return up to n bytes, where n is an int.
|
|
|
|
Returns an empty bytes object on EOF, or None if the object is
|
|
set not to block and has no data to read.
|
|
"""
|
|
if n is None:
|
|
n = -1
|
|
if n < 0:
|
|
return self.readall()
|
|
b = bytearray(n.__index__())
|
|
n = self.readinto(b)
|
|
if n is None:
|
|
return None
|
|
del b[n:]
|
|
return bytes(b)
|
|
|
|
def readall(self):
|
|
"""Read until EOF, using multiple read() call."""
|
|
res = bytearray()
|
|
while True:
|
|
data = self.read(DEFAULT_BUFFER_SIZE)
|
|
if not data:
|
|
break
|
|
res += data
|
|
if res:
|
|
return bytes(res)
|
|
else:
|
|
# b'' or None
|
|
return data
|
|
|
|
def readinto(self, b):
|
|
"""Read up to len(b) bytes into bytearray b.
|
|
|
|
Returns an int representing the number of bytes read (0 for EOF), or
|
|
None if the object is set not to block and has no data to read.
|
|
"""
|
|
self._unsupported("readinto")
|
|
|
|
def write(self, b):
|
|
"""Write the given buffer to the IO stream.
|
|
|
|
Returns the number of bytes written, which may be less than len(b).
|
|
"""
|
|
self._unsupported("write")
|
|
|
|
io.RawIOBase.register(RawIOBase)
|
|
from _io import FileIO
|
|
RawIOBase.register(FileIO)
|
|
|
|
|
|
class BufferedIOBase(IOBase):
|
|
|
|
"""Base class for buffered IO objects.
|
|
|
|
The main difference with RawIOBase is that the read() method
|
|
supports omitting the size argument, and does not have a default
|
|
implementation that defers to readinto().
|
|
|
|
In addition, read(), readinto() and write() may raise
|
|
BlockingIOError if the underlying raw stream is in non-blocking
|
|
mode and not ready; unlike their raw counterparts, they will never
|
|
return None.
|
|
|
|
A typical implementation should not inherit from a RawIOBase
|
|
implementation, but wrap one.
|
|
"""
|
|
|
|
def read(self, n=None):
|
|
"""Read and return up to n bytes, where n is an int.
|
|
|
|
If the argument is omitted, None, or negative, reads and
|
|
returns all data until EOF.
|
|
|
|
If the argument is positive, and the underlying raw stream is
|
|
not 'interactive', multiple raw reads may be issued to satisfy
|
|
the byte count (unless EOF is reached first). But for
|
|
interactive raw streams (XXX and for pipes?), at most one raw
|
|
read will be issued, and a short result does not imply that
|
|
EOF is imminent.
|
|
|
|
Returns an empty bytes array on EOF.
|
|
|
|
Raises BlockingIOError if the underlying raw stream has no
|
|
data at the moment.
|
|
"""
|
|
self._unsupported("read")
|
|
|
|
def read1(self, n=None):
|
|
"""Read up to n bytes with at most one read() system call,
|
|
where n is an int.
|
|
"""
|
|
self._unsupported("read1")
|
|
|
|
def readinto(self, b):
|
|
"""Read up to len(b) bytes into bytearray b.
|
|
|
|
Like read(), this may issue multiple reads to the underlying raw
|
|
stream, unless the latter is 'interactive'.
|
|
|
|
Returns an int representing the number of bytes read (0 for EOF).
|
|
|
|
Raises BlockingIOError if the underlying raw stream has no
|
|
data at the moment.
|
|
"""
|
|
# XXX This ought to work with anything that supports the buffer API
|
|
data = self.read(len(b))
|
|
n = len(data)
|
|
try:
|
|
b[:n] = data
|
|
except TypeError as err:
|
|
import array
|
|
if not isinstance(b, array.array):
|
|
raise err
|
|
b[:n] = array.array('b', data)
|
|
return n
|
|
|
|
def write(self, b):
|
|
"""Write the given bytes buffer to the IO stream.
|
|
|
|
Return the number of bytes written, which is never less than
|
|
len(b).
|
|
|
|
Raises BlockingIOError if the buffer is full and the
|
|
underlying raw stream cannot accept more data at the moment.
|
|
"""
|
|
self._unsupported("write")
|
|
|
|
def detach(self):
|
|
"""
|
|
Separate the underlying raw stream from the buffer and return it.
|
|
|
|
After the raw stream has been detached, the buffer is in an unusable
|
|
state.
|
|
"""
|
|
self._unsupported("detach")
|
|
|
|
io.BufferedIOBase.register(BufferedIOBase)
|
|
|
|
|
|
class _BufferedIOMixin(BufferedIOBase):
|
|
|
|
"""A mixin implementation of BufferedIOBase with an underlying raw stream.
|
|
|
|
This passes most requests on to the underlying raw stream. It
|
|
does *not* provide implementations of read(), readinto() or
|
|
write().
|
|
"""
|
|
|
|
def __init__(self, raw):
|
|
self._raw = raw
|
|
|
|
### Positioning ###
|
|
|
|
def seek(self, pos, whence=0):
|
|
new_position = self.raw.seek(pos, whence)
|
|
if new_position < 0:
|
|
raise IOError("seek() returned an invalid position")
|
|
return new_position
|
|
|
|
def tell(self):
|
|
pos = self.raw.tell()
|
|
if pos < 0:
|
|
raise IOError("tell() returned an invalid position")
|
|
return pos
|
|
|
|
def truncate(self, pos=None):
|
|
# Flush the stream. We're mixing buffered I/O with lower-level I/O,
|
|
# and a flush may be necessary to synch both views of the current
|
|
# file state.
|
|
self.flush()
|
|
|
|
if pos is None:
|
|
pos = self.tell()
|
|
# XXX: Should seek() be used, instead of passing the position
|
|
# XXX directly to truncate?
|
|
return self.raw.truncate(pos)
|
|
|
|
### Flush and close ###
|
|
|
|
def flush(self):
|
|
if self.closed:
|
|
raise ValueError("flush of closed file")
|
|
self.raw.flush()
|
|
|
|
def close(self):
|
|
if self.raw is not None and not self.closed:
|
|
self.flush()
|
|
self.raw.close()
|
|
|
|
def detach(self):
|
|
if self.raw is None:
|
|
raise ValueError("raw stream already detached")
|
|
self.flush()
|
|
raw = self._raw
|
|
self._raw = None
|
|
return raw
|
|
|
|
### Inquiries ###
|
|
|
|
def seekable(self):
|
|
return self.raw.seekable()
|
|
|
|
def readable(self):
|
|
return self.raw.readable()
|
|
|
|
def writable(self):
|
|
return self.raw.writable()
|
|
|
|
@property
|
|
def raw(self):
|
|
return self._raw
|
|
|
|
@property
|
|
def closed(self):
|
|
return self.raw.closed
|
|
|
|
@property
|
|
def name(self):
|
|
return self.raw.name
|
|
|
|
@property
|
|
def mode(self):
|
|
return self.raw.mode
|
|
|
|
def __getstate__(self):
|
|
raise TypeError("can not serialize a '{0}' object"
|
|
.format(self.__class__.__name__))
|
|
|
|
def __repr__(self):
|
|
clsname = self.__class__.__name__
|
|
try:
|
|
name = self.name
|
|
except AttributeError:
|
|
return "<_pyio.{0}>".format(clsname)
|
|
else:
|
|
return "<_pyio.{0} name={1!r}>".format(clsname, name)
|
|
|
|
### Lower-level APIs ###
|
|
|
|
def fileno(self):
|
|
return self.raw.fileno()
|
|
|
|
def isatty(self):
|
|
return self.raw.isatty()
|
|
|
|
|
|
class BytesIO(BufferedIOBase):
|
|
|
|
"""Buffered I/O implementation using an in-memory bytes buffer."""
|
|
|
|
def __init__(self, initial_bytes=None):
|
|
buf = bytearray()
|
|
if initial_bytes is not None:
|
|
buf += initial_bytes
|
|
self._buffer = buf
|
|
self._pos = 0
|
|
|
|
def __getstate__(self):
|
|
if self.closed:
|
|
raise ValueError("__getstate__ on closed file")
|
|
return self.__dict__.copy()
|
|
|
|
def getvalue(self):
|
|
"""Return the bytes value (contents) of the buffer
|
|
"""
|
|
if self.closed:
|
|
raise ValueError("getvalue on closed file")
|
|
return bytes(self._buffer)
|
|
|
|
def getbuffer(self):
|
|
"""Return a readable and writable view of the buffer.
|
|
"""
|
|
return memoryview(self._buffer)
|
|
|
|
def read(self, n=None):
|
|
if self.closed:
|
|
raise ValueError("read from closed file")
|
|
if n is None:
|
|
n = -1
|
|
if n < 0:
|
|
n = len(self._buffer)
|
|
if len(self._buffer) <= self._pos:
|
|
return b""
|
|
newpos = min(len(self._buffer), self._pos + n)
|
|
b = self._buffer[self._pos : newpos]
|
|
self._pos = newpos
|
|
return bytes(b)
|
|
|
|
def read1(self, n):
|
|
"""This is the same as read.
|
|
"""
|
|
return self.read(n)
|
|
|
|
def write(self, b):
|
|
if self.closed:
|
|
raise ValueError("write to closed file")
|
|
if isinstance(b, str):
|
|
raise TypeError("can't write str to binary stream")
|
|
n = len(b)
|
|
if n == 0:
|
|
return 0
|
|
pos = self._pos
|
|
if pos > len(self._buffer):
|
|
# Inserts null bytes between the current end of the file
|
|
# and the new write position.
|
|
padding = b'\x00' * (pos - len(self._buffer))
|
|
self._buffer += padding
|
|
self._buffer[pos:pos + n] = b
|
|
self._pos += n
|
|
return n
|
|
|
|
def seek(self, pos, whence=0):
|
|
if self.closed:
|
|
raise ValueError("seek on closed file")
|
|
try:
|
|
pos.__index__
|
|
except AttributeError as err:
|
|
raise TypeError("an integer is required") from err
|
|
if whence == 0:
|
|
if pos < 0:
|
|
raise ValueError("negative seek position %r" % (pos,))
|
|
self._pos = pos
|
|
elif whence == 1:
|
|
self._pos = max(0, self._pos + pos)
|
|
elif whence == 2:
|
|
self._pos = max(0, len(self._buffer) + pos)
|
|
else:
|
|
raise ValueError("invalid whence value")
|
|
return self._pos
|
|
|
|
def tell(self):
|
|
if self.closed:
|
|
raise ValueError("tell on closed file")
|
|
return self._pos
|
|
|
|
def truncate(self, pos=None):
|
|
if self.closed:
|
|
raise ValueError("truncate on closed file")
|
|
if pos is None:
|
|
pos = self._pos
|
|
else:
|
|
try:
|
|
pos.__index__
|
|
except AttributeError as err:
|
|
raise TypeError("an integer is required") from err
|
|
if pos < 0:
|
|
raise ValueError("negative truncate position %r" % (pos,))
|
|
del self._buffer[pos:]
|
|
return pos
|
|
|
|
def readable(self):
|
|
return True
|
|
|
|
def writable(self):
|
|
return True
|
|
|
|
def seekable(self):
|
|
return True
|
|
|
|
|
|
class BufferedReader(_BufferedIOMixin):
|
|
|
|
"""BufferedReader(raw[, buffer_size])
|
|
|
|
A buffer for a readable, sequential BaseRawIO object.
|
|
|
|
The constructor creates a BufferedReader for the given readable raw
|
|
stream and buffer_size. If buffer_size is omitted, DEFAULT_BUFFER_SIZE
|
|
is used.
|
|
"""
|
|
|
|
def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):
|
|
"""Create a new buffered reader using the given readable raw IO object.
|
|
"""
|
|
if not raw.readable():
|
|
raise IOError('"raw" argument must be readable.')
|
|
|
|
_BufferedIOMixin.__init__(self, raw)
|
|
if buffer_size <= 0:
|
|
raise ValueError("invalid buffer size")
|
|
self.buffer_size = buffer_size
|
|
self._reset_read_buf()
|
|
self._read_lock = Lock()
|
|
|
|
def _reset_read_buf(self):
|
|
self._read_buf = b""
|
|
self._read_pos = 0
|
|
|
|
def read(self, n=None):
|
|
"""Read n bytes.
|
|
|
|
Returns exactly n bytes of data unless the underlying raw IO
|
|
stream reaches EOF or if the call would block in non-blocking
|
|
mode. If n is negative, read until EOF or until read() would
|
|
block.
|
|
"""
|
|
if n is not None and n < -1:
|
|
raise ValueError("invalid number of bytes to read")
|
|
with self._read_lock:
|
|
return self._read_unlocked(n)
|
|
|
|
def _read_unlocked(self, n=None):
|
|
nodata_val = b""
|
|
empty_values = (b"", None)
|
|
buf = self._read_buf
|
|
pos = self._read_pos
|
|
|
|
# Special case for when the number of bytes to read is unspecified.
|
|
if n is None or n == -1:
|
|
self._reset_read_buf()
|
|
chunks = [buf[pos:]] # Strip the consumed bytes.
|
|
current_size = 0
|
|
while True:
|
|
# Read until EOF or until read() would block.
|
|
try:
|
|
chunk = self.raw.read()
|
|
except IOError as e:
|
|
if e.errno != EINTR:
|
|
raise
|
|
continue
|
|
if chunk in empty_values:
|
|
nodata_val = chunk
|
|
break
|
|
current_size += len(chunk)
|
|
chunks.append(chunk)
|
|
return b"".join(chunks) or nodata_val
|
|
|
|
# The number of bytes to read is specified, return at most n bytes.
|
|
avail = len(buf) - pos # Length of the available buffered data.
|
|
if n <= avail:
|
|
# Fast path: the data to read is fully buffered.
|
|
self._read_pos += n
|
|
return buf[pos:pos+n]
|
|
# Slow path: read from the stream until enough bytes are read,
|
|
# or until an EOF occurs or until read() would block.
|
|
chunks = [buf[pos:]]
|
|
wanted = max(self.buffer_size, n)
|
|
while avail < n:
|
|
try:
|
|
chunk = self.raw.read(wanted)
|
|
except IOError as e:
|
|
if e.errno != EINTR:
|
|
raise
|
|
continue
|
|
if chunk in empty_values:
|
|
nodata_val = chunk
|
|
break
|
|
avail += len(chunk)
|
|
chunks.append(chunk)
|
|
# n is more then avail only when an EOF occurred or when
|
|
# read() would have blocked.
|
|
n = min(n, avail)
|
|
out = b"".join(chunks)
|
|
self._read_buf = out[n:] # Save the extra data in the buffer.
|
|
self._read_pos = 0
|
|
return out[:n] if out else nodata_val
|
|
|
|
def peek(self, n=0):
|
|
"""Returns buffered bytes without advancing the position.
|
|
|
|
The argument indicates a desired minimal number of bytes; we
|
|
do at most one raw read to satisfy it. We never return more
|
|
than self.buffer_size.
|
|
"""
|
|
with self._read_lock:
|
|
return self._peek_unlocked(n)
|
|
|
|
def _peek_unlocked(self, n=0):
|
|
want = min(n, self.buffer_size)
|
|
have = len(self._read_buf) - self._read_pos
|
|
if have < want or have <= 0:
|
|
to_read = self.buffer_size - have
|
|
while True:
|
|
try:
|
|
current = self.raw.read(to_read)
|
|
except IOError as e:
|
|
if e.errno != EINTR:
|
|
raise
|
|
continue
|
|
break
|
|
if current:
|
|
self._read_buf = self._read_buf[self._read_pos:] + current
|
|
self._read_pos = 0
|
|
return self._read_buf[self._read_pos:]
|
|
|
|
def read1(self, n):
|
|
"""Reads up to n bytes, with at most one read() system call."""
|
|
# Returns up to n bytes. If at least one byte is buffered, we
|
|
# only return buffered bytes. Otherwise, we do one raw read.
|
|
if n < 0:
|
|
raise ValueError("number of bytes to read must be positive")
|
|
if n == 0:
|
|
return b""
|
|
with self._read_lock:
|
|
self._peek_unlocked(1)
|
|
return self._read_unlocked(
|
|
min(n, len(self._read_buf) - self._read_pos))
|
|
|
|
def tell(self):
|
|
return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos
|
|
|
|
def seek(self, pos, whence=0):
|
|
if not (0 <= whence <= 2):
|
|
raise ValueError("invalid whence value")
|
|
with self._read_lock:
|
|
if whence == 1:
|
|
pos -= len(self._read_buf) - self._read_pos
|
|
pos = _BufferedIOMixin.seek(self, pos, whence)
|
|
self._reset_read_buf()
|
|
return pos
|
|
|
|
class BufferedWriter(_BufferedIOMixin):
|
|
|
|
"""A buffer for a writeable sequential RawIO object.
|
|
|
|
The constructor creates a BufferedWriter for the given writeable raw
|
|
stream. If the buffer_size is not given, it defaults to
|
|
DEFAULT_BUFFER_SIZE.
|
|
"""
|
|
|
|
_warning_stack_offset = 2
|
|
|
|
def __init__(self, raw,
|
|
buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
|
|
if not raw.writable():
|
|
raise IOError('"raw" argument must be writable.')
|
|
|
|
_BufferedIOMixin.__init__(self, raw)
|
|
if buffer_size <= 0:
|
|
raise ValueError("invalid buffer size")
|
|
if max_buffer_size is not None:
|
|
warnings.warn("max_buffer_size is deprecated", DeprecationWarning,
|
|
self._warning_stack_offset)
|
|
self.buffer_size = buffer_size
|
|
self._write_buf = bytearray()
|
|
self._write_lock = Lock()
|
|
|
|
def write(self, b):
|
|
if self.closed:
|
|
raise ValueError("write to closed file")
|
|
if isinstance(b, str):
|
|
raise TypeError("can't write str to binary stream")
|
|
with self._write_lock:
|
|
# XXX we can implement some more tricks to try and avoid
|
|
# partial writes
|
|
if len(self._write_buf) > self.buffer_size:
|
|
# We're full, so let's pre-flush the buffer
|
|
try:
|
|
self._flush_unlocked()
|
|
except BlockingIOError as e:
|
|
# We can't accept anything else.
|
|
# XXX Why not just let the exception pass through?
|
|
raise BlockingIOError(e.errno, e.strerror, 0)
|
|
before = len(self._write_buf)
|
|
self._write_buf.extend(b)
|
|
written = len(self._write_buf) - before
|
|
if len(self._write_buf) > self.buffer_size:
|
|
try:
|
|
self._flush_unlocked()
|
|
except BlockingIOError as e:
|
|
if len(self._write_buf) > self.buffer_size:
|
|
# We've hit the buffer_size. We have to accept a partial
|
|
# write and cut back our buffer.
|
|
overage = len(self._write_buf) - self.buffer_size
|
|
written -= overage
|
|
self._write_buf = self._write_buf[:self.buffer_size]
|
|
raise BlockingIOError(e.errno, e.strerror, written)
|
|
return written
|
|
|
|
def truncate(self, pos=None):
|
|
with self._write_lock:
|
|
self._flush_unlocked()
|
|
if pos is None:
|
|
pos = self.raw.tell()
|
|
return self.raw.truncate(pos)
|
|
|
|
def flush(self):
|
|
with self._write_lock:
|
|
self._flush_unlocked()
|
|
|
|
def _flush_unlocked(self):
|
|
if self.closed:
|
|
raise ValueError("flush of closed file")
|
|
written = 0
|
|
try:
|
|
while self._write_buf:
|
|
try:
|
|
n = self.raw.write(self._write_buf)
|
|
except IOError as e:
|
|
if e.errno != EINTR:
|
|
raise
|
|
continue
|
|
if n > len(self._write_buf) or n < 0:
|
|
raise IOError("write() returned incorrect number of bytes")
|
|
del self._write_buf[:n]
|
|
written += n
|
|
except BlockingIOError as e:
|
|
n = e.characters_written
|
|
del self._write_buf[:n]
|
|
written += n
|
|
raise BlockingIOError(e.errno, e.strerror, written)
|
|
|
|
def tell(self):
|
|
return _BufferedIOMixin.tell(self) + len(self._write_buf)
|
|
|
|
def seek(self, pos, whence=0):
|
|
if not (0 <= whence <= 2):
|
|
raise ValueError("invalid whence")
|
|
with self._write_lock:
|
|
self._flush_unlocked()
|
|
return _BufferedIOMixin.seek(self, pos, whence)
|
|
|
|
|
|
class BufferedRWPair(BufferedIOBase):
|
|
|
|
"""A buffered reader and writer object together.
|
|
|
|
A buffered reader object and buffered writer object put together to
|
|
form a sequential IO object that can read and write. This is typically
|
|
used with a socket or two-way pipe.
|
|
|
|
reader and writer are RawIOBase objects that are readable and
|
|
writeable respectively. If the buffer_size is omitted it defaults to
|
|
DEFAULT_BUFFER_SIZE.
|
|
"""
|
|
|
|
# XXX The usefulness of this (compared to having two separate IO
|
|
# objects) is questionable.
|
|
|
|
def __init__(self, reader, writer,
|
|
buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
|
|
"""Constructor.
|
|
|
|
The arguments are two RawIO instances.
|
|
"""
|
|
if max_buffer_size is not None:
|
|
warnings.warn("max_buffer_size is deprecated", DeprecationWarning, 2)
|
|
|
|
if not reader.readable():
|
|
raise IOError('"reader" argument must be readable.')
|
|
|
|
if not writer.writable():
|
|
raise IOError('"writer" argument must be writable.')
|
|
|
|
self.reader = BufferedReader(reader, buffer_size)
|
|
self.writer = BufferedWriter(writer, buffer_size)
|
|
|
|
def read(self, n=None):
|
|
if n is None:
|
|
n = -1
|
|
return self.reader.read(n)
|
|
|
|
def readinto(self, b):
|
|
return self.reader.readinto(b)
|
|
|
|
def write(self, b):
|
|
return self.writer.write(b)
|
|
|
|
def peek(self, n=0):
|
|
return self.reader.peek(n)
|
|
|
|
def read1(self, n):
|
|
return self.reader.read1(n)
|
|
|
|
def readable(self):
|
|
return self.reader.readable()
|
|
|
|
def writable(self):
|
|
return self.writer.writable()
|
|
|
|
def flush(self):
|
|
return self.writer.flush()
|
|
|
|
def close(self):
|
|
self.writer.close()
|
|
self.reader.close()
|
|
|
|
def isatty(self):
|
|
return self.reader.isatty() or self.writer.isatty()
|
|
|
|
@property
|
|
def closed(self):
|
|
return self.writer.closed
|
|
|
|
|
|
class BufferedRandom(BufferedWriter, BufferedReader):
|
|
|
|
"""A buffered interface to random access streams.
|
|
|
|
The constructor creates a reader and writer for a seekable stream,
|
|
raw, given in the first argument. If the buffer_size is omitted it
|
|
defaults to DEFAULT_BUFFER_SIZE.
|
|
"""
|
|
|
|
_warning_stack_offset = 3
|
|
|
|
def __init__(self, raw,
|
|
buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
|
|
raw._checkSeekable()
|
|
BufferedReader.__init__(self, raw, buffer_size)
|
|
BufferedWriter.__init__(self, raw, buffer_size, max_buffer_size)
|
|
|
|
def seek(self, pos, whence=0):
|
|
if not (0 <= whence <= 2):
|
|
raise ValueError("invalid whence")
|
|
self.flush()
|
|
if self._read_buf:
|
|
# Undo read ahead.
|
|
with self._read_lock:
|
|
self.raw.seek(self._read_pos - len(self._read_buf), 1)
|
|
# First do the raw seek, then empty the read buffer, so that
|
|
# if the raw seek fails, we don't lose buffered data forever.
|
|
pos = self.raw.seek(pos, whence)
|
|
with self._read_lock:
|
|
self._reset_read_buf()
|
|
if pos < 0:
|
|
raise IOError("seek() returned invalid position")
|
|
return pos
|
|
|
|
def tell(self):
|
|
if self._write_buf:
|
|
return BufferedWriter.tell(self)
|
|
else:
|
|
return BufferedReader.tell(self)
|
|
|
|
def truncate(self, pos=None):
|
|
if pos is None:
|
|
pos = self.tell()
|
|
# Use seek to flush the read buffer.
|
|
return BufferedWriter.truncate(self, pos)
|
|
|
|
def read(self, n=None):
|
|
if n is None:
|
|
n = -1
|
|
self.flush()
|
|
return BufferedReader.read(self, n)
|
|
|
|
def readinto(self, b):
|
|
self.flush()
|
|
return BufferedReader.readinto(self, b)
|
|
|
|
def peek(self, n=0):
|
|
self.flush()
|
|
return BufferedReader.peek(self, n)
|
|
|
|
def read1(self, n):
|
|
self.flush()
|
|
return BufferedReader.read1(self, n)
|
|
|
|
def write(self, b):
|
|
if self._read_buf:
|
|
# Undo readahead
|
|
with self._read_lock:
|
|
self.raw.seek(self._read_pos - len(self._read_buf), 1)
|
|
self._reset_read_buf()
|
|
return BufferedWriter.write(self, b)
|
|
|
|
|
|
class TextIOBase(IOBase):
|
|
|
|
"""Base class for text I/O.
|
|
|
|
This class provides a character and line based interface to stream
|
|
I/O. There is no readinto method because Python's character strings
|
|
are immutable. There is no public constructor.
|
|
"""
|
|
|
|
def read(self, n=-1):
|
|
"""Read at most n characters from stream, where n is an int.
|
|
|
|
Read from underlying buffer until we have n characters or we hit EOF.
|
|
If n is negative or omitted, read until EOF.
|
|
|
|
Returns a string.
|
|
"""
|
|
self._unsupported("read")
|
|
|
|
def write(self, s):
|
|
"""Write string s to stream and returning an int."""
|
|
self._unsupported("write")
|
|
|
|
def truncate(self, pos=None):
|
|
"""Truncate size to pos, where pos is an int."""
|
|
self._unsupported("truncate")
|
|
|
|
def readline(self):
|
|
"""Read until newline or EOF.
|
|
|
|
Returns an empty string if EOF is hit immediately.
|
|
"""
|
|
self._unsupported("readline")
|
|
|
|
def detach(self):
|
|
"""
|
|
Separate the underlying buffer from the TextIOBase and return it.
|
|
|
|
After the underlying buffer has been detached, the TextIO is in an
|
|
unusable state.
|
|
"""
|
|
self._unsupported("detach")
|
|
|
|
@property
|
|
def encoding(self):
|
|
"""Subclasses should override."""
|
|
return None
|
|
|
|
@property
|
|
def newlines(self):
|
|
"""Line endings translated so far.
|
|
|
|
Only line endings translated during reading are considered.
|
|
|
|
Subclasses should override.
|
|
"""
|
|
return None
|
|
|
|
@property
|
|
def errors(self):
|
|
"""Error setting of the decoder or encoder.
|
|
|
|
Subclasses should override."""
|
|
return None
|
|
|
|
io.TextIOBase.register(TextIOBase)
|
|
|
|
|
|
class IncrementalNewlineDecoder(codecs.IncrementalDecoder):
|
|
r"""Codec used when reading a file in universal newlines mode. It wraps
|
|
another incremental decoder, translating \r\n and \r into \n. It also
|
|
records the types of newlines encountered. When used with
|
|
translate=False, it ensures that the newline sequence is returned in
|
|
one piece.
|
|
"""
|
|
def __init__(self, decoder, translate, errors='strict'):
|
|
codecs.IncrementalDecoder.__init__(self, errors=errors)
|
|
self.translate = translate
|
|
self.decoder = decoder
|
|
self.seennl = 0
|
|
self.pendingcr = False
|
|
|
|
def decode(self, input, final=False):
|
|
# decode input (with the eventual \r from a previous pass)
|
|
if self.decoder is None:
|
|
output = input
|
|
else:
|
|
output = self.decoder.decode(input, final=final)
|
|
if self.pendingcr and (output or final):
|
|
output = "\r" + output
|
|
self.pendingcr = False
|
|
|
|
# retain last \r even when not translating data:
|
|
# then readline() is sure to get \r\n in one pass
|
|
if output.endswith("\r") and not final:
|
|
output = output[:-1]
|
|
self.pendingcr = True
|
|
|
|
# Record which newlines are read
|
|
crlf = output.count('\r\n')
|
|
cr = output.count('\r') - crlf
|
|
lf = output.count('\n') - crlf
|
|
self.seennl |= (lf and self._LF) | (cr and self._CR) \
|
|
| (crlf and self._CRLF)
|
|
|
|
if self.translate:
|
|
if crlf:
|
|
output = output.replace("\r\n", "\n")
|
|
if cr:
|
|
output = output.replace("\r", "\n")
|
|
|
|
return output
|
|
|
|
def getstate(self):
|
|
if self.decoder is None:
|
|
buf = b""
|
|
flag = 0
|
|
else:
|
|
buf, flag = self.decoder.getstate()
|
|
flag <<= 1
|
|
if self.pendingcr:
|
|
flag |= 1
|
|
return buf, flag
|
|
|
|
def setstate(self, state):
|
|
buf, flag = state
|
|
self.pendingcr = bool(flag & 1)
|
|
if self.decoder is not None:
|
|
self.decoder.setstate((buf, flag >> 1))
|
|
|
|
def reset(self):
|
|
self.seennl = 0
|
|
self.pendingcr = False
|
|
if self.decoder is not None:
|
|
self.decoder.reset()
|
|
|
|
_LF = 1
|
|
_CR = 2
|
|
_CRLF = 4
|
|
|
|
@property
|
|
def newlines(self):
|
|
return (None,
|
|
"\n",
|
|
"\r",
|
|
("\r", "\n"),
|
|
"\r\n",
|
|
("\n", "\r\n"),
|
|
("\r", "\r\n"),
|
|
("\r", "\n", "\r\n")
|
|
)[self.seennl]
|
|
|
|
|
|
class TextIOWrapper(TextIOBase):
|
|
|
|
r"""Character and line based layer over a BufferedIOBase object, buffer.
|
|
|
|
encoding gives the name of the encoding that the stream will be
|
|
decoded or encoded with. It defaults to locale.getpreferredencoding.
|
|
|
|
errors determines the strictness of encoding and decoding (see the
|
|
codecs.register) and defaults to "strict".
|
|
|
|
newline can be None, '', '\n', '\r', or '\r\n'. It controls the
|
|
handling of line endings. If it is None, universal newlines is
|
|
enabled. With this enabled, on input, the lines endings '\n', '\r',
|
|
or '\r\n' are translated to '\n' before being returned to the
|
|
caller. Conversely, on output, '\n' is translated to the system
|
|
default line seperator, os.linesep. If newline is any other of its
|
|
legal values, that newline becomes the newline when the file is read
|
|
and it is returned untranslated. On output, '\n' is converted to the
|
|
newline.
|
|
|
|
If line_buffering is True, a call to flush is implied when a call to
|
|
write contains a newline character.
|
|
"""
|
|
|
|
_CHUNK_SIZE = 2048
|
|
|
|
def __init__(self, buffer, encoding=None, errors=None, newline=None,
|
|
line_buffering=False, write_through=False):
|
|
if newline is not None and not isinstance(newline, str):
|
|
raise TypeError("illegal newline type: %r" % (type(newline),))
|
|
if newline not in (None, "", "\n", "\r", "\r\n"):
|
|
raise ValueError("illegal newline value: %r" % (newline,))
|
|
if encoding is None:
|
|
try:
|
|
encoding = os.device_encoding(buffer.fileno())
|
|
except (AttributeError, UnsupportedOperation):
|
|
pass
|
|
if encoding is None:
|
|
try:
|
|
import locale
|
|
except ImportError:
|
|
# Importing locale may fail if Python is being built
|
|
encoding = "ascii"
|
|
else:
|
|
encoding = locale.getpreferredencoding()
|
|
|
|
if not isinstance(encoding, str):
|
|
raise ValueError("invalid encoding: %r" % encoding)
|
|
|
|
if errors is None:
|
|
errors = "strict"
|
|
else:
|
|
if not isinstance(errors, str):
|
|
raise ValueError("invalid errors: %r" % errors)
|
|
|
|
self._buffer = buffer
|
|
self._line_buffering = line_buffering
|
|
self._encoding = encoding
|
|
self._errors = errors
|
|
self._readuniversal = not newline
|
|
self._readtranslate = newline is None
|
|
self._readnl = newline
|
|
self._writetranslate = newline != ''
|
|
self._writenl = newline or os.linesep
|
|
self._encoder = None
|
|
self._decoder = None
|
|
self._decoded_chars = '' # buffer for text returned from decoder
|
|
self._decoded_chars_used = 0 # offset into _decoded_chars for read()
|
|
self._snapshot = None # info for reconstructing decoder state
|
|
self._seekable = self._telling = self.buffer.seekable()
|
|
self._has_read1 = hasattr(self.buffer, 'read1')
|
|
|
|
if self._seekable and self.writable():
|
|
position = self.buffer.tell()
|
|
if position != 0:
|
|
try:
|
|
self._get_encoder().setstate(0)
|
|
except LookupError:
|
|
# Sometimes the encoder doesn't exist
|
|
pass
|
|
|
|
# self._snapshot is either None, or a tuple (dec_flags, next_input)
|
|
# where dec_flags is the second (integer) item of the decoder state
|
|
# and next_input is the chunk of input bytes that comes next after the
|
|
# snapshot point. We use this to reconstruct decoder states in tell().
|
|
|
|
# Naming convention:
|
|
# - "bytes_..." for integer variables that count input bytes
|
|
# - "chars_..." for integer variables that count decoded characters
|
|
|
|
def __repr__(self):
|
|
result = "<_pyio.TextIOWrapper"
|
|
try:
|
|
name = self.name
|
|
except AttributeError:
|
|
pass
|
|
else:
|
|
result += " name={0!r}".format(name)
|
|
try:
|
|
mode = self.mode
|
|
except AttributeError:
|
|
pass
|
|
else:
|
|
result += " mode={0!r}".format(mode)
|
|
return result + " encoding={0!r}>".format(self.encoding)
|
|
|
|
@property
|
|
def encoding(self):
|
|
return self._encoding
|
|
|
|
@property
|
|
def errors(self):
|
|
return self._errors
|
|
|
|
@property
|
|
def line_buffering(self):
|
|
return self._line_buffering
|
|
|
|
@property
|
|
def buffer(self):
|
|
return self._buffer
|
|
|
|
def seekable(self):
|
|
return self._seekable
|
|
|
|
def readable(self):
|
|
return self.buffer.readable()
|
|
|
|
def writable(self):
|
|
return self.buffer.writable()
|
|
|
|
def flush(self):
|
|
self.buffer.flush()
|
|
self._telling = self._seekable
|
|
|
|
def close(self):
|
|
if self.buffer is not None and not self.closed:
|
|
self.flush()
|
|
self.buffer.close()
|
|
|
|
@property
|
|
def closed(self):
|
|
return self.buffer.closed
|
|
|
|
@property
|
|
def name(self):
|
|
return self.buffer.name
|
|
|
|
def fileno(self):
|
|
return self.buffer.fileno()
|
|
|
|
def isatty(self):
|
|
return self.buffer.isatty()
|
|
|
|
def write(self, s):
|
|
'Write data, where s is a str'
|
|
if self.closed:
|
|
raise ValueError("write to closed file")
|
|
if not isinstance(s, str):
|
|
raise TypeError("can't write %s to text stream" %
|
|
s.__class__.__name__)
|
|
length = len(s)
|
|
haslf = (self._writetranslate or self._line_buffering) and "\n" in s
|
|
if haslf and self._writetranslate and self._writenl != "\n":
|
|
s = s.replace("\n", self._writenl)
|
|
encoder = self._encoder or self._get_encoder()
|
|
# XXX What if we were just reading?
|
|
b = encoder.encode(s)
|
|
self.buffer.write(b)
|
|
if self._line_buffering and (haslf or "\r" in s):
|
|
self.flush()
|
|
self._snapshot = None
|
|
if self._decoder:
|
|
self._decoder.reset()
|
|
return length
|
|
|
|
def _get_encoder(self):
|
|
make_encoder = codecs.getincrementalencoder(self._encoding)
|
|
self._encoder = make_encoder(self._errors)
|
|
return self._encoder
|
|
|
|
def _get_decoder(self):
|
|
make_decoder = codecs.getincrementaldecoder(self._encoding)
|
|
decoder = make_decoder(self._errors)
|
|
if self._readuniversal:
|
|
decoder = IncrementalNewlineDecoder(decoder, self._readtranslate)
|
|
self._decoder = decoder
|
|
return decoder
|
|
|
|
# The following three methods implement an ADT for _decoded_chars.
|
|
# Text returned from the decoder is buffered here until the client
|
|
# requests it by calling our read() or readline() method.
|
|
def _set_decoded_chars(self, chars):
|
|
"""Set the _decoded_chars buffer."""
|
|
self._decoded_chars = chars
|
|
self._decoded_chars_used = 0
|
|
|
|
def _get_decoded_chars(self, n=None):
|
|
"""Advance into the _decoded_chars buffer."""
|
|
offset = self._decoded_chars_used
|
|
if n is None:
|
|
chars = self._decoded_chars[offset:]
|
|
else:
|
|
chars = self._decoded_chars[offset:offset + n]
|
|
self._decoded_chars_used += len(chars)
|
|
return chars
|
|
|
|
def _rewind_decoded_chars(self, n):
|
|
"""Rewind the _decoded_chars buffer."""
|
|
if self._decoded_chars_used < n:
|
|
raise AssertionError("rewind decoded_chars out of bounds")
|
|
self._decoded_chars_used -= n
|
|
|
|
def _read_chunk(self):
|
|
"""
|
|
Read and decode the next chunk of data from the BufferedReader.
|
|
"""
|
|
|
|
# The return value is True unless EOF was reached. The decoded
|
|
# string is placed in self._decoded_chars (replacing its previous
|
|
# value). The entire input chunk is sent to the decoder, though
|
|
# some of it may remain buffered in the decoder, yet to be
|
|
# converted.
|
|
|
|
if self._decoder is None:
|
|
raise ValueError("no decoder")
|
|
|
|
if self._telling:
|
|
# To prepare for tell(), we need to snapshot a point in the
|
|
# file where the decoder's input buffer is empty.
|
|
|
|
dec_buffer, dec_flags = self._decoder.getstate()
|
|
# Given this, we know there was a valid snapshot point
|
|
# len(dec_buffer) bytes ago with decoder state (b'', dec_flags).
|
|
|
|
# Read a chunk, decode it, and put the result in self._decoded_chars.
|
|
if self._has_read1:
|
|
input_chunk = self.buffer.read1(self._CHUNK_SIZE)
|
|
else:
|
|
input_chunk = self.buffer.read(self._CHUNK_SIZE)
|
|
eof = not input_chunk
|
|
self._set_decoded_chars(self._decoder.decode(input_chunk, eof))
|
|
|
|
if self._telling:
|
|
# At the snapshot point, len(dec_buffer) bytes before the read,
|
|
# the next input to be decoded is dec_buffer + input_chunk.
|
|
self._snapshot = (dec_flags, dec_buffer + input_chunk)
|
|
|
|
return not eof
|
|
|
|
def _pack_cookie(self, position, dec_flags=0,
|
|
bytes_to_feed=0, need_eof=0, chars_to_skip=0):
|
|
# The meaning of a tell() cookie is: seek to position, set the
|
|
# decoder flags to dec_flags, read bytes_to_feed bytes, feed them
|
|
# into the decoder with need_eof as the EOF flag, then skip
|
|
# chars_to_skip characters of the decoded result. For most simple
|
|
# decoders, tell() will often just give a byte offset in the file.
|
|
return (position | (dec_flags<<64) | (bytes_to_feed<<128) |
|
|
(chars_to_skip<<192) | bool(need_eof)<<256)
|
|
|
|
def _unpack_cookie(self, bigint):
|
|
rest, position = divmod(bigint, 1<<64)
|
|
rest, dec_flags = divmod(rest, 1<<64)
|
|
rest, bytes_to_feed = divmod(rest, 1<<64)
|
|
need_eof, chars_to_skip = divmod(rest, 1<<64)
|
|
return position, dec_flags, bytes_to_feed, need_eof, chars_to_skip
|
|
|
|
def tell(self):
|
|
if not self._seekable:
|
|
raise UnsupportedOperation("underlying stream is not seekable")
|
|
if not self._telling:
|
|
raise IOError("telling position disabled by next() call")
|
|
self.flush()
|
|
position = self.buffer.tell()
|
|
decoder = self._decoder
|
|
if decoder is None or self._snapshot is None:
|
|
if self._decoded_chars:
|
|
# This should never happen.
|
|
raise AssertionError("pending decoded text")
|
|
return position
|
|
|
|
# Skip backward to the snapshot point (see _read_chunk).
|
|
dec_flags, next_input = self._snapshot
|
|
position -= len(next_input)
|
|
|
|
# How many decoded characters have been used up since the snapshot?
|
|
chars_to_skip = self._decoded_chars_used
|
|
if chars_to_skip == 0:
|
|
# We haven't moved from the snapshot point.
|
|
return self._pack_cookie(position, dec_flags)
|
|
|
|
# Starting from the snapshot position, we will walk the decoder
|
|
# forward until it gives us enough decoded characters.
|
|
saved_state = decoder.getstate()
|
|
try:
|
|
# Note our initial start point.
|
|
decoder.setstate((b'', dec_flags))
|
|
start_pos = position
|
|
start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0
|
|
need_eof = 0
|
|
|
|
# Feed the decoder one byte at a time. As we go, note the
|
|
# nearest "safe start point" before the current location
|
|
# (a point where the decoder has nothing buffered, so seek()
|
|
# can safely start from there and advance to this location).
|
|
next_byte = bytearray(1)
|
|
for next_byte[0] in next_input:
|
|
bytes_fed += 1
|
|
chars_decoded += len(decoder.decode(next_byte))
|
|
dec_buffer, dec_flags = decoder.getstate()
|
|
if not dec_buffer and chars_decoded <= chars_to_skip:
|
|
# Decoder buffer is empty, so this is a safe start point.
|
|
start_pos += bytes_fed
|
|
chars_to_skip -= chars_decoded
|
|
start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0
|
|
if chars_decoded >= chars_to_skip:
|
|
break
|
|
else:
|
|
# We didn't get enough decoded data; signal EOF to get more.
|
|
chars_decoded += len(decoder.decode(b'', final=True))
|
|
need_eof = 1
|
|
if chars_decoded < chars_to_skip:
|
|
raise IOError("can't reconstruct logical file position")
|
|
|
|
# The returned cookie corresponds to the last safe start point.
|
|
return self._pack_cookie(
|
|
start_pos, start_flags, bytes_fed, need_eof, chars_to_skip)
|
|
finally:
|
|
decoder.setstate(saved_state)
|
|
|
|
def truncate(self, pos=None):
|
|
self.flush()
|
|
if pos is None:
|
|
pos = self.tell()
|
|
return self.buffer.truncate(pos)
|
|
|
|
def detach(self):
|
|
if self.buffer is None:
|
|
raise ValueError("buffer is already detached")
|
|
self.flush()
|
|
buffer = self._buffer
|
|
self._buffer = None
|
|
return buffer
|
|
|
|
def seek(self, cookie, whence=0):
|
|
if self.closed:
|
|
raise ValueError("tell on closed file")
|
|
if not self._seekable:
|
|
raise UnsupportedOperation("underlying stream is not seekable")
|
|
if whence == 1: # seek relative to current position
|
|
if cookie != 0:
|
|
raise UnsupportedOperation("can't do nonzero cur-relative seeks")
|
|
# Seeking to the current position should attempt to
|
|
# sync the underlying buffer with the current position.
|
|
whence = 0
|
|
cookie = self.tell()
|
|
if whence == 2: # seek relative to end of file
|
|
if cookie != 0:
|
|
raise UnsupportedOperation("can't do nonzero end-relative seeks")
|
|
self.flush()
|
|
position = self.buffer.seek(0, 2)
|
|
self._set_decoded_chars('')
|
|
self._snapshot = None
|
|
if self._decoder:
|
|
self._decoder.reset()
|
|
return position
|
|
if whence != 0:
|
|
raise ValueError("invalid whence (%r, should be 0, 1 or 2)" %
|
|
(whence,))
|
|
if cookie < 0:
|
|
raise ValueError("negative seek position %r" % (cookie,))
|
|
self.flush()
|
|
|
|
# The strategy of seek() is to go back to the safe start point
|
|
# and replay the effect of read(chars_to_skip) from there.
|
|
start_pos, dec_flags, bytes_to_feed, need_eof, chars_to_skip = \
|
|
self._unpack_cookie(cookie)
|
|
|
|
# Seek back to the safe start point.
|
|
self.buffer.seek(start_pos)
|
|
self._set_decoded_chars('')
|
|
self._snapshot = None
|
|
|
|
# Restore the decoder to its state from the safe start point.
|
|
if cookie == 0 and self._decoder:
|
|
self._decoder.reset()
|
|
elif self._decoder or dec_flags or chars_to_skip:
|
|
self._decoder = self._decoder or self._get_decoder()
|
|
self._decoder.setstate((b'', dec_flags))
|
|
self._snapshot = (dec_flags, b'')
|
|
|
|
if chars_to_skip:
|
|
# Just like _read_chunk, feed the decoder and save a snapshot.
|
|
input_chunk = self.buffer.read(bytes_to_feed)
|
|
self._set_decoded_chars(
|
|
self._decoder.decode(input_chunk, need_eof))
|
|
self._snapshot = (dec_flags, input_chunk)
|
|
|
|
# Skip chars_to_skip of the decoded characters.
|
|
if len(self._decoded_chars) < chars_to_skip:
|
|
raise IOError("can't restore logical file position")
|
|
self._decoded_chars_used = chars_to_skip
|
|
|
|
# Finally, reset the encoder (merely useful for proper BOM handling)
|
|
try:
|
|
encoder = self._encoder or self._get_encoder()
|
|
except LookupError:
|
|
# Sometimes the encoder doesn't exist
|
|
pass
|
|
else:
|
|
if cookie != 0:
|
|
encoder.setstate(0)
|
|
else:
|
|
encoder.reset()
|
|
return cookie
|
|
|
|
def read(self, n=None):
|
|
self._checkReadable()
|
|
if n is None:
|
|
n = -1
|
|
decoder = self._decoder or self._get_decoder()
|
|
try:
|
|
n.__index__
|
|
except AttributeError as err:
|
|
raise TypeError("an integer is required") from err
|
|
if n < 0:
|
|
# Read everything.
|
|
result = (self._get_decoded_chars() +
|
|
decoder.decode(self.buffer.read(), final=True))
|
|
self._set_decoded_chars('')
|
|
self._snapshot = None
|
|
return result
|
|
else:
|
|
# Keep reading chunks until we have n characters to return.
|
|
eof = False
|
|
result = self._get_decoded_chars(n)
|
|
while len(result) < n and not eof:
|
|
eof = not self._read_chunk()
|
|
result += self._get_decoded_chars(n - len(result))
|
|
return result
|
|
|
|
def __next__(self):
|
|
self._telling = False
|
|
line = self.readline()
|
|
if not line:
|
|
self._snapshot = None
|
|
self._telling = self._seekable
|
|
raise StopIteration
|
|
return line
|
|
|
|
def readline(self, limit=None):
|
|
if self.closed:
|
|
raise ValueError("read from closed file")
|
|
if limit is None:
|
|
limit = -1
|
|
elif not isinstance(limit, int):
|
|
raise TypeError("limit must be an integer")
|
|
|
|
# Grab all the decoded text (we will rewind any extra bits later).
|
|
line = self._get_decoded_chars()
|
|
|
|
start = 0
|
|
# Make the decoder if it doesn't already exist.
|
|
if not self._decoder:
|
|
self._get_decoder()
|
|
|
|
pos = endpos = None
|
|
while True:
|
|
if self._readtranslate:
|
|
# Newlines are already translated, only search for \n
|
|
pos = line.find('\n', start)
|
|
if pos >= 0:
|
|
endpos = pos + 1
|
|
break
|
|
else:
|
|
start = len(line)
|
|
|
|
elif self._readuniversal:
|
|
# Universal newline search. Find any of \r, \r\n, \n
|
|
# The decoder ensures that \r\n are not split in two pieces
|
|
|
|
# In C we'd look for these in parallel of course.
|
|
nlpos = line.find("\n", start)
|
|
crpos = line.find("\r", start)
|
|
if crpos == -1:
|
|
if nlpos == -1:
|
|
# Nothing found
|
|
start = len(line)
|
|
else:
|
|
# Found \n
|
|
endpos = nlpos + 1
|
|
break
|
|
elif nlpos == -1:
|
|
# Found lone \r
|
|
endpos = crpos + 1
|
|
break
|
|
elif nlpos < crpos:
|
|
# Found \n
|
|
endpos = nlpos + 1
|
|
break
|
|
elif nlpos == crpos + 1:
|
|
# Found \r\n
|
|
endpos = crpos + 2
|
|
break
|
|
else:
|
|
# Found \r
|
|
endpos = crpos + 1
|
|
break
|
|
else:
|
|
# non-universal
|
|
pos = line.find(self._readnl)
|
|
if pos >= 0:
|
|
endpos = pos + len(self._readnl)
|
|
break
|
|
|
|
if limit >= 0 and len(line) >= limit:
|
|
endpos = limit # reached length limit
|
|
break
|
|
|
|
# No line ending seen yet - get more data'
|
|
while self._read_chunk():
|
|
if self._decoded_chars:
|
|
break
|
|
if self._decoded_chars:
|
|
line += self._get_decoded_chars()
|
|
else:
|
|
# end of file
|
|
self._set_decoded_chars('')
|
|
self._snapshot = None
|
|
return line
|
|
|
|
if limit >= 0 and endpos > limit:
|
|
endpos = limit # don't exceed limit
|
|
|
|
# Rewind _decoded_chars to just after the line ending we found.
|
|
self._rewind_decoded_chars(len(line) - endpos)
|
|
return line[:endpos]
|
|
|
|
@property
|
|
def newlines(self):
|
|
return self._decoder.newlines if self._decoder else None
|
|
|
|
|
|
class StringIO(TextIOWrapper):
|
|
"""Text I/O implementation using an in-memory buffer.
|
|
|
|
The initial_value argument sets the value of object. The newline
|
|
argument is like the one of TextIOWrapper's constructor.
|
|
"""
|
|
|
|
def __init__(self, initial_value="", newline="\n"):
|
|
super(StringIO, self).__init__(BytesIO(),
|
|
encoding="utf-8",
|
|
errors="strict",
|
|
newline=newline)
|
|
# Issue #5645: make universal newlines semantics the same as in the
|
|
# C version, even under Windows.
|
|
if newline is None:
|
|
self._writetranslate = False
|
|
if initial_value is not None:
|
|
if not isinstance(initial_value, str):
|
|
raise TypeError("initial_value must be str or None, not {0}"
|
|
.format(type(initial_value).__name__))
|
|
initial_value = str(initial_value)
|
|
self.write(initial_value)
|
|
self.seek(0)
|
|
|
|
def getvalue(self):
|
|
self.flush()
|
|
return self.buffer.getvalue().decode(self._encoding, self._errors)
|
|
|
|
def __repr__(self):
|
|
# TextIOWrapper tells the encoding in its repr. In StringIO,
|
|
# that's a implementation detail.
|
|
return object.__repr__(self)
|
|
|
|
@property
|
|
def errors(self):
|
|
return None
|
|
|
|
@property
|
|
def encoding(self):
|
|
return None
|
|
|
|
def detach(self):
|
|
# This doesn't make sense on StringIO.
|
|
self._unsupported("detach")
|