mirrors/cpython
0
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2024-11-27 11:55:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merged revisions 64842,64853,64856,64945 via svnmerge from

Browse Source 
svn+ssh://pythondev@svn.python.org/python/trunk

........
  r64842 | robert.schuppenies | 2008-07-10 15:43:26 +0200 (Thu, 10 Jul 2008) | 2 lines

  Fixed Issue3122 and extended sys.getsizeof tests for built-in types.
........
  r64853 | robert.schuppenies | 2008-07-10 17:24:04 +0200 (Thu, 10 Jul 2008) | 3 lines

  Added additional __sizeof__ implementations and addressed comments made in
  Issue3122.
........
  r64856 | robert.schuppenies | 2008-07-10 19:13:55 +0200 (Thu, 10 Jul 2008) | 3 lines

  Added garbage collector overhead and optional default return value to
  sys.getsizeof.
........
  r64945 | robert.schuppenies | 2008-07-14 10:42:18 +0200 (Mon, 14 Jul 2008) | 2 lines

  Fixed test failure on Win64 machines.
........
This commit is contained in:
Robert Schuppenies 2008-07-14 10:13:31 +00:00
parent 3065b87a07
commit fbe94c55ca
9 changed files with 342 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Doc/library/sys.rst
View File

@ -331,13 +331,20 @@ always available.
:func:`setrecursionlimit`.
.. function:: getsizeof(object)
.. function:: getsizeof(object[, default])
Return the size of an object in bytes. The object can be any type of
object. All built-in objects will return correct results, but this
does not have to hold true for third-party extensions as it is implementation
does not have to hold true for third-party extensions as it is implementation
specific.
The *default* argument allows to define a value which will be returned
if the object type does not provide means to retrieve the size and would
cause a `TypeError`.
func:`getsizeof` calls the object's __sizeof__ method and adds an additional
garbage collector overhead if the object is managed by the garbage collector.
.. versionadded:: 2.6

295
Lib/test/test_sys.py
View File

@ -373,6 +373,9 @@ class SysModuleTest(unittest.TestCase):
class SizeofTest(unittest.TestCase):
TPFLAGS_HAVE_GC = 1<<14
TPFLAGS_HEAPTYPE = 1<<9
def setUp(self):
self.c = len(struct.pack('c', ' '))
self.H = len(struct.pack('H', 0))
@ -382,22 +385,27 @@ class SizeofTest(unittest.TestCase):
# due to missing size_t information from struct, it is assumed that
# sizeof(Py_ssize_t) = sizeof(void*)
self.header = 'PP'
self.vheader = self.header + 'P'
if hasattr(sys, "gettotalrefcount"):
self.header += '2P'
self.vheader += '2P'
import _testcapi
self.gc_headsize = _testcapi.SIZEOF_PYGC_HEAD
self.file = open(test.support.TESTFN, 'wb')
def tearDown(self):
self.file.close()
test.support.unlink(test.support.TESTFN)
def check_sizeof(self, o, size, size2=None):
"""Check size of o. Possible are size and optionally size2)."""
def check_sizeof(self, o, size):
result = sys.getsizeof(o)
msg = 'wrong size for %s: got %d, expected ' % (type(o), result)
if (size2 != None) and (result != size):
self.assertEqual(result, size2, msg + str(size2))
else:
self.assertEqual(result, size, msg + str(size))
# add GC header size
if ((type(o) == type) and (o.__flags__ & self.TPFLAGS_HEAPTYPE) or\
((type(o) != type) and (type(o).__flags__ & self.TPFLAGS_HAVE_GC))):
size += self.gc_headsize
msg = 'wrong size for %s: got %d, expected %d' \
% (type(o), result, size)
self.assertEqual(result, size, msg)
def calcsize(self, fmt):
"""Wrapper around struct.calcsize which enforces the alignment of the
@ -408,29 +416,116 @@ class SizeofTest(unittest.TestCase):
"""
return struct.calcsize(fmt + '0P')
def test_standardtypes(self):
def test_gc_head_size(self):
# Check that the gc header size is added to objects tracked by the gc.
h = self.header
vh = self.vheader
size = self.calcsize
gc_header_size = self.gc_headsize
# bool objects are not gc tracked
self.assertEqual(sys.getsizeof(True), size(vh) + self.H)
# but lists are
self.assertEqual(sys.getsizeof([]), size(vh + 'PP') + gc_header_size)
def test_default(self):
h = self.header
vh = self.vheader
size = self.calcsize
self.assertEqual(sys.getsizeof(True), size(vh) + self.H)
self.assertEqual(sys.getsizeof(True, -1), size(vh) + self.H)
def test_objecttypes(self):
# check all types defined in Objects/
h = self.header
vh = self.vheader
size = self.calcsize
check = self.check_sizeof
# bool
check(True, size(vh) + self.H)
# buffer
# XXX
# builtin_function_or_method
check(len, size(h + '3P'))
# bytearray
samples = [b'', b'u'*100000]
for sample in samples:
x = bytearray(sample)
check(x, size(vh + 'iPP') + x.__alloc__() * self.c)
# bytearray_iterator
check(iter(bytearray()), size(h + 'PP'))
# cell
def get_cell():
x = 42
def inner():
return x
return inner
self.check_sizeof(get_cell().__closure__[0], size(h + 'P'))
check(get_cell().__closure__[0], size(h + 'P'))
# code
self.check_sizeof(get_cell().__code__, size(h + '5i8Pi2P'))
check(get_cell().__code__, size(h + '5i8Pi2P'))
# complex
self.check_sizeof(complex(0,1), size(h + '2d'))
check(complex(0,1), size(h + '2d'))
# method_descriptor (descriptor object)
check(str.lower, size(h + '2PP'))
# classmethod_descriptor (descriptor object)
# XXX
# member_descriptor (descriptor object)
import datetime
check(datetime.timedelta.days, size(h + '2PP'))
# getset_descriptor (descriptor object)
import collections
check(collections.defaultdict.default_factory, size(h + '2PP'))
# wrapper_descriptor (descriptor object)
check(int.__add__, size(h + '2P2P'))
# method-wrapper (descriptor object)
check({}.__iter__, size(h + '2P'))
# dict
check({}, size(h + '3P2P' + 8*'P2P'))
longdict = {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8}
check(longdict, size(h + '3P2P' + 8*'P2P') + 16*size('P2P'))
# dictionary-keyiterator
check({}.keys(), size(h + 'P'))
# dictionary-valueiterator
check({}.values(), size(h + 'P'))
# dictionary-itemiterator
check({}.items(), size(h + 'P'))
# dictproxy
class C(object): pass
check(C.__dict__, size(h + 'P'))
# BaseException
check(BaseException(), size(h + '5P'))
# UnicodeEncodeError
check(UnicodeEncodeError("", "", 0, 0, ""), size(h + '5P 2P2PP'))
# UnicodeDecodeError
# XXX
# check(UnicodeDecodeError("", "", 0, 0, ""), size(h + '5P2PP'))
# UnicodeTranslateError
check(UnicodeTranslateError("", 0, 1, ""), size(h + '5P 2P2PP'))
# ellipses
check(Ellipsis, size(h + ''))
# EncodingMap
import codecs, encodings.iso8859_3
x = codecs.charmap_build(encodings.iso8859_3.decoding_table)
check(x, size(h + '32B2iB'))
# enumerate
self.check_sizeof(enumerate([]), size(h + 'l3P'))
check(enumerate([]), size(h + 'l3P'))
# reverse
self.check_sizeof(reversed(''), size(h + 'PP'))
check(reversed(''), size(h + 'PP'))
# float
self.check_sizeof(float(0), size(h + 'd'))
check(float(0), size(h + 'd'))
# sys.floatinfo
check(sys.float_info, size(vh) + self.P * len(sys.float_info))
# frame
import inspect
CO_MAXBLOCKS = 20
x = inspect.currentframe()
ncells = len(x.f_code.co_cellvars)
nfrees = len(x.f_code.co_freevars)
extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
ncells + nfrees - 1
check(x, size(vh + '12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
# function
def func(): pass
self.check_sizeof(func, size(h + '11P'))
check(func, size(h + '11P'))
class c():
@staticmethod
def foo():
@ -439,40 +534,99 @@ class SizeofTest(unittest.TestCase):
def bar(cls):
pass
# staticmethod
self.check_sizeof(foo, size(h + 'P'))
check(foo, size(h + 'P'))
# classmethod
self.check_sizeof(bar, size(h + 'P'))
check(bar, size(h + 'P'))
# generator
def get_gen(): yield 1
self.check_sizeof(get_gen(), size(h + 'Pi2P'))
# builtin_function_or_method
self.check_sizeof(abs, size(h + '3P'))
check(get_gen(), size(h + 'Pi2P'))
# iterator
check(iter('abc'), size(h + 'lP'))
# callable-iterator
import re
check(re.finditer('',''), size(h + '2P'))
# list
samples = [[], [1,2,3], ['1', '2', '3']]
for sample in samples:
check(sample, size(vh + 'PP') + len(sample)*self.P)
# sortwrapper (list)
# XXX
# cmpwrapper (list)
# XXX
# listiterator (list)
check(iter([]), size(h + 'lP'))
# listreverseiterator (list)
check(reversed([]), size(h + 'lP'))
# long
check(0, size(vh))
check(1, size(vh) + self.H)
check(-1, size(vh) + self.H)
check(32768, size(vh) + 2*self.H)
check(32768*32768-1, size(vh) + 2*self.H)
check(32768*32768, size(vh) + 3*self.H)
# memory
check(memoryview(b''), size(h + 'P P2P2i5P'))
# module
self.check_sizeof(unittest, size(h + '3P'))
check(unittest, size(h + '3P'))
# None
check(None, size(h + ''))
# NotImplementedType
check(NotImplemented, size(h))
# object
check(object(), size(h + ''))
# property (descriptor object)
class C(object):
def getx(self): return self.__x
def setx(self, value): self.__x = value
def delx(self): del self.__x
x = property(getx, setx, delx, "")
check(x, size(h + '4Pi'))
# PyCObject
# XXX
# rangeiterator
check(iter(range(1)), size(h + '4l'))
# reverse
check(reversed(''), size(h + 'PP'))
# range
self.check_sizeof(range(1), size(h + '3P'))
check(range(1), size(h + '3P'))
check(range(66000), size(h + '3l'))
# set
# frozenset
PySet_MINSIZE = 8
samples = [[], range(10), range(50)]
s = size(h + '3P2P' + PySet_MINSIZE*'lP' + 'lP')
for sample in samples:
minused = len(sample)
if minused == 0: tmp = 1
# the computation of minused is actually a bit more complicated
# but this suffices for the sizeof test
minused = minused*2
newsize = PySet_MINSIZE
while newsize <= minused:
newsize = newsize << 1
if newsize <= 8:
check(set(sample), s)
check(frozenset(sample), s)
else:
check(set(sample), s + newsize*struct.calcsize('lP'))
check(frozenset(sample), s + newsize*struct.calcsize('lP'))
# setiterator
check(iter(set()), size(h + 'P3P'))
# slice
self.check_sizeof(slice(0), size(h + '3P'))
h += 'P'
# bool
self.check_sizeof(True, size(h + 'H'))
# new-style class
class class_newstyle(object):
def method():
pass
# type (PyTypeObject + PyNumberMethods + PyMappingMethods +
# PySequenceMethods + PyBufferProcs)
self.check_sizeof(class_newstyle, size(h + 'P2P15Pl4PP9PP11PI') +\
size('16Pi17P 3P 10P 2P 2P'))
def test_specialtypes(self):
h = self.header
size = self.calcsize
# dict
self.check_sizeof({}, size(h + '3P2P') + 8*size('P2P'))
longdict = {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8}
self.check_sizeof(longdict, size(h + '3P2P') + (8+16)*size('P2P'))
check(slice(0), size(h + '3P'))
# super
check(super(int), size(h + '3P'))
# tuple
check((), size(vh))
check((1,2,3), size(vh) + 3*self.P)
# type
# (PyTypeObject + PyNumberMethods + PyMappingMethods +
# PySequenceMethods + PyBufferProcs)
s = size(vh + 'P2P15Pl4PP9PP11PI') + size('16Pi17P 3P 10P 2P 2P')
check(int, s)
# class
class newstyleclass(object): pass
check(newstyleclass, s)
# unicode
usize = len('\0'.encode('unicode-internal'))
samples = ['', '1'*100]
@ -480,31 +634,38 @@ class SizeofTest(unittest.TestCase):
# has been cached
for s in samples:
basicsize = size(h + 'PPliP') + usize * (len(s) + 1)
defenc = bytes(s, 'ascii')
self.check_sizeof(s, basicsize,
size2=basicsize + sys.getsizeof(defenc))
# trigger caching encoded version as bytes object
try:
getattr(sys, s)
except AttributeError:
pass
finally:
self.check_sizeof(s, basicsize + sys.getsizeof(defenc))
check(s, basicsize)
# weakref
import weakref
check(weakref.ref(int), size(h + '2Pl2P'))
# weakproxy
# XXX
# weakcallableproxy
check(weakref.proxy(int), size(h + '2Pl2P'))
h += 'P'
# list
self.check_sizeof([], size(h + 'PP'))
self.check_sizeof([1, 2, 3], size(h + 'PP') + 3*self.P)
# long
self.check_sizeof(0, size(h + 'H'))
self.check_sizeof(1, size(h + 'H'))
self.check_sizeof(-1, size(h + 'H'))
self.check_sizeof(32768, size(h + 'H') + self.H)
self.check_sizeof(32768*32768-1, size(h + 'H') + self.H)
self.check_sizeof(32768*32768, size(h + 'H') + 2*self.H)
# tuple
self.check_sizeof((), size(h))
self.check_sizeof((1,2,3), size(h) + 3*self.P)
def test_pythontypes(self):
# check all types defined in Python/
h = self.header
vh = self.vheader
size = self.calcsize
check = self.check_sizeof
# _ast.AST
import _ast
check(_ast.AST(), size(h + ''))
# imp.NullImporter
import imp
check(imp.NullImporter(self.file.name), size(h + ''))
try:
raise TypeError
except TypeError:
tb = sys.exc_info()[2]
# traceback
if tb != None:
check(tb, size(h + '2P2i'))
# symtable entry
# XXX
# sys.flags
check(sys.flags, size(vh) + self.P * len(sys.flags))
def test_main():

1
Modules/_testcapimodule.c
View File

@ -1182,6 +1182,7 @@ PyInit__testcapi(void)
PyModule_AddObject(m, "ULLONG_MAX", PyLong_FromUnsignedLongLong(PY_ULLONG_MAX));
PyModule_AddObject(m, "PY_SSIZE_T_MAX", PyLong_FromSsize_t(PY_SSIZE_T_MAX));
PyModule_AddObject(m, "PY_SSIZE_T_MIN", PyLong_FromSsize_t(PY_SSIZE_T_MIN));
PyModule_AddObject(m, "SIZEOF_PYGC_HEAD", PyLong_FromSsize_t(sizeof(PyGC_Head)));
TestError = PyErr_NewException("_testcapi.error", NULL, NULL);
Py_INCREF(TestError);

14
Objects/bytearrayobject.c
View File

@ -3033,6 +3033,19 @@ bytes_reduce(PyByteArrayObject *self)
return Py_BuildValue("(O(Ns)N)", Py_TYPE(self), latin1, "latin-1", dict);
}
PyDoc_STRVAR(sizeof_doc,
"B.__sizeof__() -> int\n\
\n\
Returns the size of B in memory, in bytes");
static PyObject *
bytes_sizeof(PyByteArrayObject *self)
{
Py_ssize_t res;
res = sizeof(PyByteArrayObject) + self->ob_alloc * sizeof(char);
return PyLong_FromSsize_t(res);
}
static PySequenceMethods bytes_as_sequence = {
(lenfunc)bytes_length, /* sq_length */
(binaryfunc)PyByteArray_Concat, /* sq_concat */
@ -3061,6 +3074,7 @@ static PyMethodDef
bytes_methods[] = {
{"__alloc__", (PyCFunction)bytes_alloc, METH_NOARGS, alloc_doc},
{"__reduce__", (PyCFunction)bytes_reduce, METH_NOARGS, reduce_doc},
{"__sizeof__", (PyCFunction)bytes_sizeof, METH_NOARGS, sizeof_doc},
{"append", (PyCFunction)bytes_append, METH_O, append__doc__},
{"capitalize", (PyCFunction)stringlib_capitalize, METH_NOARGS,
_Py_capitalize__doc__},

25
Objects/frameobject.c
View File

@ -513,6 +513,29 @@ frame_clear(PyFrameObject *f)
}
}
static PyObject *
frame_sizeof(PyFrameObject *f)
{
Py_ssize_t res, extras, ncells, nfrees;
ncells = PyTuple_GET_SIZE(f->f_code->co_cellvars);
nfrees = PyTuple_GET_SIZE(f->f_code->co_freevars);
extras = f->f_code->co_stacksize + f->f_code->co_nlocals +
ncells + nfrees;
// subtract one as it is already included in PyFrameObject
res = sizeof(PyFrameObject) + (extras-1) * sizeof(PyObject *);
return PyLong_FromSsize_t(res);
}
PyDoc_STRVAR(sizeof__doc__,
"F.__sizeof__() -> size of F in memory, in bytes");
static PyMethodDef frame_methods[] = {
{"__sizeof__", (PyCFunction)frame_sizeof, METH_NOARGS,
sizeof__doc__},
{NULL, NULL} /* sentinel */
};
PyTypeObject PyFrame_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
@ -542,7 +565,7 @@ PyTypeObject PyFrame_Type = {
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
0, /* tp_methods */
frame_methods, /* tp_methods */
frame_memberlist, /* tp_members */
frame_getsetlist, /* tp_getset */
0, /* tp_base */

4
Objects/longobject.c
View File

@ -3638,9 +3638,7 @@ long_sizeof(PyLongObject *v)
{
Py_ssize_t res;
res = sizeof(PyLongObject) + abs(Py_SIZE(v)) * sizeof(digit);
if (Py_SIZE(v) != 0)
res -= sizeof(digit);
res = sizeof(PyVarObject) + abs(Py_SIZE(v))*sizeof(digit);
return PyLong_FromSsize_t(res);
}

16
Objects/setobject.c
View File

@ -1944,6 +1944,18 @@ done:
PyDoc_STRVAR(reduce_doc, "Return state information for pickling.");
static PyObject *
set_sizeof(PySetObject *so)
{
Py_ssize_t res;
res = sizeof(PySetObject);
if (so->table != so->smalltable)
res = res + (so->mask + 1) * sizeof(setentry);
return PyLong_FromSsize_t(res);
}
PyDoc_STRVAR(sizeof_doc, "S.__sizeof__() -> size of S in memory, in bytes");
static int
set_init(PySetObject *self, PyObject *args, PyObject *kwds)
{
@ -2011,6 +2023,8 @@ static PyMethodDef set_methods[] = {
reduce_doc},
{"remove", (PyCFunction)set_remove, METH_O,
remove_doc},
{"__sizeof__", (PyCFunction)set_sizeof, METH_NOARGS,
sizeof_doc},
{"symmetric_difference",(PyCFunction)set_symmetric_difference, METH_O,
symmetric_difference_doc},
{"symmetric_difference_update",(PyCFunction)set_symmetric_difference_update, METH_O,
@ -2127,6 +2141,8 @@ static PyMethodDef frozenset_methods[] = {
issuperset_doc},
{"__reduce__", (PyCFunction)set_reduce, METH_NOARGS,
reduce_doc},
{"__sizeof__", (PyCFunction)set_sizeof, METH_NOARGS,
sizeof_doc},
{"symmetric_difference",(PyCFunction)set_symmetric_difference, METH_O,
symmetric_difference_doc},
{"union", (PyCFunction)set_union, METH_VARARGS,

16
Objects/unicodeobject.c
View File

@ -8350,20 +8350,8 @@ PyDoc_STRVAR(p_format__doc__,
static PyObject *
unicode__sizeof__(PyUnicodeObject *v)
{
PyObject *res = NULL, *defsize = NULL;
res = PyLong_FromSsize_t(sizeof(PyUnicodeObject) +
sizeof(Py_UNICODE) * (v->length + 1));
if (v->defenc) {
defsize = PyObject_CallMethod(v->defenc, "__sizeof__", NULL);
if (defsize == NULL) {
Py_DECREF(res);
return NULL;
}
res = PyNumber_Add(res, defsize);
Py_DECREF(defsize);
}
return res;
return PyLong_FromSsize_t(sizeof(PyUnicodeObject) +
sizeof(Py_UNICODE) * (v->length + 1));
}
PyDoc_STRVAR(sizeof__doc__,

63
Python/sysmodule.c
View File

@ -610,9 +610,16 @@ sys_mdebug(PyObject *self, PyObject *args)
#endif /* USE_MALLOPT */
static PyObject *
sys_getsizeof(PyObject *self, PyObject *args)
sys_getsizeof(PyObject *self, PyObject *args, PyObject *kwds)
{
static PyObject * str__sizeof__ = NULL;
PyObject *res = NULL;
static PyObject *str__sizeof__, *gc_head_size = NULL;
static char *kwlist[] = {"object", "default", 0};
PyObject *o, *dflt = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:getsizeof",
kwlist, &o, &dflt))
return NULL;
/* Initialize static variable needed by _PyType_Lookup */
if (str__sizeof__ == NULL) {
@ -621,24 +628,47 @@ sys_getsizeof(PyObject *self, PyObject *args)
return NULL;
}
/* Type objects */
if (PyType_Check(args)){
PyObject *method = _PyType_Lookup(Py_TYPE(args),
str__sizeof__);
if (method == NULL) {
PyErr_Format(PyExc_TypeError,
"Type %.100s doesn't define __sizeof__",
Py_TYPE(args)->tp_name);
/* Initialize static variable for GC head size */
if (gc_head_size == NULL) {
gc_head_size = PyLong_FromSsize_t(sizeof(PyGC_Head));
if (gc_head_size == NULL)
return NULL;
}
return PyObject_CallFunctionObjArgs(method, args, NULL);
}
}
/* Make sure the type is initialized. float gets initialized late */
if (PyType_Ready(Py_TYPE(o)) < 0)
return NULL;
PyObject *method = _PyType_Lookup(Py_TYPE(o), str__sizeof__);
if (method == NULL)
PyErr_Format(PyExc_TypeError,
"Type %.100s doesn't define __sizeof__",
Py_TYPE(o)->tp_name);
else
return PyObject_CallMethod(args, "__sizeof__", NULL);
res = PyObject_CallFunctionObjArgs(method, o, NULL);
/* Has a default value been given */
if ((res == NULL) && (dflt != NULL) &&
PyErr_ExceptionMatches(PyExc_TypeError))
{
PyErr_Clear();
Py_INCREF(dflt);
return dflt;
}
else if (res == NULL)
return res;
/* add gc_head size */
if (PyObject_IS_GC(o)) {
PyObject *tmp = res;
res = PyNumber_Add(tmp, gc_head_size);
Py_DECREF(tmp);
}
return res;
}
PyDoc_STRVAR(getsizeof_doc,
"getsizeof(object) -> int\n\
"getsizeof(object, default) -> int\n\
\n\
Return the size of object in bytes.");
@ -845,7 +875,8 @@ static PyMethodDef sys_methods[] = {
{"getrefcount", (PyCFunction)sys_getrefcount, METH_O, getrefcount_doc},
{"getrecursionlimit", (PyCFunction)sys_getrecursionlimit, METH_NOARGS,
getrecursionlimit_doc},
{"getsizeof", sys_getsizeof, METH_O, getsizeof_doc},
{"getsizeof", (PyCFunction)sys_getsizeof,
METH_VARARGS | METH_KEYWORDS, getsizeof_doc},
{"_getframe", sys_getframe, METH_VARARGS, getframe_doc},
#ifdef MS_WINDOWS
{"getwindowsversion", (PyCFunction)sys_getwindowsversion, METH_NOARGS,