mirrors/cpython
0
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2024-11-28 04:15:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

gh-109979: Auto-generate the target for DEOPT_IF() (#110193)

Browse Source 
In Python/bytecodes.c, you now write
```
    DEOPT_IF(condition);
```
The code generator expands this to
```
    DEOPT_IF(condition, opcode);
```
where `opcode` is the name of the unspecialized instruction.
This works inside macro expansions too.

**CAVEAT:** The entire `DEOPT_IF(condition)` statement must be on a single line.
If it isn't, the substitution will fail; an error will be printed by the code generator
and the C compiler will report some errors.
This commit is contained in:
Guido van Rossum 2023-10-03 10:13:50 -07:00 committed by GitHub
parent d73501602f
commit d67edcf0b3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 315 additions and 315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Lib/test/test_generated_cases.py
View File

@ -239,16 +239,22 @@ class TestGeneratedCases(unittest.TestCase):
def test_predictions_and_eval_breaker(self):
input = """
inst(OP1, (--)) {
inst(OP1, (arg -- rest)) {
}
inst(OP3, (arg -- res)) {
DEOPT_IF(xxx, OP1);
DEOPT_IF(xxx);
CHECK_EVAL_BREAKER();
}
family(OP1, INLINE_CACHE_ENTRIES_OP1) = { OP3 };
"""
output = """
TARGET(OP1) {
PREDICTED(OP1);
static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size");
PyObject *arg;
PyObject *rest;
arg = stack_pointer[-1];
stack_pointer[-1] = rest;
DISPATCH();
}
@ -371,6 +377,7 @@ class TestGeneratedCases(unittest.TestCase):
}
TARGET(OP) {
PREDICTED(OP);
static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size");
PyObject *right;
PyObject *left;

352
Python/bytecodes.c
View File

@ -41,8 +41,6 @@
#include "ceval_macros.h"
/* Flow control macros */
#define DEOPT_IF(cond, instname) ((void)0)
#define ERROR_IF(cond, labelname) ((void)0)
#define GO_TO_INSTRUCTION(instname) ((void)0)
#define inst(name, ...) case name:
@ -153,13 +151,12 @@ dummy_func(
inst(RESUME_CHECK, (--)) {
#if defined(__EMSCRIPTEN__)
DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME);
DEOPT_IF(_Py_emscripten_signal_clock == 0);
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
/* Possibly combine these two checks */
DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version
!= tstate->interp->monitoring_version, RESUME);
DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME);
DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version);
DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker));
}
inst(INSTRUMENTED_RESUME, (--)) {
@ -325,12 +322,12 @@ dummy_func(
}
inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) {
DEOPT_IF(!PyBool_Check(value), TO_BOOL);
DEOPT_IF(!PyBool_Check(value));
STAT_INC(TO_BOOL, hit);
}
inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) {
DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL);
DEOPT_IF(!PyLong_CheckExact(value));
STAT_INC(TO_BOOL, hit);
if (_PyLong_IsZero((PyLongObject *)value)) {
assert(_Py_IsImmortal(value));
@ -343,7 +340,7 @@ dummy_func(
}
inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) {
DEOPT_IF(!PyList_CheckExact(value), TO_BOOL);
DEOPT_IF(!PyList_CheckExact(value));
STAT_INC(TO_BOOL, hit);
res = Py_SIZE(value) ? Py_True : Py_False;
DECREF_INPUTS();
@ -351,13 +348,13 @@ dummy_func(
inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) {
// This one is a bit weird, because we expect *some* failures:
DEOPT_IF(!Py_IsNone(value), TO_BOOL);
DEOPT_IF(!Py_IsNone(value));
STAT_INC(TO_BOOL, hit);
res = Py_False;
}
inst(TO_BOOL_STR, (unused/1, unused/2, value -- res)) {
DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL);
DEOPT_IF(!PyUnicode_CheckExact(value));
STAT_INC(TO_BOOL, hit);
if (value == &_Py_STR(empty)) {
assert(_Py_IsImmortal(value));
@ -373,7 +370,7 @@ dummy_func(
inst(TO_BOOL_ALWAYS_TRUE, (unused/1, version/2, value -- res)) {
// This one is a bit weird, because we expect *some* failures:
assert(version);
DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL);
DEOPT_IF(Py_TYPE(value)->tp_version_tag != version);
STAT_INC(TO_BOOL, hit);
DECREF_INPUTS();
res = Py_True;
@ -397,8 +394,8 @@ dummy_func(
};
op(_GUARD_BOTH_INT, (left, right -- left, right)) {
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(left));
DEOPT_IF(!PyLong_CheckExact(right));
}
op(_BINARY_OP_MULTIPLY_INT, (unused/1, left, right -- res)) {
@ -433,8 +430,8 @@ dummy_func(
_GUARD_BOTH_INT + _BINARY_OP_SUBTRACT_INT;
op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) {
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(left));
DEOPT_IF(!PyFloat_CheckExact(right));
}
op(_BINARY_OP_MULTIPLY_FLOAT, (unused/1, left, right -- res)) {
@ -469,8 +466,8 @@ dummy_func(
_GUARD_BOTH_FLOAT + _BINARY_OP_SUBTRACT_FLOAT;
op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) {
DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(left));
DEOPT_IF(!PyUnicode_CheckExact(right));
}
op(_BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) {
@ -494,7 +491,7 @@ dummy_func(
_Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP];
assert(true_next.op.code == STORE_FAST);
PyObject **target_local = &GETLOCAL(true_next.op.arg);
DEOPT_IF(*target_local != left, BINARY_OP);
DEOPT_IF(*target_local != left);
STAT_INC(BINARY_OP, hit);
/* Handle `left = left + right` or `left += right` for str.
*
@ -574,13 +571,13 @@ dummy_func(
}
inst(BINARY_SUBSCR_LIST_INT, (unused/1, list, sub -- res)) {
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);
DEOPT_IF(!PyLong_CheckExact(sub));
DEOPT_IF(!PyList_CheckExact(list));
// Deopt unless 0 <= sub < PyList_Size(list)
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR);
DEOPT_IF(index >= PyList_GET_SIZE(list));
STAT_INC(BINARY_SUBSCR, hit);
res = PyList_GET_ITEM(list, index);
assert(res != NULL);
@ -590,14 +587,14 @@ dummy_func(
}
inst(BINARY_SUBSCR_STR_INT, (unused/1, str, sub -- res)) {
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR);
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
DEOPT_IF(!PyLong_CheckExact(sub));
DEOPT_IF(!PyUnicode_CheckExact(str));
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR);
DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index);
// Specialize for reading an ASCII character from any string:
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR);
DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c);
STAT_INC(BINARY_SUBSCR, hit);
res = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
@ -605,13 +602,13 @@ dummy_func(
}
inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple, sub -- res)) {
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);
DEOPT_IF(!PyLong_CheckExact(sub));
DEOPT_IF(!PyTuple_CheckExact(tuple));
// Deopt unless 0 <= sub < PyTuple_Size(list)
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR);
DEOPT_IF(index >= PyTuple_GET_SIZE(tuple));
STAT_INC(BINARY_SUBSCR, hit);
res = PyTuple_GET_ITEM(tuple, index);
assert(res != NULL);
@ -621,7 +618,7 @@ dummy_func(
}
inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) {
DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR);
DEOPT_IF(!PyDict_CheckExact(dict));
STAT_INC(BINARY_SUBSCR, hit);
res = PyDict_GetItemWithError(dict, sub);
if (res == NULL) {
@ -636,19 +633,19 @@ dummy_func(
}
inst(BINARY_SUBSCR_GETITEM, (unused/1, container, sub -- unused)) {
DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR);
DEOPT_IF(tstate->interp->eval_frame);
PyTypeObject *tp = Py_TYPE(container);
DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR);
DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE));
PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;
PyObject *cached = ht->_spec_cache.getitem;
DEOPT_IF(cached == NULL, BINARY_SUBSCR);
DEOPT_IF(cached == NULL);
assert(PyFunction_Check(cached));
PyFunctionObject *getitem = (PyFunctionObject *)cached;
uint32_t cached_version = ht->_spec_cache.getitem_version;
DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR);
DEOPT_IF(getitem->func_version != cached_version);
PyCodeObject *code = (PyCodeObject *)getitem->func_code;
assert(code->co_argcount == 2);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));
STAT_INC(BINARY_SUBSCR, hit);
Py_INCREF(getitem);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2);
@ -693,14 +690,14 @@ dummy_func(
}
inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) {
DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);
DEOPT_IF(!PyLong_CheckExact(sub));
DEOPT_IF(!PyList_CheckExact(list));
// Ensure nonnegative, zero-or-one-digit ints.
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR);
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
// Ensure index < len(list)
DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR);
DEOPT_IF(index >= PyList_GET_SIZE(list));
STAT_INC(STORE_SUBSCR, hit);
PyObject *old_value = PyList_GET_ITEM(list, index);
@ -712,7 +709,7 @@ dummy_func(
}
inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) {
DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
DEOPT_IF(!PyDict_CheckExact(dict));
STAT_INC(STORE_SUBSCR, hit);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
Py_DECREF(dict);
@ -1009,11 +1006,10 @@ dummy_func(
}
inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) {
DEOPT_IF(tstate->interp->eval_frame, SEND);
DEOPT_IF(tstate->interp->eval_frame);
PyGenObject *gen = (PyGenObject *)receiver;
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type &&
Py_TYPE(gen) != &PyCoro_Type, SEND);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND);
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING);
STAT_INC(SEND, hit);
_PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
STACK_SHRINK(1);
@ -1194,8 +1190,8 @@ dummy_func(
}
inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- values[oparg])) {
DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE);
DEOPT_IF(!PyTuple_CheckExact(seq));
DEOPT_IF(PyTuple_GET_SIZE(seq) != 2);
assert(oparg == 2);
STAT_INC(UNPACK_SEQUENCE, hit);
values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1));
@ -1204,8 +1200,8 @@ dummy_func(
}
inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) {
DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
DEOPT_IF(!PyTuple_CheckExact(seq));
DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg);
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyTuple_ITEMS(seq);
for (int i = oparg; --i >= 0; ) {
@ -1215,8 +1211,8 @@ dummy_func(
}
inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) {
DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE);
DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
DEOPT_IF(!PyList_CheckExact(seq));
DEOPT_IF(PyList_GET_SIZE(seq) != oparg);
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyList_ITEMS(seq);
for (int i = oparg; --i >= 0; ) {
@ -1412,15 +1408,15 @@ dummy_func(
op(_GUARD_GLOBALS_VERSION, (version/1 --)) {
PyDictObject *dict = (PyDictObject *)GLOBALS();
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
DEOPT_IF(!PyDict_CheckExact(dict));
DEOPT_IF(dict->ma_keys->dk_version != version);
assert(DK_IS_UNICODE(dict->ma_keys));
}
op(_GUARD_BUILTINS_VERSION, (version/1 --)) {
PyDictObject *dict = (PyDictObject *)BUILTINS();
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
DEOPT_IF(!PyDict_CheckExact(dict));
DEOPT_IF(dict->ma_keys->dk_version != version);
assert(DK_IS_UNICODE(dict->ma_keys));
}
@ -1428,7 +1424,7 @@ dummy_func(
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
res = entries[index].me_value;
DEOPT_IF(res == NULL, LOAD_GLOBAL);
DEOPT_IF(res == NULL);
Py_INCREF(res);
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
@ -1438,7 +1434,7 @@ dummy_func(
PyDictObject *bdict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
res = entries[index].me_value;
DEOPT_IF(res == NULL, LOAD_GLOBAL);
DEOPT_IF(res == NULL);
Py_INCREF(res);
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
@ -1763,8 +1759,8 @@ dummy_func(
inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- attr, unused if (0))) {
assert(!(oparg & 1));
DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
DEOPT_IF(global_super != (PyObject *)&PySuper_Type);
DEOPT_IF(!PyType_Check(class));
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
@ -1774,8 +1770,8 @@ dummy_func(
inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- attr, self_or_null)) {
assert(oparg & 1);
DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
DEOPT_IF(global_super != (PyObject *)&PySuper_Type);
DEOPT_IF(!PyType_Check(class));
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
PyTypeObject *cls = (PyTypeObject *)class;
@ -1864,22 +1860,20 @@ dummy_func(
op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) {
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(tp->tp_version_tag != type_version);
}
op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) {
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv),
LOAD_ATTR);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv));
}
op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) {
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
attr = _PyDictOrValues_GetValues(dorv)->values[index];
DEOPT_IF(attr == NULL, LOAD_ATTR);
DEOPT_IF(attr == NULL);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
@ -1894,15 +1888,15 @@ dummy_func(
unused/5; // Skip over rest of cache
inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) {
DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR);
DEOPT_IF(!PyModule_CheckExact(owner));
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
assert(dict != NULL);
DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR);
DEOPT_IF(dict->ma_keys->dk_version != type_version);
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < dict->ma_keys->dk_nentries);
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
attr = ep->me_value;
DEOPT_IF(attr == NULL, LOAD_ATTR);
DEOPT_IF(attr == NULL);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
@ -1912,27 +1906,27 @@ dummy_func(
inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) {
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(tp->tp_version_tag != type_version);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR);
DEOPT_IF(_PyDictOrValues_IsValues(dorv));
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
DEOPT_IF(dict == NULL, LOAD_ATTR);
DEOPT_IF(dict == NULL);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
uint16_t hint = index;
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries);
if (DK_IS_UNICODE(dict->ma_keys)) {
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, LOAD_ATTR);
DEOPT_IF(ep->me_key != name);
attr = ep->me_value;
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, LOAD_ATTR);
DEOPT_IF(ep->me_key != name);
attr = ep->me_value;
}
DEOPT_IF(attr == NULL, LOAD_ATTR);
DEOPT_IF(attr == NULL);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
@ -1942,7 +1936,7 @@ dummy_func(
op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) {
char *addr = (char *)owner + index;
attr = *(PyObject **)addr;
DEOPT_IF(attr == NULL, LOAD_ATTR);
DEOPT_IF(attr == NULL);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
@ -1957,9 +1951,8 @@ dummy_func(
inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, null if (oparg & 1))) {
DEOPT_IF(!PyType_Check(owner), LOAD_ATTR);
DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version,
LOAD_ATTR);
DEOPT_IF(!PyType_Check(owner));
DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version);
assert(type_version != 0);
STAT_INC(LOAD_ATTR, hit);
@ -1972,18 +1965,18 @@ dummy_func(
inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused, unused if (0))) {
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
DEOPT_IF(tstate->interp->eval_frame);
PyTypeObject *cls = Py_TYPE(owner);
DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(cls->tp_version_tag != type_version);
assert(type_version != 0);
assert(Py_IS_TYPE(fget, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)fget;
assert(func_version != 0);
DEOPT_IF(f->func_version != func_version, LOAD_ATTR);
DEOPT_IF(f->func_version != func_version);
PyCodeObject *code = (PyCodeObject *)f->func_code;
assert(code->co_argcount == 1);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(fget);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1);
@ -1997,17 +1990,17 @@ dummy_func(
inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) {
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
DEOPT_IF(tstate->interp->eval_frame);
PyTypeObject *cls = Py_TYPE(owner);
DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(cls->tp_version_tag != type_version);
assert(type_version != 0);
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)getattribute;
assert(func_version != 0);
DEOPT_IF(f->func_version != func_version, LOAD_ATTR);
DEOPT_IF(f->func_version != func_version);
PyCodeObject *code = (PyCodeObject *)f->func_code;
assert(code->co_argcount == 2);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));
STAT_INC(LOAD_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
@ -2025,7 +2018,7 @@ dummy_func(
op(_GUARD_DORV_VALUES, (owner -- owner)) {
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR);
DEOPT_IF(!_PyDictOrValues_IsValues(dorv));
}
op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) {
@ -2052,30 +2045,30 @@ dummy_func(
inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) {
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
DEOPT_IF(tp->tp_version_tag != type_version);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR);
DEOPT_IF(_PyDictOrValues_IsValues(dorv));
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
DEOPT_IF(dict == NULL, STORE_ATTR);
DEOPT_IF(dict == NULL);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries);
PyObject *old_value;
uint64_t new_version;
if (DK_IS_UNICODE(dict->ma_keys)) {
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, STORE_ATTR);
DEOPT_IF(ep->me_key != name);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
DEOPT_IF(old_value == NULL);
new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value);
ep->me_value = value;
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, STORE_ATTR);
DEOPT_IF(ep->me_key != name);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
DEOPT_IF(old_value == NULL);
new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value);
ep->me_value = value;
}
@ -2093,7 +2086,7 @@ dummy_func(
op(_GUARD_TYPE_VERSION_STORE, (type_version/2, owner -- owner)) {
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
DEOPT_IF(tp->tp_version_tag != type_version);
}
op(_STORE_ATTR_SLOT, (index/1, value, owner --)) {
@ -2140,8 +2133,8 @@ dummy_func(
}
inst(COMPARE_OP_FLOAT, (unused/1, left, right -- res)) {
DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP);
DEOPT_IF(!PyFloat_CheckExact(left));
DEOPT_IF(!PyFloat_CheckExact(right));
STAT_INC(COMPARE_OP, hit);
double dleft = PyFloat_AS_DOUBLE(left);
double dright = PyFloat_AS_DOUBLE(right);
@ -2155,10 +2148,10 @@ dummy_func(
// Similar to COMPARE_OP_FLOAT
inst(COMPARE_OP_INT, (unused/1, left, right -- res)) {
DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP);
DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP);
DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP);
DEOPT_IF(!PyLong_CheckExact(left));
DEOPT_IF(!PyLong_CheckExact(right));
DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left));
DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right));
STAT_INC(COMPARE_OP, hit);
assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 &&
_PyLong_DigitCount((PyLongObject *)right) <= 1);
@ -2174,8 +2167,8 @@ dummy_func(
// Similar to COMPARE_OP_FLOAT, but for ==, != only
inst(COMPARE_OP_STR, (unused/1, left, right -- res)) {
DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP);
DEOPT_IF(!PyUnicode_CheckExact(left));
DEOPT_IF(!PyUnicode_CheckExact(right));
STAT_INC(COMPARE_OP, hit);
int eq = _PyUnicode_Equal(left, right);
assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE);
@ -2492,7 +2485,7 @@ dummy_func(
}
op(_ITER_CHECK_LIST, (iter -- iter)) {
DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER);
DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type);
}
op(_ITER_JUMP_LIST, (iter -- iter)) {
@ -2548,7 +2541,7 @@ dummy_func(
_ITER_NEXT_LIST;
op(_ITER_CHECK_TUPLE, (iter -- iter)) {
DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER);
DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type);
}
op(_ITER_JUMP_TUPLE, (iter -- iter)) {
@ -2605,7 +2598,7 @@ dummy_func(
op(_ITER_CHECK_RANGE, (iter -- iter)) {
_PyRangeIterObject *r = (_PyRangeIterObject *)iter;
DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER);
DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type);
}
op(_ITER_JUMP_RANGE, (iter -- iter)) {
@ -2647,10 +2640,10 @@ dummy_func(
_ITER_NEXT_RANGE;
inst(FOR_ITER_GEN, (unused/1, iter -- iter, unused)) {
DEOPT_IF(tstate->interp->eval_frame, FOR_ITER);
DEOPT_IF(tstate->interp->eval_frame);
PyGenObject *gen = (PyGenObject *)iter;
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER);
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING);
STAT_INC(FOR_ITER, hit);
_PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
_PyFrame_StackPush(gen_frame, Py_None);
@ -2790,16 +2783,13 @@ dummy_func(
op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) {
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv),
LOAD_ATTR);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv));
}
op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) {
PyTypeObject *owner_cls = Py_TYPE(owner);
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version !=
keys_version, LOAD_ATTR);
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version);
}
op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) {
@ -2839,15 +2829,12 @@ dummy_func(
assert((oparg & 1) == 0);
PyTypeObject *owner_cls = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(owner_cls->tp_version_tag != type_version);
assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv),
LOAD_ATTR);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version !=
keys_version, LOAD_ATTR);
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
DECREF_INPUTS();
@ -2858,7 +2845,7 @@ dummy_func(
assert((oparg & 1) == 0);
PyTypeObject *owner_cls = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(owner_cls->tp_version_tag != type_version);
assert(owner_cls->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -2869,12 +2856,12 @@ dummy_func(
inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, self if (1))) {
assert(oparg & 1);
PyTypeObject *owner_cls = Py_TYPE(owner);
DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(owner_cls->tp_version_tag != type_version);
Py_ssize_t dictoffset = owner_cls->tp_dictoffset;
assert(dictoffset > 0);
PyObject *dict = *(PyObject **)((char *)owner + dictoffset);
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL, LOAD_ATTR);
DEOPT_IF(dict != NULL);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
@ -3002,8 +2989,8 @@ dummy_func(
}
op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) {
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
DEOPT_IF(null != NULL);
DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type);
}
op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) {
@ -3016,22 +3003,22 @@ dummy_func(
}
op(_CHECK_PEP_523, (--)) {
DEOPT_IF(tstate->interp->eval_frame, CALL);
DEOPT_IF(tstate->interp->eval_frame);
}
op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) {
DEOPT_IF(!PyFunction_Check(callable), CALL);
DEOPT_IF(!PyFunction_Check(callable));
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != func_version, CALL);
DEOPT_IF(func->func_version != func_version);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL);
DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL));
}
op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) {
PyFunctionObject *func = (PyFunctionObject *)callable;
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));
DEOPT_IF(tstate->py_recursion_remaining <= 1);
}
op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) {
@ -3094,24 +3081,24 @@ dummy_func(
_PUSH_FRAME;
inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) {
DEOPT_IF(tstate->interp->eval_frame, CALL);
DEOPT_IF(tstate->interp->eval_frame);
int argcount = oparg;
if (self_or_null != NULL) {
args--;
argcount++;
}
DEOPT_IF(!PyFunction_Check(callable), CALL);
DEOPT_IF(!PyFunction_Check(callable));
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != func_version, CALL);
DEOPT_IF(func->func_version != func_version);
PyCodeObject *code = (PyCodeObject *)func->func_code;
assert(func->func_defaults);
assert(PyTuple_CheckExact(func->func_defaults));
int defcount = (int)PyTuple_GET_SIZE(func->func_defaults);
assert(defcount <= code->co_argcount);
int min_args = code->co_argcount - defcount;
DEOPT_IF(argcount > code->co_argcount, CALL);
DEOPT_IF(argcount < min_args, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(argcount > code->co_argcount);
DEOPT_IF(argcount < min_args);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));
STAT_INC(CALL, hit);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount);
for (int i = 0; i < argcount; i++) {
@ -3130,9 +3117,9 @@ dummy_func(
inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) {
assert(oparg == 1);
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(null != NULL);
PyObject *obj = args[0];
DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL);
DEOPT_IF(callable != (PyObject *)&PyType_Type);
STAT_INC(CALL, hit);
res = Py_NewRef(Py_TYPE(obj));
Py_DECREF(obj);
@ -3141,8 +3128,8 @@ dummy_func(
inst(CALL_STR_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) {
assert(oparg == 1);
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL);
DEOPT_IF(null != NULL);
DEOPT_IF(callable != (PyObject *)&PyUnicode_Type);
STAT_INC(CALL, hit);
PyObject *arg = args[0];
res = PyObject_Str(arg);
@ -3154,8 +3141,8 @@ dummy_func(
inst(CALL_TUPLE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) {
assert(oparg == 1);
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL);
DEOPT_IF(null != NULL);
DEOPT_IF(callable != (PyObject *)&PyTuple_Type);
STAT_INC(CALL, hit);
PyObject *arg = args[0];
res = PySequence_Tuple(arg);
@ -3172,15 +3159,15 @@ dummy_func(
* 3. Pushes the frame for ``__init__`` to the frame stack
* */
_PyCallCache *cache = (_PyCallCache *)next_instr;
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(!PyType_Check(callable), CALL);
DEOPT_IF(null != NULL);
DEOPT_IF(!PyType_Check(callable));
PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL);
DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version));
PyHeapTypeObject *cls = (PyHeapTypeObject *)callable;
PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
PyCodeObject *code = (PyCodeObject *)init->func_code;
DEOPT_IF(code->co_argcount != oparg+1, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL);
DEOPT_IF(code->co_argcount != oparg+1);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize));
STAT_INC(CALL, hit);
PyObject *self = _PyType_NewManagedObject(tp);
if (self == NULL) {
@ -3233,9 +3220,9 @@ dummy_func(
args--;
total_args++;
}
DEOPT_IF(!PyType_Check(callable), CALL);
DEOPT_IF(!PyType_Check(callable));
PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
DEOPT_IF(tp->tp_vectorcall == NULL);
STAT_INC(CALL, hit);
res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL);
/* Free the arguments. */
@ -3254,9 +3241,9 @@ dummy_func(
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
DEOPT_IF(total_args != 1);
DEOPT_IF(!PyCFunction_CheckExact(callable));
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
// This is slower but CPython promises to check all non-vectorcall
@ -3282,8 +3269,8 @@ dummy_func(
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL);
DEOPT_IF(!PyCFunction_CheckExact(callable));
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
/* res = func(self, args, nargs) */
@ -3314,9 +3301,8 @@ dummy_func(
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
(METH_FASTCALL | METH_KEYWORDS), CALL);
DEOPT_IF(!PyCFunction_CheckExact(callable));
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS));
STAT_INC(CALL, hit);
/* res = func(self, args, nargs, kwnames) */
_PyCFunctionFastWithKeywords cfunc =
@ -3341,9 +3327,9 @@ dummy_func(
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
DEOPT_IF(total_args != 1);
PyInterpreterState *interp = tstate->interp;
DEOPT_IF(callable != interp->callable_cache.len, CALL);
DEOPT_IF(callable != interp->callable_cache.len);
STAT_INC(CALL, hit);
PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
@ -3365,9 +3351,9 @@ dummy_func(
args--;
total_args++;
}
DEOPT_IF(total_args != 2, CALL);
DEOPT_IF(total_args != 2);
PyInterpreterState *interp = tstate->interp;
DEOPT_IF(callable != interp->callable_cache.isinstance, CALL);
DEOPT_IF(callable != interp->callable_cache.isinstance);
STAT_INC(CALL, hit);
PyObject *cls = args[1];
PyObject *inst = args[0];
@ -3388,9 +3374,9 @@ dummy_func(
inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) {
assert(oparg == 1);
PyInterpreterState *interp = tstate->interp;
DEOPT_IF(callable != interp->callable_cache.list_append, CALL);
DEOPT_IF(callable != interp->callable_cache.list_append);
assert(self != NULL);
DEOPT_IF(!PyList_Check(self), CALL);
DEOPT_IF(!PyList_Check(self));
STAT_INC(CALL, hit);
if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) {
goto pop_1_error; // Since arg is DECREF'ed already
@ -3411,13 +3397,13 @@ dummy_func(
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
DEOPT_IF(total_args != 2, CALL);
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
DEOPT_IF(total_args != 2);
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != METH_O, CALL);
DEOPT_IF(meth->ml_flags != METH_O);
PyObject *arg = args[1];
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
// This is slower but CPython promises to check all non-vectorcall
@ -3442,12 +3428,12 @@ dummy_func(
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL);
DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS));
PyTypeObject *d_type = method->d_common.d_type;
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL);
DEOPT_IF(!Py_IS_TYPE(self, d_type));
STAT_INC(CALL, hit);
int nargs = total_args - 1;
_PyCFunctionFastWithKeywords cfunc =
@ -3471,13 +3457,13 @@ dummy_func(
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
DEOPT_IF(total_args != 1);
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
PyMethodDef *meth = method->d_method;
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
DEOPT_IF(meth->ml_flags != METH_NOARGS);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
// This is slower but CPython promises to check all non-vectorcall
@ -3502,11 +3488,11 @@ dummy_func(
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
/* Builtin METH_FASTCALL methods, without keywords */
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
DEOPT_IF(meth->ml_flags != METH_FASTCALL);
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
STAT_INC(CALL, hit);
_PyCFunctionFast cfunc =
(_PyCFunctionFast)(void(*)(void))meth->ml_meth;

73
Python/executor_cases.c.h generated
View File

@ -13,8 +13,7 @@
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
/* Possibly combine these two checks */
DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version
!= tstate->interp->monitoring_version, RESUME);
DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version, RESUME);
DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME);
break;
}
@ -236,8 +235,8 @@
PyObject *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(left), _GUARD_BOTH_INT);
DEOPT_IF(!PyLong_CheckExact(right), _GUARD_BOTH_INT);
break;
}
@ -294,8 +293,8 @@
PyObject *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(left), _GUARD_BOTH_FLOAT);
DEOPT_IF(!PyFloat_CheckExact(right), _GUARD_BOTH_FLOAT);
break;
}
@ -352,8 +351,8 @@
PyObject *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(left), _GUARD_BOTH_UNICODE);
DEOPT_IF(!PyUnicode_CheckExact(right), _GUARD_BOTH_UNICODE);
break;
}
@ -1189,8 +1188,8 @@
case _GUARD_GLOBALS_VERSION: {
uint16_t version = (uint16_t)operand;
PyDictObject *dict = (PyDictObject *)GLOBALS();
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION);
DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION);
assert(DK_IS_UNICODE(dict->ma_keys));
break;
}
@ -1198,8 +1197,8 @@
case _GUARD_BUILTINS_VERSION: {
uint16_t version = (uint16_t)operand;
PyDictObject *dict = (PyDictObject *)BUILTINS();
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION);
DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION);
assert(DK_IS_UNICODE(dict->ma_keys));
break;
}
@ -1211,7 +1210,7 @@
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
res = entries[index].me_value;
DEOPT_IF(res == NULL, LOAD_GLOBAL);
DEOPT_IF(res == NULL, _LOAD_GLOBAL_MODULE);
Py_INCREF(res);
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
@ -1229,7 +1228,7 @@
PyDictObject *bdict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
res = entries[index].me_value;
DEOPT_IF(res == NULL, LOAD_GLOBAL);
DEOPT_IF(res == NULL, _LOAD_GLOBAL_BUILTINS);
Py_INCREF(res);
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
@ -1679,7 +1678,7 @@
uint32_t type_version = (uint32_t)operand;
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION);
break;
}
@ -1689,9 +1688,7 @@
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv),
LOAD_ATTR);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _CHECK_MANAGED_OBJECT_HAS_VALUES);
break;
}
@ -1703,7 +1700,7 @@
uint16_t index = (uint16_t)operand;
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
attr = _PyDictOrValues_GetValues(dorv)->values[index];
DEOPT_IF(attr == NULL, LOAD_ATTR);
DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
@ -1722,7 +1719,7 @@
uint16_t index = (uint16_t)operand;
char *addr = (char *)owner + index;
attr = *(PyObject **)addr;
DEOPT_IF(attr == NULL, LOAD_ATTR);
DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
@ -1738,7 +1735,7 @@
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR);
DEOPT_IF(!_PyDictOrValues_IsValues(dorv), _GUARD_DORV_VALUES);
break;
}
@ -1770,7 +1767,7 @@
uint32_t type_version = (uint32_t)operand;
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION_STORE);
break;
}
@ -2114,7 +2111,7 @@
case _ITER_CHECK_LIST: {
PyObject *iter;
iter = stack_pointer[-1];
DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER);
DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, _ITER_CHECK_LIST);
break;
}
@ -2159,7 +2156,7 @@
case _ITER_CHECK_TUPLE: {
PyObject *iter;
iter = stack_pointer[-1];
DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER);
DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, _ITER_CHECK_TUPLE);
break;
}
@ -2205,7 +2202,7 @@
PyObject *iter;
iter = stack_pointer[-1];
_PyRangeIterObject *r = (_PyRangeIterObject *)iter;
DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER);
DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, _ITER_CHECK_RANGE);
break;
}
@ -2300,9 +2297,7 @@
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv),
LOAD_ATTR);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT);
break;
}
@ -2312,8 +2307,7 @@
uint32_t keys_version = (uint32_t)operand;
PyTypeObject *owner_cls = Py_TYPE(owner);
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version !=
keys_version, LOAD_ATTR);
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION);
break;
}
@ -2360,8 +2354,8 @@
PyObject *callable;
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
DEOPT_IF(null != NULL, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS);
DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS);
break;
}
@ -2382,7 +2376,7 @@
}
case _CHECK_PEP_523: {
DEOPT_IF(tstate->interp->eval_frame, CALL);
DEOPT_IF(tstate->interp->eval_frame, _CHECK_PEP_523);
break;
}
@ -2392,11 +2386,11 @@
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
uint32_t func_version = (uint32_t)operand;
DEOPT_IF(!PyFunction_Check(callable), CALL);
DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS);
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != func_version, CALL);
DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL);
DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), _CHECK_FUNCTION_EXACT_ARGS);
break;
}
@ -2405,8 +2399,8 @@
callable = stack_pointer[-2 - oparg];
PyFunctionObject *func = (PyFunctionObject *)callable;
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), _CHECK_STACK_SPACE);
DEOPT_IF(tstate->py_recursion_remaining <= 1, _CHECK_STACK_SPACE);
break;
}
@ -2671,8 +2665,7 @@
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
(METH_FASTCALL | METH_KEYWORDS), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL);
STAT_INC(CALL, hit);
/* res = func(self, args, nargs, kwnames) */
_PyCFunctionFastWithKeywords cfunc =

132
Python/generated_cases.c.h generated
View File

@ -32,8 +32,7 @@
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
/* Possibly combine these two checks */
DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version
!= tstate->interp->monitoring_version, RESUME);
DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version, RESUME);
DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME);
DISPATCH();
}
@ -391,8 +390,8 @@
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
}
// _BINARY_OP_MULTIPLY_INT
{
@ -416,8 +415,8 @@
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
}
// _BINARY_OP_ADD_INT
{
@ -441,8 +440,8 @@
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
}
// _BINARY_OP_SUBTRACT_INT
{
@ -466,8 +465,8 @@
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
}
// _BINARY_OP_MULTIPLY_FLOAT
{
@ -491,8 +490,8 @@
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
}
// _BINARY_OP_ADD_FLOAT
{
@ -516,8 +515,8 @@
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
}
// _BINARY_OP_SUBTRACT_FLOAT
{
@ -541,8 +540,8 @@
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP);
}
// _BINARY_OP_ADD_UNICODE
{
@ -565,15 +564,15 @@
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP);
}
// _BINARY_OP_INPLACE_ADD_UNICODE
{
_Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP];
assert(true_next.op.code == STORE_FAST);
PyObject **target_local = &GETLOCAL(true_next.op.arg);
DEOPT_IF(*target_local != left, BINARY_OP);
DEOPT_IF(*target_local != left, BINARY_OP);
STAT_INC(BINARY_OP, hit);
/* Handle `left = left + right` or `left += right` for str.
*
@ -1294,8 +1293,7 @@
receiver = stack_pointer[-2];
DEOPT_IF(tstate->interp->eval_frame, SEND);
PyGenObject *gen = (PyGenObject *)receiver;
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type &&
Py_TYPE(gen) != &PyCoro_Type, SEND);
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND);
STAT_INC(SEND, hit);
_PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
@ -1806,8 +1804,8 @@
{
uint16_t version = read_u16(&next_instr[1].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
assert(DK_IS_UNICODE(dict->ma_keys));
}
// _LOAD_GLOBAL_MODULE
@ -1816,7 +1814,7 @@
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
res = entries[index].me_value;
DEOPT_IF(res == NULL, LOAD_GLOBAL);
DEOPT_IF(res == NULL, LOAD_GLOBAL);
Py_INCREF(res);
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
@ -1836,16 +1834,16 @@
{
uint16_t version = read_u16(&next_instr[1].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
assert(DK_IS_UNICODE(dict->ma_keys));
}
// _GUARD_BUILTINS_VERSION
{
uint16_t version = read_u16(&next_instr[2].cache);
PyDictObject *dict = (PyDictObject *)BUILTINS();
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
assert(DK_IS_UNICODE(dict->ma_keys));
}
// _LOAD_GLOBAL_BUILTINS
@ -1854,7 +1852,7 @@
PyDictObject *bdict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
res = entries[index].me_value;
DEOPT_IF(res == NULL, LOAD_GLOBAL);
DEOPT_IF(res == NULL, LOAD_GLOBAL);
Py_INCREF(res);
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
@ -2403,23 +2401,21 @@
uint32_t type_version = read_u32(&next_instr[1].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _CHECK_MANAGED_OBJECT_HAS_VALUES
{
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv),
LOAD_ATTR);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR);
}
// _LOAD_ATTR_INSTANCE_VALUE
{
uint16_t index = read_u16(&next_instr[3].cache);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
attr = _PyDictOrValues_GetValues(dorv)->values[index];
DEOPT_IF(attr == NULL, LOAD_ATTR);
DEOPT_IF(attr == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
@ -2510,14 +2506,14 @@
uint32_t type_version = read_u32(&next_instr[1].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _LOAD_ATTR_SLOT
{
uint16_t index = read_u16(&next_instr[3].cache);
char *addr = (char *)owner + index;
attr = *(PyObject **)addr;
DEOPT_IF(attr == NULL, LOAD_ATTR);
DEOPT_IF(attr == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
@ -2539,8 +2535,7 @@
PyObject *descr = read_obj(&next_instr[5].cache);
DEOPT_IF(!PyType_Check(owner), LOAD_ATTR);
DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version,
LOAD_ATTR);
DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR);
assert(type_version != 0);
STAT_INC(LOAD_ATTR, hit);
@ -2627,13 +2622,13 @@
uint32_t type_version = read_u32(&next_instr[1].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
}
// _GUARD_DORV_VALUES
{
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR);
DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR);
}
// _STORE_ATTR_INSTANCE_VALUE
value = stack_pointer[-2];
@ -2716,7 +2711,7 @@
uint32_t type_version = read_u32(&next_instr[1].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
}
// _STORE_ATTR_SLOT
value = stack_pointer[-2];
@ -3297,7 +3292,7 @@
// _ITER_CHECK_LIST
iter = stack_pointer[-1];
{
DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER);
DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER);
}
// _ITER_JUMP_LIST
{
@ -3339,7 +3334,7 @@
// _ITER_CHECK_TUPLE
iter = stack_pointer[-1];
{
DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER);
DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER);
}
// _ITER_JUMP_TUPLE
{
@ -3382,7 +3377,7 @@
iter = stack_pointer[-1];
{
_PyRangeIterObject *r = (_PyRangeIterObject *)iter;
DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER);
DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER);
}
// _ITER_JUMP_RANGE
{
@ -3585,23 +3580,20 @@
uint32_t type_version = read_u32(&next_instr[1].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv),
LOAD_ATTR);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&next_instr[3].cache);
PyTypeObject *owner_cls = Py_TYPE(owner);
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version !=
keys_version, LOAD_ATTR);
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
}
// _LOAD_ATTR_METHOD_WITH_VALUES
{
@ -3631,7 +3623,7 @@
uint32_t type_version = read_u32(&next_instr[1].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _LOAD_ATTR_METHOD_NO_DICT
{
@ -3664,12 +3656,9 @@
DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR);
assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv),
LOAD_ATTR);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR);
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version !=
keys_version, LOAD_ATTR);
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
Py_DECREF(owner);
@ -3845,14 +3834,14 @@
_PyInterpreterFrame *new_frame;
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, CALL);
DEOPT_IF(tstate->interp->eval_frame, CALL);
}
// _CHECK_CALL_BOUND_METHOD_EXACT_ARGS
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
}
// _INIT_CALL_BOUND_METHOD_EXACT_ARGS
{
@ -3868,18 +3857,18 @@
callable = func;
{
uint32_t func_version = read_u32(&next_instr[1].cache);
DEOPT_IF(!PyFunction_Check(callable), CALL);
DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != func_version, CALL);
DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL);
DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL);
}
// _CHECK_STACK_SPACE
{
PyFunctionObject *func = (PyFunctionObject *)callable;
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
}
// _INIT_CALL_PY_EXACT_ARGS
args = stack_pointer - oparg;
@ -3939,25 +3928,25 @@
_PyInterpreterFrame *new_frame;
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, CALL);
DEOPT_IF(tstate->interp->eval_frame, CALL);
}
// _CHECK_FUNCTION_EXACT_ARGS
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
uint32_t func_version = read_u32(&next_instr[1].cache);
DEOPT_IF(!PyFunction_Check(callable), CALL);
DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != func_version, CALL);
DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL);
DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL);
}
// _CHECK_STACK_SPACE
{
PyFunctionObject *func = (PyFunctionObject *)callable;
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
}
// _INIT_CALL_PY_EXACT_ARGS
args = stack_pointer - oparg;
@ -4328,8 +4317,7 @@
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
(METH_FASTCALL | METH_KEYWORDS), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL);
STAT_INC(CALL, hit);
/* res = func(self, args, nargs, kwnames) */
_PyCFunctionFastWithKeywords cfunc =

26
Tools/cases_generator/analysis.py
View File

@ -26,7 +26,7 @@ RESERVED_WORDS = {
"co_names": "Use FRAME_CO_NAMES.",
}
RE_PREDICTED = r"^\s*(?:GO_TO_INSTRUCTION\(|DEOPT_IF\(.*?,\s*)(\w+)\);\s*(?://.*)?$"
RE_GO_TO_INSTR = r"^\s*GO_TO_INSTRUCTION\((\w+)\);\s*(?://.*)?$"
class Analyzer:
@ -187,16 +187,23 @@ class Analyzer:
Raises SystemExit if there is an error.
"""
self.analyze_macros_and_pseudos()
self.find_predictions()
self.map_families()
self.mark_predictions()
self.check_families()
def find_predictions(self) -> None:
"""Find the instructions that need PREDICTED() labels."""
def mark_predictions(self) -> None:
"""Mark the instructions that need PREDICTED() labels."""
# Start with family heads
for family in self.families.values():
if family.name in self.instrs:
self.instrs[family.name].predicted = True
if family.name in self.macro_instrs:
self.macro_instrs[family.name].predicted = True
# Also look for GO_TO_INSTRUCTION() calls
for instr in self.instrs.values():
targets: set[str] = set()
for line in instr.block_text:
if m := re.match(RE_PREDICTED, line):
if m := re.match(RE_GO_TO_INSTR, line):
targets.add(m.group(1))
for target in targets:
if target_instr := self.instrs.get(target):
@ -225,11 +232,18 @@ class Analyzer:
)
else:
member_instr.family = family
elif not self.macro_instrs.get(member):
if member_mac := self.macro_instrs.get(member):
assert member_mac.family is None, (member, member_mac.family.name)
member_mac.family = family
if not member_instr and not member_mac:
self.error(
f"Unknown instruction {member!r} referenced in family {family.name!r}",
family,
)
# A sanctioned exception:
# This opcode is a member of the family but it doesn't pass the checks.
if mac := self.macro_instrs.get("BINARY_OP_INPLACE_ADD_UNICODE"):
mac.family = self.families.get("BINARY_OP")
def check_families(self) -> None:
"""Check each family:

4
Tools/cases_generator/generate_cases.py
View File

@ -781,9 +781,7 @@ class Generator(Analyzer):
case parsing.Macro():
n_macros += 1
mac = self.macro_instrs[thing.name]
stacking.write_macro_instr(
mac, self.out, self.families.get(mac.name)
)
stacking.write_macro_instr(mac, self.out)
case parsing.Pseudo():
pass
case _:

16
Tools/cases_generator/instructions.py
View File

@ -144,7 +144,8 @@ class Instruction:
out: Formatter,
dedent: int,
active_caches: list[ActiveCacheEffect],
tier: Tiers = TIER_ONE,
tier: Tiers,
family: parsing.Family | None,
) -> None:
"""Write the instruction body."""
# Write cache effect variable declarations and initializations
@ -207,6 +208,14 @@ class Instruction:
)
else:
out.write_raw(f"{space}if ({cond}) goto {label};\n")
elif m := re.match(r"(\s*)DEOPT_IF\((.+)\);\s*(?://.*)?$", line):
space, cond = m.groups()
target = family.name if family else self.name
out.write_raw(f"{space}DEOPT_IF({cond}, {target});\n")
elif "DEOPT" in line:
filename = context.owner.filename
lineno = context.owner.tokens[context.begin].line
print(f"{filename}:{lineno}: ERROR: DEOPT_IF() must be all on one line")
elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line):
out.reset_lineno()
space = extra + m.group(1)
@ -244,7 +253,8 @@ class AbstractInstruction(Instruction):
out: Formatter,
dedent: int,
active_caches: list[ActiveCacheEffect],
tier: Tiers = TIER_ONE,
tier: Tiers,
family: parsing.Family | None,
) -> None:
pass
@ -268,7 +278,9 @@ class MacroInstruction:
macro: parsing.Macro
parts: MacroParts
cache_offset: int
# Set later
predicted: bool = False
family: parsing.Family | None = None
@dataclasses.dataclass

16
Tools/cases_generator/stacking.py
View File

@ -351,14 +351,13 @@ def write_single_instr(
out,
tier,
0,
instr.family,
)
except AssertionError as err:
raise AssertionError(f"Error writing instruction {instr.name}") from err
def write_macro_instr(
mac: MacroInstruction, out: Formatter, family: Family | None
) -> None:
def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None:
parts = [
part
for part in mac.parts
@ -368,9 +367,11 @@ def write_macro_instr(
with out.block(f"TARGET({mac.name})"):
if mac.predicted:
out.emit(f"PREDICTED({mac.name});")
out.static_assert_family_size(mac.name, family, mac.cache_offset)
out.static_assert_family_size(mac.name, mac.family, mac.cache_offset)
try:
next_instr_is_set = write_components(parts, out, TIER_ONE, mac.cache_offset)
next_instr_is_set = write_components(
parts, out, TIER_ONE, mac.cache_offset, mac.family
)
except AssertionError as err:
raise AssertionError(f"Error writing macro {mac.name}") from err
if not parts[-1].instr.always_exits:
@ -386,6 +387,7 @@ def write_components(
out: Formatter,
tier: Tiers,
cache_offset: int,
family: Family | None,
) -> bool:
managers = get_managers(parts)
@ -454,10 +456,10 @@ def write_components(
assert_no_pokes(managers)
if len(parts) == 1:
mgr.instr.write_body(out, 0, mgr.active_caches, tier)
mgr.instr.write_body(out, 0, mgr.active_caches, tier, family)
else:
with out.block(""):
mgr.instr.write_body(out, -4, mgr.active_caches, tier)
mgr.instr.write_body(out, -4, mgr.active_caches, tier, family)
if mgr is managers[-1] and not next_instr_is_set and not mgr.instr.always_exits:
# Adjust the stack to its final depth, *then* write the