From 7b36b67b1e585c541b418eaa190c56e73f9a2d87 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 18 Jul 2024 14:24:58 -0700 Subject: [PATCH] GH-118093: Add tier two support to several instructions (GH-121884) --- Include/internal/pycore_ceval.h | 2 + Include/internal/pycore_opcode_metadata.h | 16 ++- Include/internal/pycore_uop_ids.h | 43 +++--- Include/internal/pycore_uop_metadata.h | 26 +++- Python/bytecodes.c | 58 ++++---- Python/ceval.c | 11 +- Python/executor_cases.c.h | 163 +++++++++++++++++++++- Python/generated_cases.c.h | 90 +++++++----- Python/optimizer.c | 18 ++- Python/optimizer_bytecodes.c | 5 + Python/optimizer_cases.c.h | 48 ++++++- 11 files changed, 372 insertions(+), 108 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index b472d5d446b..fac4a4d2280 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -259,6 +259,8 @@ PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObjec PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); +PyAPI_FUNC(PyObject *) _PyEval_ImportFrom(PyThreadState *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v, int argcnt, int argcntafter, _PyStackRef *sp); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 7b495238d7a..40e582a5e94 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -557,7 +557,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL_LEN: return 1; case CALL_LIST_APPEND: - return 1; + return 0; case CALL_METHOD_DESCRIPTOR_FAST: return 1; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: @@ -835,7 +835,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case SEND: return 2; case SEND_GEN: - return 2; + return 1; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -1007,7 +1007,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, @@ -1026,7 +1026,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1131,7 +1131,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1236,6 +1236,7 @@ _PyOpcode_macro_expansion[256] = { [BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { _BUILD_CONST_KEY_MAP, 0, 0 } } }, [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, 0, 0 } } }, [BUILD_MAP] = { .nuops = 1, .uops = { { _BUILD_MAP, 0, 0 } } }, + [BUILD_SET] = { .nuops = 1, .uops = { { _BUILD_SET, 0, 0 } } }, [BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } }, [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, @@ -1249,6 +1250,7 @@ _PyOpcode_macro_expansion[256] = { [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, 0, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, 0, 0 } } }, [CALL_LEN] = { .nuops = 1, .uops = { { _CALL_LEN, 0, 0 } } }, + [CALL_LIST_APPEND] = { .nuops = 1, .uops = { { _CALL_LIST_APPEND, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, @@ -1295,6 +1297,8 @@ _PyOpcode_macro_expansion[256] = { [GET_ITER] = { .nuops = 1, .uops = { { _GET_ITER, 0, 0 } } }, [GET_LEN] = { .nuops = 1, .uops = { { _GET_LEN, 0, 0 } } }, [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { _GET_YIELD_FROM_ITER, 0, 0 } } }, + [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, 0, 0 } } }, + [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, 0, 0 } } }, [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, 0, 0 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, 0, 0 } } }, [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, 0, 0 } } }, @@ -1322,6 +1326,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } }, + [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, 0, 0 } } }, [LOAD_SPECIAL] = { .nuops = 1, .uops = { { _LOAD_SPECIAL, 0, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, @@ -1345,6 +1350,7 @@ _PyOpcode_macro_expansion[256] = { [RETURN_CONST] = { .nuops = 2, .uops = { { _LOAD_CONST, 0, 0 }, { _RETURN_VALUE, 0, 0 } } }, [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, 0, 0 } } }, [RETURN_VALUE] = { .nuops = 1, .uops = { { _RETURN_VALUE, 0, 0 } } }, + [SEND_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, 0, 0 }, { _SEND_GEN_FRAME, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, 0, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, 0, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index bd1d27b03b3..aa7ee7775fa 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -45,6 +45,7 @@ extern "C" { #define _CALL_ISINSTANCE CALL_ISINSTANCE #define _CALL_KW CALL_KW #define _CALL_LEN CALL_LEN +#define _CALL_LIST_APPEND CALL_LIST_APPEND #define _CALL_METHOD_DESCRIPTOR_FAST 316 #define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 317 #define _CALL_METHOD_DESCRIPTOR_NOARGS 318 @@ -128,6 +129,8 @@ extern "C" { #define _GUARD_TOS_FLOAT 370 #define _GUARD_TOS_INT 371 #define _GUARD_TYPE_VERSION 372 +#define _IMPORT_FROM IMPORT_FROM +#define _IMPORT_NAME IMPORT_NAME #define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 373 #define _INIT_CALL_PY_EXACT_ARGS 374 #define _INIT_CALL_PY_EXACT_ARGS_0 375 @@ -238,37 +241,37 @@ extern "C" { #define _RETURN_VALUE RETURN_VALUE #define _SAVE_RETURN_OFFSET 431 #define _SEND 432 -#define _SEND_GEN SEND_GEN +#define _SEND_GEN_FRAME 433 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 433 -#define _STORE_ATTR 434 -#define _STORE_ATTR_INSTANCE_VALUE 435 -#define _STORE_ATTR_SLOT 436 -#define _STORE_ATTR_WITH_HINT 437 +#define _START_EXECUTOR 434 +#define _STORE_ATTR 435 +#define _STORE_ATTR_INSTANCE_VALUE 436 +#define _STORE_ATTR_SLOT 437 +#define _STORE_ATTR_WITH_HINT 438 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 438 -#define _STORE_FAST_0 439 -#define _STORE_FAST_1 440 -#define _STORE_FAST_2 441 -#define _STORE_FAST_3 442 -#define _STORE_FAST_4 443 -#define _STORE_FAST_5 444 -#define _STORE_FAST_6 445 -#define _STORE_FAST_7 446 +#define _STORE_FAST 439 +#define _STORE_FAST_0 440 +#define _STORE_FAST_1 441 +#define _STORE_FAST_2 442 +#define _STORE_FAST_3 443 +#define _STORE_FAST_4 444 +#define _STORE_FAST_5 445 +#define _STORE_FAST_6 446 +#define _STORE_FAST_7 447 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 447 +#define _STORE_SUBSCR 448 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 448 -#define _TO_BOOL 449 +#define _TIER2_RESUME_CHECK 449 +#define _TO_BOOL 450 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -278,13 +281,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 450 +#define _UNPACK_SEQUENCE 451 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 450 +#define MAX_UOP_ID 451 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 2a2d6e923b7..ea48f9d2060 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -91,6 +91,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, @@ -107,6 +108,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG, [_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG, @@ -124,6 +126,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_CONST_KEY_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -163,6 +166,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_IMPORT_FROM] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IS_NONE] = 0, [_GET_LEN] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -171,7 +176,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, [_ITER_NEXT_LIST] = 0, @@ -222,6 +227,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -286,6 +292,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BUILD_CONST_KEY_MAP] = "_BUILD_CONST_KEY_MAP", [_BUILD_LIST] = "_BUILD_LIST", [_BUILD_MAP] = "_BUILD_MAP", + [_BUILD_SET] = "_BUILD_SET", [_BUILD_SLICE] = "_BUILD_SLICE", [_BUILD_STRING] = "_BUILD_STRING", [_BUILD_TUPLE] = "_BUILD_TUPLE", @@ -297,6 +304,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", [_CALL_LEN] = "_CALL_LEN", + [_CALL_LIST_APPEND] = "_CALL_LIST_APPEND", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", @@ -380,6 +388,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", + [_IMPORT_FROM] = "_IMPORT_FROM", + [_IMPORT_NAME] = "_IMPORT_NAME", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", [_INIT_CALL_PY_EXACT_ARGS_0] = "_INIT_CALL_PY_EXACT_ARGS_0", @@ -441,6 +451,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", [_LOAD_LOCALS] = "_LOAD_LOCALS", + [_LOAD_NAME] = "_LOAD_NAME", [_LOAD_SPECIAL] = "_LOAD_SPECIAL", [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", @@ -464,6 +475,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_RETURN_GENERATOR] = "_RETURN_GENERATOR", [_RETURN_VALUE] = "_RETURN_VALUE", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", + [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", [_SET_ADD] = "_SET_ADD", [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", @@ -658,6 +670,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GET_AWAITABLE: return 1; + case _SEND_GEN_FRAME: + return 2; case _YIELD_VALUE: return 1; case _POP_EXCEPT: @@ -690,6 +704,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_LOCALS: return 0; + case _LOAD_NAME: + return 0; case _LOAD_GLOBAL: return 0; case _GUARD_GLOBALS_VERSION: @@ -724,6 +740,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 2 + (oparg-1); case _SET_UPDATE: return 2 + (oparg-1); + case _BUILD_SET: + return oparg; case _BUILD_MAP: return oparg*2; case _SETUP_ANNOTATIONS: @@ -802,6 +820,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _CHECK_EXC_MATCH: return 2; + case _IMPORT_NAME: + return 2; + case _IMPORT_FROM: + return 1; case _IS_NONE: return 1; case _GET_LEN: @@ -920,6 +942,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 2 + oparg; case _CALL_ISINSTANCE: return 2 + oparg; + case _CALL_LIST_APPEND: + return 3; case _CALL_METHOD_DESCRIPTOR_O: return 2 + oparg; case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 142f97daeb0..480045069c2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1154,23 +1154,26 @@ dummy_func( macro(SEND) = _SPECIALIZE_SEND + _SEND; - inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) { - DEOPT_IF(tstate->interp->eval_frame); + op(_SEND_GEN_FRAME, (receiver, v -- receiver, gen_frame: _PyInterpreterFrame *)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(SEND, hit); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - STACK_SHRINK(1); + gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); + assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); } + macro(SEND_GEN) = + unused/1 + + _CHECK_PEP_523 + + _SEND_GEN_FRAME + + _PUSH_FRAME; + inst(INSTRUMENTED_YIELD_VALUE, (retval -- unused)) { assert(frame != &entry_frame); frame->instr_ptr = next_instr; @@ -1547,22 +1550,16 @@ dummy_func( ERROR_IF(true, error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) { - ERROR_NO_POP(); - } + ERROR_IF(PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0, error); if (v_o == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) { - ERROR_NO_POP(); - } + ERROR_IF(PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0, error); if (v_o == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) { - ERROR_NO_POP(); - } + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0, error); if (v_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - ERROR_NO_POP(); + ERROR_IF(true, error); } } } @@ -1828,7 +1825,8 @@ dummy_func( inst(BUILD_SET, (values[oparg] -- set)) { PyObject *set_o = PySet_New(NULL); if (set_o == NULL) { - ERROR_NO_POP(); + DECREF_INPUTS(); + ERROR_IF(true, error); } int err = 0; for (int i = 0; i < oparg; i++) { @@ -2616,9 +2614,9 @@ dummy_func( b = res ? PyStackRef_True : PyStackRef_False; } - tier1 inst(IMPORT_NAME, (level, fromlist -- res)) { + inst(IMPORT_NAME, (level, fromlist -- res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *res_o = import_name(tstate, frame, name, + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); DECREF_INPUTS(); @@ -2626,9 +2624,9 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - tier1 inst(IMPORT_FROM, (from -- from, res)) { + inst(IMPORT_FROM, (from -- from, res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *res_o = import_from(tstate, PyStackRef_AsPyObjectBorrow(from), name); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2898,7 +2896,7 @@ dummy_func( } /* iterator ended normally */ /* The translator sets the deopt target just past the matching END_FOR */ - DEOPT_IF(true); + EXIT_IF(true); } next = PyStackRef_FromPyObjectSteal(next_o); // Common case: no jump, leave it to the code generator @@ -3942,7 +3940,7 @@ dummy_func( } // This is secretly a super-instruction - tier1 inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, arg -- unused)) { + inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, arg -- )) { assert(oparg == 1); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); @@ -3952,16 +3950,16 @@ dummy_func( assert(self_o != NULL); DEOPT_IF(!PyList_Check(self_o)); STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); PyStackRef_CLOSE(self); PyStackRef_CLOSE(callable); - STACK_SHRINK(3); - // Skip POP_TOP + ERROR_IF(err, error); + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: assert(next_instr->op.code == POP_TOP); SKIP_OVER(1); - DISPATCH(); + #endif } op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res)) { diff --git a/Python/ceval.c b/Python/ceval.c index 97d4b82e05d..1e911d3ba17 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -245,9 +245,6 @@ static void monitor_throw(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); -static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *, - PyObject *, PyObject *, PyObject *); -static PyObject * import_from(PyThreadState *, PyObject *, PyObject *); static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg); static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); static _PyInterpreterFrame * @@ -2727,8 +2724,8 @@ _PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi) return 1; } -static PyObject * -import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, +PyObject * +_PyEval_ImportName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name, PyObject *fromlist, PyObject *level) { PyObject *import_func; @@ -2766,8 +2763,8 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, return res; } -static PyObject * -import_from(PyThreadState *tstate, PyObject *v, PyObject *name) +PyObject * +_PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) { PyObject *x; PyObject *fullmodname, *pkgname, *pkgpath, *pkgname_or_unknown, *errmsg; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 375e1fb0584..e9f73f032bf 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1227,7 +1227,33 @@ /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - /* _SEND_GEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _SEND_GEN_FRAME: { + _PyStackRef v; + _PyStackRef receiver; + _PyInterpreterFrame *gen_frame; + oparg = CURRENT_OPARG(); + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(SEND, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + stack_pointer[-1].bits = (uintptr_t)gen_frame; + break; + } /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 because it is instrumented */ @@ -1551,7 +1577,36 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - /* _LOAD_NAME is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ + case _LOAD_NAME: { + _PyStackRef v; + oparg = CURRENT_OPARG(); + PyObject *v_o; + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) JUMP_TO_ERROR(); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) JUMP_TO_ERROR(); + if (v_o == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) JUMP_TO_ERROR(); + if (v_o == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) JUMP_TO_ERROR(); + if (v_o == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) JUMP_TO_ERROR(); + } + } + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _LOAD_GLOBAL: { _PyStackRef res; @@ -1890,7 +1945,36 @@ break; } - /* _BUILD_SET is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ + case _BUILD_SET: { + _PyStackRef *values; + _PyStackRef set; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; + PyObject *set_o = PySet_New(NULL); + if (set_o == NULL) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) JUMP_TO_ERROR(); + } + int err = 0; + for (int i = 0; i < oparg; i++) { + PyObject *item = PyStackRef_AsPyObjectSteal(values[i]); + if (err == 0) { + err = PySet_Add(set_o, item); + } + Py_DECREF(item); + } + if (err != 0) { + Py_DECREF(set_o); + if (true) JUMP_TO_ERROR(); + } + set = PyStackRef_FromPyObjectSteal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _BUILD_MAP: { _PyStackRef *values; @@ -2843,6 +2927,42 @@ break; } + case _IMPORT_NAME: { + _PyStackRef fromlist; + _PyStackRef level; + _PyStackRef res; + oparg = CURRENT_OPARG(); + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + PyStackRef_CLOSE(level); + PyStackRef_CLOSE(fromlist); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _IMPORT_FROM: { + _PyStackRef from; + _PyStackRef res; + oparg = CURRENT_OPARG(); + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ @@ -4251,6 +4371,43 @@ break; } + case _CALL_LIST_APPEND: { + _PyStackRef arg; + _PyStackRef self; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; + assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(self_o != NULL); + if (!PyList_Check(self_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + PyStackRef_CLOSE(self); + PyStackRef_CLOSE(callable); + if (err) JUMP_TO_ERROR(); + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _CALL_METHOD_DESCRIPTOR_O: { _PyStackRef *args; _PyStackRef self_or_null; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 382288440a7..585e6825a34 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -688,7 +688,10 @@ values = &stack_pointer[-oparg]; PyObject *set_o = PySet_New(NULL); if (set_o == NULL) { - goto error; + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) { stack_pointer += -oparg; goto error; } } int err = 0; for (int i = 0; i < oparg; i++) { @@ -1744,15 +1747,18 @@ assert(self_o != NULL); DEOPT_IF(!PyList_Check(self_o), CALL); STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); PyStackRef_CLOSE(self); PyStackRef_CLOSE(callable); - STACK_SHRINK(3); - // Skip POP_TOP + if (err) goto pop_3_error; + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: assert(next_instr->op.code == POP_TOP); SKIP_OVER(1); + #endif + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3488,7 +3494,7 @@ _PyStackRef res; from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *res_o = import_from(tstate, PyStackRef_AsPyObjectBorrow(from), name); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); if (res_o == NULL) goto error; res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -3507,7 +3513,7 @@ fromlist = stack_pointer[-1]; level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *res_o = import_name(tstate, frame, name, + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); PyStackRef_CLOSE(level); @@ -4980,22 +4986,16 @@ if (true) goto error; } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) { - goto error; - } + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) goto error; if (v_o == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) { - goto error; - } + if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) goto error; if (v_o == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) { - goto error; - } + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) goto error; if (v_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; + if (true) goto error; } } } @@ -5806,29 +5806,53 @@ } TARGET(SEND_GEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; _PyStackRef v; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, SEND); + } + // _SEND_GEN_FRAME v = stack_pointer[-1]; receiver = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, SEND); - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); - STAT_INC(SEND, hit); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); + { + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + } + // _PUSH_FRAME + new_frame = gen_frame; + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } TARGET(SETUP_ANNOTATIONS) { diff --git a/Python/optimizer.c b/Python/optimizer.c index 561ec4efa4e..a43eed45f09 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -797,6 +797,13 @@ top: // Jump here after _PUSH_FRAME or likely branches if (uop == _PUSH_FRAME) { assert(i + 1 == nuops); + if (opcode == FOR_ITER_GEN || opcode == SEND_GEN) { + DPRINTF(2, "Bailing due to dynamic target\n"); + ADD_TO_TRACE(uop, oparg, 0, target); + ADD_TO_TRACE(_DYNAMIC_EXIT, 0, 0, 0); + goto done; + } + assert(_PyOpcode_Deopt[opcode] == CALL); int func_version_offset = offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) // Add one to account for the actual opcode/oparg pair: @@ -828,12 +835,6 @@ top: // Jump here after _PUSH_FRAME or likely branches ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } - if (opcode == FOR_ITER_GEN) { - DPRINTF(2, "Bailing due to dynamic target\n"); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_DYNAMIC_EXIT, 0, 0, 0); - goto done; - } // Increment IP to the return address instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; TRACE_STACK_PUSH(); @@ -886,6 +887,11 @@ top: // Jump here after _PUSH_FRAME or likely branches instr++; // Add cache size for opcode instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + + if (opcode == CALL_LIST_APPEND) { + assert(instr->op.code == POP_TOP); + instr++; + } } // End for (;;) done: diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 2ea839f5d6d..a506f9948fd 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -658,6 +658,11 @@ dummy_func(void) { ctx->done = true; } + op(_SEND_GEN_FRAME, ( -- )) { + // We are about to hit the end of the trace: + ctx->done = true; + } + op(_CHECK_STACK_SPACE, ( --)) { assert(corresponding_check_stack == NULL); corresponding_check_stack = this_instr; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 978aa911b52..60cfb214835 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -644,7 +644,11 @@ /* _SEND is not a viable micro-op for tier 2 */ - /* _SEND_GEN is not a viable micro-op for tier 2 */ + case _SEND_GEN_FRAME: { + // We are about to hit the end of the trace: + ctx->done = true; + break; + } /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ @@ -787,7 +791,14 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ - /* _LOAD_NAME is not a viable micro-op for tier 2 */ + case _LOAD_NAME: { + _Py_UopsSymbol *v; + v = sym_new_not_null(ctx); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _LOAD_GLOBAL: { _Py_UopsSymbol *res; @@ -910,7 +921,14 @@ break; } - /* _BUILD_SET is not a viable micro-op for tier 2 */ + case _BUILD_SET: { + _Py_UopsSymbol *set; + set = sym_new_not_null(ctx); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _BUILD_MAP: { _Py_UopsSymbol *map; @@ -1307,6 +1325,24 @@ break; } + case _IMPORT_NAME: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _IMPORT_FROM: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ @@ -1846,6 +1882,12 @@ break; } + case _CALL_LIST_APPEND: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _CALL_METHOD_DESCRIPTOR_O: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx);