From c13e7d98fb8581014a225b900b1b88ccbfc28097 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 16 Aug 2024 17:11:24 +0100 Subject: [PATCH] GH-118093: Specialize `CALL_KW` (GH-123006) --- Include/internal/pycore_code.h | 3 + Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 32 +- Include/internal/pycore_uop_ids.h | 298 +++++++++--------- Include/internal/pycore_uop_metadata.h | 24 ++ Include/opcode_ids.h | 109 +++---- Lib/_opcode_metadata.py | 114 +++---- Lib/opcode.py | 4 + ...-08-14-11-38-56.gh-issue-118093.3BywDP.rst | 5 + Python/bytecodes.c | 152 ++++++++- Python/executor_cases.c.h | 180 +++++++++++ Python/generated_cases.c.h | 292 ++++++++++++++++- Python/opcode_targets.h | 6 +- Python/optimizer.c | 2 +- Python/optimizer_bytecodes.c | 9 + Python/optimizer_cases.c.h | 56 ++++ Python/specialize.c | 67 ++++ 17 files changed, 1083 insertions(+), 273 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 67aeab24db2..2e76e9f64d4 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -156,6 +156,7 @@ typedef struct { } _PyCallCache; #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache) +#define INLINE_CACHE_ENTRIES_CALL_KW CACHE_ENTRIES(_PyCallCache) typedef struct { _Py_BackoffCounter counter; @@ -335,6 +336,8 @@ extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); +extern void _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, + int nargs); extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg, _PyStackRef *locals); extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 0af6e3f3cd4..095eb0f8a89 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -257,6 +257,7 @@ Known values: Python 3.14a1 3603 (Remove BUILD_CONST_KEY_MAP) Python 3.14a1 3604 (Do not duplicate test at end of while statements) Python 3.14a1 3605 (Move ENTER_EXECUTOR to opcode 255) + Python 3.14a1 3606 (Specialize CALL_KW) Python 3.15 will start with 3650 @@ -269,7 +270,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3605 +#define PYC_MAGIC_NUMBER 3606 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index a39d2390b74..8e38855175a 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -105,6 +105,12 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_KW: return 3 + oparg; + case CALL_KW_BOUND_METHOD: + return 3 + oparg; + case CALL_KW_NON_PY: + return 3 + oparg; + case CALL_KW_PY: + return 3 + oparg; case CALL_LEN: return 2 + oparg; case CALL_LIST_APPEND: @@ -554,6 +560,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_KW: return 1; + case CALL_KW_BOUND_METHOD: + return 0; + case CALL_KW_NON_PY: + return 1; + case CALL_KW_PY: + return 0; case CALL_LEN: return 1; case CALL_LIST_APPEND: @@ -1027,7 +1039,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1084,7 +1099,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1255,6 +1270,9 @@ _PyOpcode_macro_expansion[256] = { [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, 0, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, 0, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, 0, 0 } } }, + [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, 0, 0 }, { _PY_FRAME_KW, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, 0, 0 }, { _CALL_KW_NON_PY, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_KW_PY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _PY_FRAME_KW, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_LEN] = { .nuops = 1, .uops = { { _CALL_LEN, 0, 0 } } }, [CALL_LIST_APPEND] = { .nuops = 1, .uops = { { _CALL_LIST_APPEND, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, @@ -1436,6 +1454,9 @@ const char *_PyOpcode_OpName[264] = { [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", [CALL_ISINSTANCE] = "CALL_ISINSTANCE", [CALL_KW] = "CALL_KW", + [CALL_KW_BOUND_METHOD] = "CALL_KW_BOUND_METHOD", + [CALL_KW_NON_PY] = "CALL_KW_NON_PY", + [CALL_KW_PY] = "CALL_KW_PY", [CALL_LEN] = "CALL_LEN", [CALL_LIST_APPEND] = "CALL_LIST_APPEND", [CALL_METHOD_DESCRIPTOR_FAST] = "CALL_METHOD_DESCRIPTOR_FAST", @@ -1643,6 +1664,7 @@ const uint8_t _PyOpcode_Caches[256] = { [POP_JUMP_IF_NOT_NONE] = 1, [FOR_ITER] = 1, [CALL] = 3, + [CALL_KW] = 3, [BINARY_OP] = 1, }; #endif @@ -1686,6 +1708,9 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, [CALL_ISINSTANCE] = CALL, [CALL_KW] = CALL_KW, + [CALL_KW_BOUND_METHOD] = CALL_KW, + [CALL_KW_NON_PY] = CALL_KW, + [CALL_KW_PY] = CALL_KW, [CALL_LEN] = CALL, [CALL_LIST_APPEND] = CALL, [CALL_METHOD_DESCRIPTOR_FAST] = CALL, @@ -1898,9 +1923,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 146: \ case 147: \ case 148: \ - case 223: \ - case 224: \ - case 225: \ case 226: \ case 227: \ case 228: \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 74c7cc9cac6..2a255143c3e 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -42,41 +42,45 @@ extern "C" { #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 #define _CALL_ISINSTANCE CALL_ISINSTANCE +#define _CALL_KW_NON_PY 318 #define _CALL_LEN CALL_LEN #define _CALL_LIST_APPEND CALL_LIST_APPEND -#define _CALL_METHOD_DESCRIPTOR_FAST 318 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 319 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 320 -#define _CALL_METHOD_DESCRIPTOR_O 321 -#define _CALL_NON_PY_GENERAL 322 -#define _CALL_STR_1 323 -#define _CALL_TUPLE_1 324 +#define _CALL_METHOD_DESCRIPTOR_FAST 319 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 320 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 321 +#define _CALL_METHOD_DESCRIPTOR_O 322 +#define _CALL_NON_PY_GENERAL 323 +#define _CALL_STR_1 324 +#define _CALL_TUPLE_1 325 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_ATTR_CLASS 325 -#define _CHECK_ATTR_METHOD_LAZY_DICT 326 -#define _CHECK_ATTR_MODULE 327 -#define _CHECK_ATTR_WITH_HINT 328 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 329 +#define _CHECK_ATTR_CLASS 326 +#define _CHECK_ATTR_METHOD_LAZY_DICT 327 +#define _CHECK_ATTR_MODULE 328 +#define _CHECK_ATTR_WITH_HINT 329 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 330 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 330 -#define _CHECK_FUNCTION_EXACT_ARGS 331 -#define _CHECK_FUNCTION_VERSION 332 -#define _CHECK_IS_NOT_PY_CALLABLE 333 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 334 -#define _CHECK_METHOD_VERSION 335 -#define _CHECK_PEP_523 336 -#define _CHECK_PERIODIC 337 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 338 -#define _CHECK_STACK_SPACE 339 -#define _CHECK_STACK_SPACE_OPERAND 340 -#define _CHECK_VALIDITY 341 -#define _CHECK_VALIDITY_AND_SET_IP 342 -#define _COMPARE_OP 343 -#define _COMPARE_OP_FLOAT 344 -#define _COMPARE_OP_INT 345 -#define _COMPARE_OP_STR 346 -#define _CONTAINS_OP 347 +#define _CHECK_FUNCTION 331 +#define _CHECK_FUNCTION_EXACT_ARGS 332 +#define _CHECK_FUNCTION_VERSION 333 +#define _CHECK_FUNCTION_VERSION_KW 334 +#define _CHECK_IS_NOT_PY_CALLABLE 335 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 336 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 337 +#define _CHECK_METHOD_VERSION 338 +#define _CHECK_METHOD_VERSION_KW 339 +#define _CHECK_PEP_523 340 +#define _CHECK_PERIODIC 341 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 342 +#define _CHECK_STACK_SPACE 343 +#define _CHECK_STACK_SPACE_OPERAND 344 +#define _CHECK_VALIDITY 345 +#define _CHECK_VALIDITY_AND_SET_IP 346 +#define _COMPARE_OP 347 +#define _COMPARE_OP_FLOAT 348 +#define _COMPARE_OP_INT 349 +#define _COMPARE_OP_STR 350 +#define _CONTAINS_OP 351 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE @@ -88,57 +92,58 @@ extern "C" { #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 348 +#define _DEOPT 352 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 349 -#define _DO_CALL_KW 350 -#define _DYNAMIC_EXIT 351 +#define _DO_CALL 353 +#define _DO_CALL_KW 354 +#define _DYNAMIC_EXIT 355 #define _END_SEND END_SEND -#define _ERROR_POP_N 352 +#define _ERROR_POP_N 356 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 353 -#define _FATAL_ERROR 354 +#define _EXPAND_METHOD 357 +#define _EXPAND_METHOD_KW 358 +#define _FATAL_ERROR 359 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 355 -#define _FOR_ITER_GEN_FRAME 356 -#define _FOR_ITER_TIER_TWO 357 +#define _FOR_ITER 360 +#define _FOR_ITER_GEN_FRAME 361 +#define _FOR_ITER_TIER_TWO 362 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 358 -#define _GUARD_BOTH_INT 359 -#define _GUARD_BOTH_UNICODE 360 -#define _GUARD_BUILTINS_VERSION 361 -#define _GUARD_DORV_NO_DICT 362 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 363 -#define _GUARD_GLOBALS_VERSION 364 -#define _GUARD_IS_FALSE_POP 365 -#define _GUARD_IS_NONE_POP 366 -#define _GUARD_IS_NOT_NONE_POP 367 -#define _GUARD_IS_TRUE_POP 368 -#define _GUARD_KEYS_VERSION 369 -#define _GUARD_NOS_FLOAT 370 -#define _GUARD_NOS_INT 371 -#define _GUARD_NOT_EXHAUSTED_LIST 372 -#define _GUARD_NOT_EXHAUSTED_RANGE 373 -#define _GUARD_NOT_EXHAUSTED_TUPLE 374 -#define _GUARD_TOS_FLOAT 375 -#define _GUARD_TOS_INT 376 -#define _GUARD_TYPE_VERSION 377 +#define _GUARD_BOTH_FLOAT 363 +#define _GUARD_BOTH_INT 364 +#define _GUARD_BOTH_UNICODE 365 +#define _GUARD_BUILTINS_VERSION 366 +#define _GUARD_DORV_NO_DICT 367 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 368 +#define _GUARD_GLOBALS_VERSION 369 +#define _GUARD_IS_FALSE_POP 370 +#define _GUARD_IS_NONE_POP 371 +#define _GUARD_IS_NOT_NONE_POP 372 +#define _GUARD_IS_TRUE_POP 373 +#define _GUARD_KEYS_VERSION 374 +#define _GUARD_NOS_FLOAT 375 +#define _GUARD_NOS_INT 376 +#define _GUARD_NOT_EXHAUSTED_LIST 377 +#define _GUARD_NOT_EXHAUSTED_RANGE 378 +#define _GUARD_NOT_EXHAUSTED_TUPLE 379 +#define _GUARD_TOS_FLOAT 380 +#define _GUARD_TOS_INT 381 +#define _GUARD_TYPE_VERSION 382 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 378 -#define _INIT_CALL_PY_EXACT_ARGS 379 -#define _INIT_CALL_PY_EXACT_ARGS_0 380 -#define _INIT_CALL_PY_EXACT_ARGS_1 381 -#define _INIT_CALL_PY_EXACT_ARGS_2 382 -#define _INIT_CALL_PY_EXACT_ARGS_3 383 -#define _INIT_CALL_PY_EXACT_ARGS_4 384 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 383 +#define _INIT_CALL_PY_EXACT_ARGS 384 +#define _INIT_CALL_PY_EXACT_ARGS_0 385 +#define _INIT_CALL_PY_EXACT_ARGS_1 386 +#define _INIT_CALL_PY_EXACT_ARGS_2 387 +#define _INIT_CALL_PY_EXACT_ARGS_3 388 +#define _INIT_CALL_PY_EXACT_ARGS_4 389 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER @@ -150,65 +155,65 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 385 -#define _IS_NONE 386 +#define _INTERNAL_INCREMENT_OPT_COUNTER 390 +#define _IS_NONE 391 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 387 -#define _ITER_CHECK_RANGE 388 -#define _ITER_CHECK_TUPLE 389 -#define _ITER_JUMP_LIST 390 -#define _ITER_JUMP_RANGE 391 -#define _ITER_JUMP_TUPLE 392 -#define _ITER_NEXT_LIST 393 -#define _ITER_NEXT_RANGE 394 -#define _ITER_NEXT_TUPLE 395 -#define _JUMP_TO_TOP 396 +#define _ITER_CHECK_LIST 392 +#define _ITER_CHECK_RANGE 393 +#define _ITER_CHECK_TUPLE 394 +#define _ITER_JUMP_LIST 395 +#define _ITER_JUMP_RANGE 396 +#define _ITER_JUMP_TUPLE 397 +#define _ITER_NEXT_LIST 398 +#define _ITER_NEXT_RANGE 399 +#define _ITER_NEXT_TUPLE 400 +#define _JUMP_TO_TOP 401 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 397 -#define _LOAD_ATTR_CLASS 398 -#define _LOAD_ATTR_CLASS_0 399 -#define _LOAD_ATTR_CLASS_1 400 +#define _LOAD_ATTR 402 +#define _LOAD_ATTR_CLASS 403 +#define _LOAD_ATTR_CLASS_0 404 +#define _LOAD_ATTR_CLASS_1 405 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 401 -#define _LOAD_ATTR_INSTANCE_VALUE_0 402 -#define _LOAD_ATTR_INSTANCE_VALUE_1 403 -#define _LOAD_ATTR_METHOD_LAZY_DICT 404 -#define _LOAD_ATTR_METHOD_NO_DICT 405 -#define _LOAD_ATTR_METHOD_WITH_VALUES 406 -#define _LOAD_ATTR_MODULE 407 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 408 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 409 -#define _LOAD_ATTR_PROPERTY_FRAME 410 -#define _LOAD_ATTR_SLOT 411 -#define _LOAD_ATTR_SLOT_0 412 -#define _LOAD_ATTR_SLOT_1 413 -#define _LOAD_ATTR_WITH_HINT 414 +#define _LOAD_ATTR_INSTANCE_VALUE 406 +#define _LOAD_ATTR_INSTANCE_VALUE_0 407 +#define _LOAD_ATTR_INSTANCE_VALUE_1 408 +#define _LOAD_ATTR_METHOD_LAZY_DICT 409 +#define _LOAD_ATTR_METHOD_NO_DICT 410 +#define _LOAD_ATTR_METHOD_WITH_VALUES 411 +#define _LOAD_ATTR_MODULE 412 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 413 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 414 +#define _LOAD_ATTR_PROPERTY_FRAME 415 +#define _LOAD_ATTR_SLOT 416 +#define _LOAD_ATTR_SLOT_0 417 +#define _LOAD_ATTR_SLOT_1 418 +#define _LOAD_ATTR_WITH_HINT 419 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 415 -#define _LOAD_CONST_INLINE_BORROW 416 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 417 -#define _LOAD_CONST_INLINE_WITH_NULL 418 +#define _LOAD_CONST_INLINE 420 +#define _LOAD_CONST_INLINE_BORROW 421 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 422 +#define _LOAD_CONST_INLINE_WITH_NULL 423 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 419 -#define _LOAD_FAST_0 420 -#define _LOAD_FAST_1 421 -#define _LOAD_FAST_2 422 -#define _LOAD_FAST_3 423 -#define _LOAD_FAST_4 424 -#define _LOAD_FAST_5 425 -#define _LOAD_FAST_6 426 -#define _LOAD_FAST_7 427 +#define _LOAD_FAST 424 +#define _LOAD_FAST_0 425 +#define _LOAD_FAST_1 426 +#define _LOAD_FAST_2 427 +#define _LOAD_FAST_3 428 +#define _LOAD_FAST_4 429 +#define _LOAD_FAST_5 430 +#define _LOAD_FAST_6 431 +#define _LOAD_FAST_7 432 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 428 -#define _LOAD_GLOBAL_BUILTINS 429 -#define _LOAD_GLOBAL_MODULE 430 +#define _LOAD_GLOBAL 433 +#define _LOAD_GLOBAL_BUILTINS 434 +#define _LOAD_GLOBAL_MODULE 435 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SPECIAL LOAD_SPECIAL @@ -221,58 +226,59 @@ extern "C" { #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 431 -#define _MONITOR_CALL 432 -#define _MONITOR_JUMP_BACKWARD 433 -#define _MONITOR_RESUME 434 +#define _MAYBE_EXPAND_METHOD 436 +#define _MONITOR_CALL 437 +#define _MONITOR_JUMP_BACKWARD 438 +#define _MONITOR_RESUME 439 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 435 -#define _POP_JUMP_IF_TRUE 436 +#define _POP_JUMP_IF_FALSE 440 +#define _POP_JUMP_IF_TRUE 441 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 437 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 442 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 438 +#define _PUSH_FRAME 443 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 439 -#define _QUICKEN_RESUME 440 -#define _REPLACE_WITH_TRUE 441 +#define _PY_FRAME_GENERAL 444 +#define _PY_FRAME_KW 445 +#define _QUICKEN_RESUME 446 +#define _REPLACE_WITH_TRUE 447 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 442 -#define _SEND 443 -#define _SEND_GEN_FRAME 444 +#define _SAVE_RETURN_OFFSET 448 +#define _SEND 449 +#define _SEND_GEN_FRAME 450 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 445 -#define _STORE_ATTR 446 -#define _STORE_ATTR_INSTANCE_VALUE 447 -#define _STORE_ATTR_SLOT 448 -#define _STORE_ATTR_WITH_HINT 449 +#define _START_EXECUTOR 451 +#define _STORE_ATTR 452 +#define _STORE_ATTR_INSTANCE_VALUE 453 +#define _STORE_ATTR_SLOT 454 +#define _STORE_ATTR_WITH_HINT 455 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 450 -#define _STORE_FAST_0 451 -#define _STORE_FAST_1 452 -#define _STORE_FAST_2 453 -#define _STORE_FAST_3 454 -#define _STORE_FAST_4 455 -#define _STORE_FAST_5 456 -#define _STORE_FAST_6 457 -#define _STORE_FAST_7 458 +#define _STORE_FAST 456 +#define _STORE_FAST_0 457 +#define _STORE_FAST_1 458 +#define _STORE_FAST_2 459 +#define _STORE_FAST_3 460 +#define _STORE_FAST_4 461 +#define _STORE_FAST_5 462 +#define _STORE_FAST_6 463 +#define _STORE_FAST_7 464 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 459 +#define _STORE_SUBSCR 465 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 460 -#define _TO_BOOL 461 +#define _TIER2_RESUME_CHECK 466 +#define _TO_BOOL 467 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -282,14 +288,14 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 462 +#define _UNPACK_SEQUENCE 468 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE #define __DO_CALL_FUNCTION_EX _DO_CALL_FUNCTION_EX -#define MAX_UOP_ID 462 +#define MAX_UOP_ID 468 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 69833e484e9..98757b6c8d8 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -237,6 +237,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_METHOD_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_EXPAND_METHOD_KW] = HAS_ARG_FLAG, + [_CHECK_IS_NOT_PY_CALLABLE_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_KW_NON_PY] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, @@ -310,6 +316,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", + [_CALL_KW_NON_PY] = "_CALL_KW_NON_PY", [_CALL_LEN] = "_CALL_LEN", [_CALL_LIST_APPEND] = "_CALL_LIST_APPEND", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", @@ -330,9 +337,12 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_FUNCTION] = "_CHECK_FUNCTION", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_FUNCTION_VERSION_KW] = "_CHECK_FUNCTION_VERSION_KW", [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", + [_CHECK_IS_NOT_PY_CALLABLE_KW] = "_CHECK_IS_NOT_PY_CALLABLE_KW", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", + [_CHECK_METHOD_VERSION_KW] = "_CHECK_METHOD_VERSION_KW", [_CHECK_PEP_523] = "_CHECK_PEP_523", [_CHECK_PERIODIC] = "_CHECK_PERIODIC", [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM", @@ -365,6 +375,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", [_EXPAND_METHOD] = "_EXPAND_METHOD", + [_EXPAND_METHOD_KW] = "_EXPAND_METHOD_KW", [_FATAL_ERROR] = "_FATAL_ERROR", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", @@ -480,6 +491,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_PUSH_FRAME] = "_PUSH_FRAME", [_PUSH_NULL] = "_PUSH_NULL", [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", + [_PY_FRAME_KW] = "_PY_FRAME_KW", [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", [_RESUME_CHECK] = "_RESUME_CHECK", [_RETURN_GENERATOR] = "_RETURN_GENERATOR", @@ -972,6 +984,18 @@ int _PyUop_num_popped(int opcode, int oparg) return 2 + oparg; case _CALL_METHOD_DESCRIPTOR_FAST: return 2 + oparg; + case _PY_FRAME_KW: + return 3 + oparg; + case _CHECK_FUNCTION_VERSION_KW: + return 3 + oparg; + case _CHECK_METHOD_VERSION_KW: + return 3 + oparg; + case _EXPAND_METHOD_KW: + return 3 + oparg; + case _CHECK_IS_NOT_PY_CALLABLE_KW: + return 3 + oparg; + case _CALL_KW_NON_PY: + return 3 + oparg; case _MAKE_FUNCTION: return 1; case _SET_FUNCTION_ATTRIBUTE: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 95984a98df8..1189712e12e 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -148,59 +148,62 @@ extern "C" { #define CALL_BUILTIN_FAST_WITH_KEYWORDS 167 #define CALL_BUILTIN_O 168 #define CALL_ISINSTANCE 169 -#define CALL_LEN 170 -#define CALL_LIST_APPEND 171 -#define CALL_METHOD_DESCRIPTOR_FAST 172 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 173 -#define CALL_METHOD_DESCRIPTOR_NOARGS 174 -#define CALL_METHOD_DESCRIPTOR_O 175 -#define CALL_NON_PY_GENERAL 176 -#define CALL_PY_EXACT_ARGS 177 -#define CALL_PY_GENERAL 178 -#define CALL_STR_1 179 -#define CALL_TUPLE_1 180 -#define CALL_TYPE_1 181 -#define COMPARE_OP_FLOAT 182 -#define COMPARE_OP_INT 183 -#define COMPARE_OP_STR 184 -#define CONTAINS_OP_DICT 185 -#define CONTAINS_OP_SET 186 -#define FOR_ITER_GEN 187 -#define FOR_ITER_LIST 188 -#define FOR_ITER_RANGE 189 -#define FOR_ITER_TUPLE 190 -#define LOAD_ATTR_CLASS 191 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 192 -#define LOAD_ATTR_INSTANCE_VALUE 193 -#define LOAD_ATTR_METHOD_LAZY_DICT 194 -#define LOAD_ATTR_METHOD_NO_DICT 195 -#define LOAD_ATTR_METHOD_WITH_VALUES 196 -#define LOAD_ATTR_MODULE 197 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 198 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 199 -#define LOAD_ATTR_PROPERTY 200 -#define LOAD_ATTR_SLOT 201 -#define LOAD_ATTR_WITH_HINT 202 -#define LOAD_GLOBAL_BUILTIN 203 -#define LOAD_GLOBAL_MODULE 204 -#define LOAD_SUPER_ATTR_ATTR 205 -#define LOAD_SUPER_ATTR_METHOD 206 -#define RESUME_CHECK 207 -#define SEND_GEN 208 -#define STORE_ATTR_INSTANCE_VALUE 209 -#define STORE_ATTR_SLOT 210 -#define STORE_ATTR_WITH_HINT 211 -#define STORE_SUBSCR_DICT 212 -#define STORE_SUBSCR_LIST_INT 213 -#define TO_BOOL_ALWAYS_TRUE 214 -#define TO_BOOL_BOOL 215 -#define TO_BOOL_INT 216 -#define TO_BOOL_LIST 217 -#define TO_BOOL_NONE 218 -#define TO_BOOL_STR 219 -#define UNPACK_SEQUENCE_LIST 220 -#define UNPACK_SEQUENCE_TUPLE 221 -#define UNPACK_SEQUENCE_TWO_TUPLE 222 +#define CALL_KW_BOUND_METHOD 170 +#define CALL_KW_NON_PY 171 +#define CALL_KW_PY 172 +#define CALL_LEN 173 +#define CALL_LIST_APPEND 174 +#define CALL_METHOD_DESCRIPTOR_FAST 175 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 176 +#define CALL_METHOD_DESCRIPTOR_NOARGS 177 +#define CALL_METHOD_DESCRIPTOR_O 178 +#define CALL_NON_PY_GENERAL 179 +#define CALL_PY_EXACT_ARGS 180 +#define CALL_PY_GENERAL 181 +#define CALL_STR_1 182 +#define CALL_TUPLE_1 183 +#define CALL_TYPE_1 184 +#define COMPARE_OP_FLOAT 185 +#define COMPARE_OP_INT 186 +#define COMPARE_OP_STR 187 +#define CONTAINS_OP_DICT 188 +#define CONTAINS_OP_SET 189 +#define FOR_ITER_GEN 190 +#define FOR_ITER_LIST 191 +#define FOR_ITER_RANGE 192 +#define FOR_ITER_TUPLE 193 +#define LOAD_ATTR_CLASS 194 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 195 +#define LOAD_ATTR_INSTANCE_VALUE 196 +#define LOAD_ATTR_METHOD_LAZY_DICT 197 +#define LOAD_ATTR_METHOD_NO_DICT 198 +#define LOAD_ATTR_METHOD_WITH_VALUES 199 +#define LOAD_ATTR_MODULE 200 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 201 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 202 +#define LOAD_ATTR_PROPERTY 203 +#define LOAD_ATTR_SLOT 204 +#define LOAD_ATTR_WITH_HINT 205 +#define LOAD_GLOBAL_BUILTIN 206 +#define LOAD_GLOBAL_MODULE 207 +#define LOAD_SUPER_ATTR_ATTR 208 +#define LOAD_SUPER_ATTR_METHOD 209 +#define RESUME_CHECK 210 +#define SEND_GEN 211 +#define STORE_ATTR_INSTANCE_VALUE 212 +#define STORE_ATTR_SLOT 213 +#define STORE_ATTR_WITH_HINT 214 +#define STORE_SUBSCR_DICT 215 +#define STORE_SUBSCR_LIST_INT 216 +#define TO_BOOL_ALWAYS_TRUE 217 +#define TO_BOOL_BOOL 218 +#define TO_BOOL_INT 219 +#define TO_BOOL_LIST 220 +#define TO_BOOL_NONE 221 +#define TO_BOOL_STR 222 +#define UNPACK_SEQUENCE_LIST 223 +#define UNPACK_SEQUENCE_TUPLE 224 +#define UNPACK_SEQUENCE_TWO_TUPLE 225 #define INSTRUMENTED_END_FOR 236 #define INSTRUMENTED_END_SEND 237 #define INSTRUMENTED_LOAD_SUPER_ATTR 238 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 94c8a0accf6..0f3d9734897 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -107,6 +107,11 @@ _specializations = { "CALL_BOUND_METHOD_GENERAL", "CALL_NON_PY_GENERAL", ], + "CALL_KW": [ + "CALL_KW_BOUND_METHOD", + "CALL_KW_PY", + "CALL_KW_NON_PY", + ], } _specialized_opmap = { @@ -131,59 +136,62 @@ _specialized_opmap = { 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 167, 'CALL_BUILTIN_O': 168, 'CALL_ISINSTANCE': 169, - 'CALL_LEN': 170, - 'CALL_LIST_APPEND': 171, - 'CALL_METHOD_DESCRIPTOR_FAST': 172, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 173, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 174, - 'CALL_METHOD_DESCRIPTOR_O': 175, - 'CALL_NON_PY_GENERAL': 176, - 'CALL_PY_EXACT_ARGS': 177, - 'CALL_PY_GENERAL': 178, - 'CALL_STR_1': 179, - 'CALL_TUPLE_1': 180, - 'CALL_TYPE_1': 181, - 'COMPARE_OP_FLOAT': 182, - 'COMPARE_OP_INT': 183, - 'COMPARE_OP_STR': 184, - 'CONTAINS_OP_DICT': 185, - 'CONTAINS_OP_SET': 186, - 'FOR_ITER_GEN': 187, - 'FOR_ITER_LIST': 188, - 'FOR_ITER_RANGE': 189, - 'FOR_ITER_TUPLE': 190, - 'LOAD_ATTR_CLASS': 191, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 192, - 'LOAD_ATTR_INSTANCE_VALUE': 193, - 'LOAD_ATTR_METHOD_LAZY_DICT': 194, - 'LOAD_ATTR_METHOD_NO_DICT': 195, - 'LOAD_ATTR_METHOD_WITH_VALUES': 196, - 'LOAD_ATTR_MODULE': 197, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 198, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 199, - 'LOAD_ATTR_PROPERTY': 200, - 'LOAD_ATTR_SLOT': 201, - 'LOAD_ATTR_WITH_HINT': 202, - 'LOAD_GLOBAL_BUILTIN': 203, - 'LOAD_GLOBAL_MODULE': 204, - 'LOAD_SUPER_ATTR_ATTR': 205, - 'LOAD_SUPER_ATTR_METHOD': 206, - 'RESUME_CHECK': 207, - 'SEND_GEN': 208, - 'STORE_ATTR_INSTANCE_VALUE': 209, - 'STORE_ATTR_SLOT': 210, - 'STORE_ATTR_WITH_HINT': 211, - 'STORE_SUBSCR_DICT': 212, - 'STORE_SUBSCR_LIST_INT': 213, - 'TO_BOOL_ALWAYS_TRUE': 214, - 'TO_BOOL_BOOL': 215, - 'TO_BOOL_INT': 216, - 'TO_BOOL_LIST': 217, - 'TO_BOOL_NONE': 218, - 'TO_BOOL_STR': 219, - 'UNPACK_SEQUENCE_LIST': 220, - 'UNPACK_SEQUENCE_TUPLE': 221, - 'UNPACK_SEQUENCE_TWO_TUPLE': 222, + 'CALL_KW_BOUND_METHOD': 170, + 'CALL_KW_NON_PY': 171, + 'CALL_KW_PY': 172, + 'CALL_LEN': 173, + 'CALL_LIST_APPEND': 174, + 'CALL_METHOD_DESCRIPTOR_FAST': 175, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 176, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 177, + 'CALL_METHOD_DESCRIPTOR_O': 178, + 'CALL_NON_PY_GENERAL': 179, + 'CALL_PY_EXACT_ARGS': 180, + 'CALL_PY_GENERAL': 181, + 'CALL_STR_1': 182, + 'CALL_TUPLE_1': 183, + 'CALL_TYPE_1': 184, + 'COMPARE_OP_FLOAT': 185, + 'COMPARE_OP_INT': 186, + 'COMPARE_OP_STR': 187, + 'CONTAINS_OP_DICT': 188, + 'CONTAINS_OP_SET': 189, + 'FOR_ITER_GEN': 190, + 'FOR_ITER_LIST': 191, + 'FOR_ITER_RANGE': 192, + 'FOR_ITER_TUPLE': 193, + 'LOAD_ATTR_CLASS': 194, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 195, + 'LOAD_ATTR_INSTANCE_VALUE': 196, + 'LOAD_ATTR_METHOD_LAZY_DICT': 197, + 'LOAD_ATTR_METHOD_NO_DICT': 198, + 'LOAD_ATTR_METHOD_WITH_VALUES': 199, + 'LOAD_ATTR_MODULE': 200, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 201, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 202, + 'LOAD_ATTR_PROPERTY': 203, + 'LOAD_ATTR_SLOT': 204, + 'LOAD_ATTR_WITH_HINT': 205, + 'LOAD_GLOBAL_BUILTIN': 206, + 'LOAD_GLOBAL_MODULE': 207, + 'LOAD_SUPER_ATTR_ATTR': 208, + 'LOAD_SUPER_ATTR_METHOD': 209, + 'RESUME_CHECK': 210, + 'SEND_GEN': 211, + 'STORE_ATTR_INSTANCE_VALUE': 212, + 'STORE_ATTR_SLOT': 213, + 'STORE_ATTR_WITH_HINT': 214, + 'STORE_SUBSCR_DICT': 215, + 'STORE_SUBSCR_LIST_INT': 216, + 'TO_BOOL_ALWAYS_TRUE': 217, + 'TO_BOOL_BOOL': 218, + 'TO_BOOL_INT': 219, + 'TO_BOOL_LIST': 220, + 'TO_BOOL_NONE': 221, + 'TO_BOOL_STR': 222, + 'UNPACK_SEQUENCE_LIST': 223, + 'UNPACK_SEQUENCE_TUPLE': 224, + 'UNPACK_SEQUENCE_TWO_TUPLE': 225, } opmap = { diff --git a/Lib/opcode.py b/Lib/opcode.py index 2698609cd56..974f4d35e2a 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -85,6 +85,10 @@ _cache_format = { "counter": 1, "func_version": 2, }, + "CALL_KW": { + "counter": 1, + "func_version": 2, + }, "STORE_SUBSCR": { "counter": 1, }, diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst new file mode 100644 index 00000000000..4a3a094a6fc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst @@ -0,0 +1,5 @@ +Add three specializations for :opcode:`CALL_KW`: + +* :opcode:`!CALL_KW_PY` for calls to Python functions +* :opcode:`!CALL_KW_BOUND_METHOD` for calls to bound methods +* :opcode:`!CALL_KW_NON_PY` for all other calls diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 97e37bc28b8..ec57c07104d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4007,7 +4007,14 @@ dummy_func( _CALL_METHOD_DESCRIPTOR_FAST + _CHECK_PERIODIC; - inst(INSTRUMENTED_CALL_KW, ( -- )) { + // Cache layout: counter/1, func_version/2 + family(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW) = { + CALL_KW_BOUND_METHOD, + CALL_KW_PY, + CALL_KW_NON_PY, + }; + + inst(INSTRUMENTED_CALL_KW, (counter/1, version/2 -- )) { int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); int total_args = oparg + is_meth; PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); @@ -4017,6 +4024,7 @@ dummy_func( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); ERROR_IF(err, error); + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL_KW); } @@ -4062,8 +4070,8 @@ dummy_func( if (new_frame == NULL) { ERROR_NO_POP(); } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; + assert(next_instr - this_instr == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL_KW; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ @@ -4104,8 +4112,144 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _PyInterpreterFrame*)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o + ); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + SYNC_SP(); + if (new_frame == NULL) { + ERROR_NO_POP(); + } + } + + op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, self_or_null, unused[oparg], kwnames -- callable, self_or_null, unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(!PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + EXIT_IF(func->func_version != func_version); + } + + macro(CALL_KW_PY) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_FUNCTION_VERSION_KW + + _PY_FRAME_KW + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null, unused[oparg], kwnames -- callable, null, unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + + EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + EXIT_IF(!PyFunction_Check(func)); + EXIT_IF(((PyFunctionObject *)func)->func_version != func_version); + EXIT_IF(!PyStackRef_IsNull(null)); + } + + op(_EXPAND_METHOD_KW, (callable, null, unused[oparg], kwnames -- method, self, unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method)); + PyStackRef_CLOSE(callable); + } + + macro(CALL_KW_BOUND_METHOD) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_METHOD_VERSION_KW + + _EXPAND_METHOD_KW + + flush + // so that self is in the argument array + _PY_FRAME_KW + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null, args[oparg], kwnames -- callable, self_or_null, args[oparg], kwnames)) { + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CALL, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + macro(CALL_KW) = - _DO_CALL_KW + + _SPECIALIZE_CALL_KW + + unused/2 + + _DO_CALL_KW; + + op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg], kwnames -- callable, unused, unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(PyFunction_Check(callable_o)); + EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); + } + + + op(_CALL_KW_NON_PY, (callable, self_or_null, args[oparg], kwnames -- res)) { +#if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); +#endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); + } + + macro(CALL_KW_NON_PY) = + unused/1 + // Skip over the counter + unused/2 + + _CHECK_IS_NOT_PY_CALLABLE_KW + + _CALL_KW_NON_PY + _CHECK_PERIODIC; inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b8785381cdd..d699e8d7994 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4679,6 +4679,186 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _PY_FRAME_KW: { + _PyStackRef kwnames; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyInterpreterFrame *new_frame; + oparg = CURRENT_OPARG(); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o + ); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + JUMP_TO_ERROR(); + } + stack_pointer[0].bits = (uintptr_t)new_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION_KW: { + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CHECK_METHOD_VERSION_KW: { + _PyStackRef null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _EXPAND_METHOD_KW: { + _PyStackRef kwnames; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef method; + _PyStackRef self; + oparg = CURRENT_OPARG(); + kwnames = stack_pointer[-1]; + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + stack_pointer[-2 - oparg] = self; + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-3 - oparg] = method; + assert(PyStackRef_FunctionCheck(method)); + PyStackRef_CLOSE(callable); + stack_pointer[-1] = kwnames; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_KW: { + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CALL_KW_NON_PY: { + _PyStackRef kwnames; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (true) JUMP_TO_ERROR(); + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */ /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7582b06a764..486d356dfb9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1720,21 +1720,36 @@ TARGET(CALL_KW) { frame->instr_ptr = next_instr; - next_instr += 1; + next_instr += 4; INSTRUCTION_STATS(CALL_KW); PREDICTED(CALL_KW); - _Py_CODEUNIT *this_instr = next_instr - 1; + _Py_CODEUNIT *this_instr = next_instr - 4; (void)this_instr; _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef kwnames; _PyStackRef res; + // _SPECIALIZE_CALL_KW + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CALL, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 2 cache entries */ // _DO_CALL_KW kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); @@ -1776,8 +1791,8 @@ if (new_frame == NULL) { goto error; } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; + assert(next_instr - this_instr == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL_KW; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ @@ -1830,6 +1845,183 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(CALL_KW_BOUND_METHOD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef kwnames; + _PyStackRef method; + _PyStackRef self; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_METHOD_VERSION_KW + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null), CALL_KW); + } + // _EXPAND_METHOD_KW + kwnames = stack_pointer[-1]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + stack_pointer[-2 - oparg] = self; + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-3 - oparg] = method; + assert(PyStackRef_FunctionCheck(method)); + PyStackRef_CLOSE(callable); + } + // flush + // _PY_FRAME_KW + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o + ); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + goto error; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_KW_NON_PY) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef kwnames; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + callable = stack_pointer[-3 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + } + // _CALL_KW_NON_PY + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + { + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (true) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); @@ -1850,6 +2042,87 @@ DISPATCH(); } + TARGET(CALL_KW_PY) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef kwnames; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_FUNCTION_VERSION_KW + callable = stack_pointer[-3 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL_KW); + } + // _PY_FRAME_KW + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o + ); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + goto error; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(CALL_LEN) { frame->instr_ptr = next_instr; next_instr += 4; @@ -3906,8 +4179,12 @@ TARGET(INSTRUMENTED_CALL_KW) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; (void)this_instr; - next_instr += 1; + next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + uint32_t version = read_u32(&this_instr[2].cache); + (void)version; int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); int total_args = oparg + is_meth; PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); @@ -3917,6 +4194,7 @@ tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); if (err) goto error; + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL_KW); } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 9ea01e26842..0418105dee5 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -169,6 +169,9 @@ static void *opcode_targets[256] = { &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_CALL_BUILTIN_O, &&TARGET_CALL_ISINSTANCE, + &&TARGET_CALL_KW_BOUND_METHOD, + &&TARGET_CALL_KW_NON_PY, + &&TARGET_CALL_KW_PY, &&TARGET_CALL_LEN, &&TARGET_CALL_LIST_APPEND, &&TARGET_CALL_METHOD_DESCRIPTOR_FAST, @@ -232,9 +235,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, diff --git a/Python/optimizer.c b/Python/optimizer.c index dbd5467f0d4..9198e410627 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -807,7 +807,7 @@ translate_bytecode_to_trace( ADD_TO_TRACE(_DYNAMIC_EXIT, 0, 0, 0); goto done; } - assert(_PyOpcode_Deopt[opcode] == CALL); + assert(_PyOpcode_Deopt[opcode] == CALL || _PyOpcode_Deopt[opcode] == CALL_KW); int func_version_offset = offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) // Add one to account for the actual opcode/oparg pair: diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 010733e6ac2..97e4c642225 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -627,6 +627,15 @@ dummy_func(void) { ctx->done = true; } + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame*)) { + (void)callable; + (void)self_or_null; + (void)args; + (void)kwnames; + new_frame = NULL; + ctx->done = true; + } + op(_RETURN_VALUE, (retval -- res)) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 866d7d95b58..3ec2e6985d6 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1974,6 +1974,62 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ + case _PY_FRAME_KW: { + _Py_UopsSymbol *kwnames; + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UOpsAbstractFrame *new_frame; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + (void)callable; + (void)self_or_null; + (void)args; + (void)kwnames; + new_frame = NULL; + ctx->done = true; + stack_pointer[-3 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION_KW: { + break; + } + + case _CHECK_METHOD_VERSION_KW: { + break; + } + + case _EXPAND_METHOD_KW: { + _Py_UopsSymbol *method; + _Py_UopsSymbol *self; + _Py_UopsSymbol *kwnames; + method = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); + kwnames = sym_new_not_null(ctx); + stack_pointer[-3 - oparg] = method; + stack_pointer[-2 - oparg] = self; + stack_pointer[-1] = kwnames; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_KW: { + break; + } + + case _CALL_KW_NON_PY: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ diff --git a/Python/specialize.c b/Python/specialize.c index 4a227381a8b..4fec5a36fc9 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1904,6 +1904,33 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return 0; } + +static int +specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, + bool bound_method) +{ + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + PyCodeObject *code = (PyCodeObject *)func->func_code; + int kind = function_kind(code); + /* Don't specialize if PEP 523 is active */ + if (_PyInterpreterState_GET()->eval_frame) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); + return -1; + } + if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); + return -1; + } + int version = _PyFunction_GetVersionForCurrentState(func); + if (version == 0) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); + return -1; + } + write_u32(cache->func_version, version); + instr->op.code = bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY; + return 0; +} + static int specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) { @@ -1999,6 +2026,46 @@ _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) } } +void +_Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) +{ + PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); + + assert(ENABLE_SPECIALIZATION); + assert(_PyOpcode_Caches[CALL_KW] == INLINE_CACHE_ENTRIES_CALL_KW); + assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL_KW); + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + int fail; + if (PyFunction_Check(callable)) { + fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false); + } + else if (PyMethod_Check(callable)) { + PyObject *func = ((PyMethodObject *)callable)->im_func; + if (PyFunction_Check(func)) { + fail = specialize_py_call_kw((PyFunctionObject *)func, instr, nargs, true); + } + else { + SPECIALIZATION_FAIL(CALL_KW, SPEC_FAIL_CALL_BOUND_METHOD); + fail = -1; + } + } + else { + instr->op.code = CALL_KW_NON_PY; + fail = 0; + } + if (fail) { + STAT_INC(CALL, failure); + assert(!PyErr_Occurred()); + instr->op.code = CALL_KW; + cache->counter = adaptive_counter_backoff(cache->counter); + } + else { + STAT_INC(CALL, success); + assert(!PyErr_Occurred()); + cache->counter = adaptive_counter_cooldown(); + } +} + #ifdef Py_STATS static int binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs)