mirrors/php-src
0
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2024-11-23 18:04:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

Preallocate stack space for JIT in execute_ex() to eliminate JIT prologue/epilogue.

Browse Source
This commit is contained in:
Dmitry Stogov 2020-11-30 17:56:08 +03:00
parent 5f36d0497f
commit 841b00f641
5 changed files with 105 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Zend/zend_execute.c
View File

@ -4757,6 +4757,11 @@ static zend_always_inline zend_execute_data *_zend_vm_stack_push_call_frame(uint
} \
} while (0)
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
/* This callback disables optimization of "vm_stack_data" variable in VM */
void (*zend_touch_vm_stack_data)(void *vm_stack_data) = NULL;
#endif
#include "zend_vm_execute.h"
ZEND_API zend_result zend_set_user_opcode_handler(zend_uchar opcode, user_opcode_handler_t handler)

33
Zend/zend_vm_execute.h
View File

@ -50826,15 +50826,34 @@ ZEND_API void execute_ex(zend_execute_data *ex)
{
DCL_OPLINE
#if defined(ZEND_VM_IP_GLOBAL_REG) || defined(ZEND_VM_IP_GLOBAL_REG)
struct {
#ifdef ZEND_VM_IP_GLOBAL_REG
const zend_op *orig_opline = opline;
const zend_op *orig_opline;
#endif
#ifdef ZEND_VM_FP_GLOBAL_REG
zend_execute_data *orig_execute_data = execute_data;
zend_execute_data *orig_execute_data;
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
char hybrid_jit_red_zone[ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE];
#endif
#endif
} vm_stack_data;
#endif
#ifdef ZEND_VM_IP_GLOBAL_REG
vm_stack_data.orig_opline = opline;
#endif
#ifdef ZEND_VM_FP_GLOBAL_REG
vm_stack_data.orig_execute_data = execute_data;
execute_data = ex;
#else
zend_execute_data *execute_data = ex;
#endif
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
memset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE);
if (zend_touch_vm_stack_data) {
zend_touch_vm_stack_data(&vm_stack_data);
}
#endif
#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)
if (UNEXPECTED(execute_data == NULL)) {
@ -58788,10 +58807,10 @@ zend_leave_helper_SPEC_LABEL:
HYBRID_BREAK();
HYBRID_CASE(HYBRID_HALT):
#ifdef ZEND_VM_FP_GLOBAL_REG
execute_data = orig_execute_data;
execute_data = vm_stack_data.orig_execute_data;
#endif
#ifdef ZEND_VM_IP_GLOBAL_REG
opline = orig_opline;
opline = vm_stack_data.orig_opline;
#endif
return;
HYBRID_DEFAULT:
@ -58800,9 +58819,9 @@ zend_leave_helper_SPEC_LABEL:
HYBRID_BREAK(); /* Never reached */
#else
#ifdef ZEND_VM_FP_GLOBAL_REG
execute_data = orig_execute_data;
execute_data = vm_stack_data.orig_execute_data;
# ifdef ZEND_VM_IP_GLOBAL_REG
opline = orig_opline;
opline = vm_stack_data.orig_opline;
# endif
return;
#else
@ -58811,7 +58830,7 @@ zend_leave_helper_SPEC_LABEL:
ZEND_VM_LOOP_INTERRUPT_CHECK();
} else {
# ifdef ZEND_VM_IP_GLOBAL_REG
opline = orig_opline;
opline = vm_stack_data.orig_opline;
# endif
return;
}

39
Zend/zend_vm_gen.php
View File

@ -1780,10 +1780,10 @@ function gen_executor_code($f, $spec, $kind, $prolog, &$switch_labels = array())
case ZEND_VM_KIND_HYBRID:
out($f,"\t\t\tHYBRID_CASE(HYBRID_HALT):\n");
out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n");
out($f,"\t\t\t\texecute_data = orig_execute_data;\n");
out($f,"\t\t\t\texecute_data = vm_stack_data.orig_execute_data;\n");
out($f,"#endif\n");
out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n");
out($f,"\t\t\t\topline = orig_opline;\n");
out($f,"\t\t\t\topline = vm_stack_data.orig_opline;\n");
out($f,"#endif\n");
out($f,"\t\t\t\treturn;\n");
out($f,"\t\t\tHYBRID_DEFAULT:\n");
@ -2065,15 +2065,34 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
out($f,$m[1]."zend_execute_data *execute_data = ex;\n");
out($f,"#endif\n");
} else {
out($f,"#if defined(ZEND_VM_IP_GLOBAL_REG) || defined(ZEND_VM_IP_GLOBAL_REG)\n");
out($f,$m[1]."struct {\n");
out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n");
out($f,$m[1]."const zend_op *orig_opline = opline;\n");
out($f,$m[1]."\tconst zend_op *orig_opline;\n");
out($f,"#endif\n");
out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n");
out($f,$m[1]."zend_execute_data *orig_execute_data = execute_data;\n");
out($f,$m[1]."\tzend_execute_data *orig_execute_data;\n");
out($f,"#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE\n");
out($f,$m[1]."\tchar hybrid_jit_red_zone[ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE];\n");
out($f,"#endif\n");
out($f,"#endif\n");
out($f,$m[1]."} vm_stack_data;\n");
out($f,"#endif\n");
out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n");
out($f,$m[1]."vm_stack_data.orig_opline = opline;\n");
out($f,"#endif\n");
out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n");
out($f,$m[1]."vm_stack_data.orig_execute_data = execute_data;\n");
out($f,$m[1]."execute_data = ex;\n");
out($f,"#else\n");
out($f,$m[1]."zend_execute_data *execute_data = ex;\n");
out($f,"#endif\n");
out($f,"#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE\n");
out($f,$m[1]."memset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE);\n");
out($f,$m[1]."if (zend_touch_vm_stack_data) {\n");
out($f,$m[1]."\tzend_touch_vm_stack_data(&vm_stack_data);\n");
out($f,$m[1]."}\n");
out($f,"#endif\n");
}
break;
case "INTERNAL_LABELS":
@ -2159,9 +2178,9 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
}
out($f,
"#ifdef ZEND_VM_FP_GLOBAL_REG\n" .
$m[1]."execute_data = orig_execute_data;\n" .
$m[1]."execute_data = vm_stack_data.orig_execute_data;\n" .
"# ifdef ZEND_VM_IP_GLOBAL_REG\n" .
$m[1]."opline = orig_opline;\n" .
$m[1]."opline = vm_stack_data.orig_opline;\n" .
"# endif\n" .
$m[1]."return;\n" .
"#else\n" .
@ -2170,7 +2189,7 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
$m[1]."\tZEND_VM_LOOP_INTERRUPT_CHECK();\n".
$m[1]."} else {\n" .
"# ifdef ZEND_VM_IP_GLOBAL_REG\n" .
$m[1]."\topline = orig_opline;\n" .
$m[1]."\topline = vm_stack_data.orig_opline;\n" .
"# endif\n".
$m[1]."\treturn;\n".
$m[1]."}\n".
@ -2578,6 +2597,12 @@ function gen_vm($def, $skel) {
fputs($f, "#define ZEND_VM_KIND\t\t" . $GLOBALS["vm_kind_name"][ZEND_VM_KIND] . "\n");
}
fputs($f, "\n");
fputs($f, "#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__)\n");
fputs($f, "# if (defined(i386) || defined(__x86_64__) || defined(_M_X64))\n");
fputs($f, "# define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16\n");
fputs($f, "# endif\n");
fputs($f, "#endif\n");
fputs($f, "\n");
foreach($vm_op_flags as $name => $val) {
fprintf($f, "#define %-24s 0x%08x\n", $name, $val);
}

6
Zend/zend_vm_opcodes.h
View File

@ -34,6 +34,12 @@
# define ZEND_VM_KIND ZEND_VM_KIND_CALL
#endif
#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__)
# if (defined(i386) || defined(__x86_64__) || defined(_M_X64))
# define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16
# endif
#endif
#define ZEND_VM_OP_SPEC 0x00000001
#define ZEND_VM_OP_CONST 0x00000002
#define ZEND_VM_OP_TMPVAR 0x00000004

52
ext/opcache/jit/zend_jit_x86.dasc
View File

@ -171,8 +171,21 @@ static void* dasm_labels[zend_lb_MAX];
#define BP_JIT_IS 6
#define CAN_USE_AVX() (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX)
|.macro ADD_HYBRID_SPAD
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
| add r4, HYBRID_SPAD
||#endif
|.endmacro
|.macro SUB_HYBRID_SPAD
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
| sub r4, HYBRID_SPAD
||#endif
|.endmacro
|.macro LOAD_ADDR, reg, addr
| .if X64
|| if (IS_32BIT(addr)) {
@ -1752,7 +1765,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst)
}
| //ZEND_VM_CONTINUE()
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| add r4, HYBRID_SPAD
| ADD_HYBRID_SPAD
| JMP_IP
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
@ -1774,7 +1787,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst)
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
const void *handler = zend_get_opcode_handler_func(EG(exception_op));
| add r4, HYBRID_SPAD
| ADD_HYBRID_SPAD
| EXT_CALL handler, r0
| JMP_IP
} else {
@ -1827,11 +1840,11 @@ static int zend_jit_leave_function_stub(dasm_State **Dst)
| test FCARG1d, ZEND_CALL_TOP
| jnz >1
| EXT_CALL zend_jit_leave_nested_func_helper, r0
| add r4, HYBRID_SPAD // stack alignment
| ADD_HYBRID_SPAD
| JMP_IP
|1:
| EXT_CALL zend_jit_leave_top_func_helper, r0
| add r4, HYBRID_SPAD // stack alignment
| ADD_HYBRID_SPAD
| JMP_IP
} else {
if (GCC_GLOBAL_REGS) {
@ -2413,7 +2426,7 @@ static int zend_jit_trace_halt_stub(dasm_State **Dst)
{
|->trace_halt:
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| add r4, HYBRID_SPAD
| ADD_HYBRID_SPAD
| EXT_JMP zend_jit_halt_op->handler, r0
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
@ -2505,7 +2518,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst)
| LOAD_IP
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| add r4, HYBRID_SPAD
| ADD_HYBRID_SPAD
| JMP_IP
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
@ -2531,7 +2544,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst)
| jne ->interrupt_handler
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| add r4, HYBRID_SPAD
| ADD_HYBRID_SPAD
| mov r0, EX->func
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
@ -2566,7 +2579,7 @@ static int zend_jit_trace_escape_stub(dasm_State **Dst)
|->trace_escape:
|
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| add r4, HYBRID_SPAD
| ADD_HYBRID_SPAD
| JMP_IP
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
@ -2608,7 +2621,7 @@ static int zend_jit_context_threaded_call_stub(dasm_State **Dst)
|->context_threaded_call:
| pop r0
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| add r4, HYBRID_SPAD
| ADD_HYBRID_SPAD
| jmp aword [IP]
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
@ -2990,7 +3003,7 @@ static int zend_jit_align_func(dasm_State **Dst)
static int zend_jit_prologue(dasm_State **Dst)
{
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| sub r4, HYBRID_SPAD
| SUB_HYBRID_SPAD
} else if (GCC_GLOBAL_REGS) {
| sub r4, SPAD // stack alignment
} else {
@ -3344,10 +3357,13 @@ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t,
/* Skip prologue. */
// TODO: don't hardcode this ???
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
prologue_size = 0;
#elif defined(__x86_64__) || defined(_M_X64)
// sub r4, HYBRID_SPAD
#if defined(__x86_64__) || defined(_M_X64)
prologue_size = 4;
#else
// sub r4, HYBRID_SPAD
prologue_size = 3;
#endif
} else if (GCC_GLOBAL_REGS) {
@ -3387,7 +3403,7 @@ static int zend_jit_trace_return(dasm_State **Dst, zend_bool original_handler)
| jmp ->trace_escape
#else
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| add r4, HYBRID_SPAD
| ADD_HYBRID_SPAD
if (!original_handler) {
| JMP_IP
} else {
@ -3652,13 +3668,13 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline)
/* Use inlined HYBRID VM handler */
const void *handler = opline->handler;
| add r4, HYBRID_SPAD
| ADD_HYBRID_SPAD
| EXT_JMP handler, r0
} else {
const void *handler = zend_get_opcode_handler_func(opline);
| EXT_CALL handler, r0
| add r4, HYBRID_SPAD
| ADD_HYBRID_SPAD
| JMP_IP
}
} else {
@ -10044,7 +10060,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend
}
#else
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| add r4, HYBRID_SPAD
| ADD_HYBRID_SPAD
| JMP_IP
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
@ -11283,7 +11299,11 @@ static int zend_jit_leave_func(dasm_State **Dst,
trace_info->flags |= ZEND_JIT_TRACE_LOOP;
| CMP_IP next_opline
| je =>0 // LOOP
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
| JMP_IP
#else
| jmp ->trace_escape
#endif
} else {
| CMP_IP next_opline
| jne ->trace_escape
@ -11313,7 +11333,7 @@ static int zend_jit_leave_func(dasm_State **Dst,
}
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| add r4, HYBRID_SPAD
| ADD_HYBRID_SPAD
#ifdef CONTEXT_THREADED_JIT
| push aword [IP]
| ret