Skip to content

Commit 25d7616

Browse files
bwoebibwoebi
authored
Make internal run_time_cache a persistent allocation (#15040)
We also add zend_map_ptr_static, so that we do not incur the overhead of constantly recreating the internal run_time_cache pointers on each request. This mechanism might be extended for mutable_data of internal classes too.
1 parent 2f3224e commit 25d7616

File tree

11 files changed

+101
-20
lines changed

11 files changed

+101
-20
lines changed

‎Zend/zend.c

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -735,14 +735,16 @@ static void compiler_globals_ctor(zend_compiler_globals *compiler_globals) /* {{
735735
compiler_globals->map_ptr_base = ZEND_MAP_PTR_BIASED_BASE(NULL);
736736
compiler_globals->map_ptr_size = 0;
737737
compiler_globals->map_ptr_last = global_map_ptr_last;
738-
if (compiler_globals->map_ptr_last) {
738+
compiler_globals->internal_run_time_cache = NULL;
739+
if (compiler_globals->map_ptr_last || zend_map_ptr_static_size) {
739740
/* Allocate map_ptr table */
740741
compiler_globals->map_ptr_size = ZEND_MM_ALIGNED_SIZE_EX(compiler_globals->map_ptr_last, 4096);
741-
void *base = pemalloc(compiler_globals->map_ptr_size * sizeof(void*), 1);
742+
void *base = pemalloc((zend_map_ptr_static_size + compiler_globals->map_ptr_size) * sizeof(void*), 1);
742743
compiler_globals->map_ptr_real_base = base;
743744
compiler_globals->map_ptr_base = ZEND_MAP_PTR_BIASED_BASE(base);
744-
memset(base, 0, compiler_globals->map_ptr_last * sizeof(void*));
745+
memset(base, 0, (zend_map_ptr_static_size + compiler_globals->map_ptr_last) * sizeof(void*));
745746
}
747+
zend_init_internal_run_time_cache();
746748
}
747749
/* }}} */
748750

@@ -785,6 +787,10 @@ static void compiler_globals_dtor(zend_compiler_globals *compiler_globals) /* {{
785787
compiler_globals->map_ptr_base = ZEND_MAP_PTR_BIASED_BASE(NULL);
786788
compiler_globals->map_ptr_size = 0;
787789
}
790+
if (compiler_globals->internal_run_time_cache) {
791+
pefree(compiler_globals->internal_run_time_cache, 1);
792+
compiler_globals->internal_run_time_cache = NULL;
793+
}
788794
}
789795
/* }}} */
790796

@@ -1115,6 +1121,10 @@ zend_result zend_post_startup(void) /* {{{ */
11151121
}
11161122
compiler_globals->map_ptr_real_base = NULL;
11171123
compiler_globals->map_ptr_base = ZEND_MAP_PTR_BIASED_BASE(NULL);
1124+
if (compiler_globals->internal_run_time_cache) {
1125+
pefree(compiler_globals->internal_run_time_cache, 1);
1126+
}
1127+
compiler_globals->internal_run_time_cache = NULL;
11181128
if ((script_encoding_list = (zend_encoding **)compiler_globals->script_encoding_list)) {
11191129
compiler_globals_ctor(compiler_globals);
11201130
compiler_globals->script_encoding_list = (const zend_encoding **)script_encoding_list;
@@ -1198,6 +1208,9 @@ void zend_shutdown(void) /* {{{ */
11981208
CG(script_encoding_list_size) = 0;
11991209
}
12001210
#endif
1211+
zend_map_ptr_static_last = 0;
1212+
zend_map_ptr_static_size = 0;
1213+
12011214
zend_destroy_rsrc_list_dtors();
12021215

12031216
zend_unload_modules();
@@ -1297,9 +1310,9 @@ ZEND_API void zend_activate(void) /* {{{ */
12971310
init_executor();
12981311
startup_scanner();
12991312
if (CG(map_ptr_last)) {
1300-
memset(CG(map_ptr_real_base), 0, CG(map_ptr_last) * sizeof(void*));
1313+
memset((void **)CG(map_ptr_real_base) + zend_map_ptr_static_size, 0, CG(map_ptr_last) * sizeof(void*));
13011314
}
1302-
zend_init_internal_run_time_cache();
1315+
zend_reset_internal_run_time_cache();
13031316
zend_observer_activate();
13041317
}
13051318
/* }}} */
@@ -1984,6 +1997,9 @@ void free_estring(char **str_p) /* {{{ */
19841997
}
19851998
/* }}} */
19861999

2000+
ZEND_API size_t zend_map_ptr_static_size;
2001+
ZEND_API size_t zend_map_ptr_static_last;
2002+
19872003
ZEND_API void zend_map_ptr_reset(void)
19882004
{
19892005
CG(map_ptr_last) = global_map_ptr_last;
@@ -1996,15 +2012,36 @@ ZEND_API void *zend_map_ptr_new(void)
19962012
if (CG(map_ptr_last) >= CG(map_ptr_size)) {
19972013
/* Grow map_ptr table */
19982014
CG(map_ptr_size) = ZEND_MM_ALIGNED_SIZE_EX(CG(map_ptr_last) + 1, 4096);
1999-
CG(map_ptr_real_base) = perealloc(CG(map_ptr_real_base), CG(map_ptr_size) * sizeof(void*), 1);
2015+
CG(map_ptr_real_base) = perealloc(CG(map_ptr_real_base), (zend_map_ptr_static_size + CG(map_ptr_size)) * sizeof(void*), 1);
20002016
CG(map_ptr_base) = ZEND_MAP_PTR_BIASED_BASE(CG(map_ptr_real_base));
20012017
}
2002-
ptr = (void**)CG(map_ptr_real_base) + CG(map_ptr_last);
2018+
ptr = (void**)CG(map_ptr_real_base) + zend_map_ptr_static_size + CG(map_ptr_last);
20032019
*ptr = NULL;
20042020
CG(map_ptr_last)++;
20052021
return ZEND_MAP_PTR_PTR2OFFSET(ptr);
20062022
}
20072023

2024+
ZEND_API void *zend_map_ptr_new_static(void)
2025+
{
2026+
void **ptr;
2027+
2028+
if (zend_map_ptr_static_last >= zend_map_ptr_static_size) {
2029+
zend_map_ptr_static_size += 4096;
2030+
/* Grow map_ptr table */
2031+
void *new_base = pemalloc((zend_map_ptr_static_size + CG(map_ptr_size)) * sizeof(void*), 1);
2032+
if (CG(map_ptr_real_base)) {
2033+
memcpy((void **)new_base + 4096, CG(map_ptr_real_base), (CG(map_ptr_last) + zend_map_ptr_static_size - 4096) * sizeof(void *));
2034+
pefree(CG(map_ptr_real_base), 1);
2035+
}
2036+
CG(map_ptr_real_base) = new_base;
2037+
CG(map_ptr_base) = ZEND_MAP_PTR_BIASED_BASE(new_base);
2038+
}
2039+
ptr = (void**)CG(map_ptr_real_base) + (zend_map_ptr_static_last & 4095);
2040+
*ptr = NULL;
2041+
zend_map_ptr_static_last++;
2042+
return ZEND_MAP_PTR_PTR2OFFSET(ptr);
2043+
}
2044+
20082045
ZEND_API void zend_map_ptr_extend(size_t last)
20092046
{
20102047
if (last > CG(map_ptr_last)) {
@@ -2013,10 +2050,10 @@ ZEND_API void zend_map_ptr_extend(size_t last)
20132050
if (last >= CG(map_ptr_size)) {
20142051
/* Grow map_ptr table */
20152052
CG(map_ptr_size) = ZEND_MM_ALIGNED_SIZE_EX(last, 4096);
2016-
CG(map_ptr_real_base) = perealloc(CG(map_ptr_real_base), CG(map_ptr_size) * sizeof(void*), 1);
2053+
CG(map_ptr_real_base) = perealloc(CG(map_ptr_real_base), (zend_map_ptr_static_size + CG(map_ptr_size)) * sizeof(void*), 1);
20172054
CG(map_ptr_base) = ZEND_MAP_PTR_BIASED_BASE(CG(map_ptr_real_base));
20182055
}
2019-
ptr = (void**)CG(map_ptr_real_base) + CG(map_ptr_last);
2056+
ptr = (void**)CG(map_ptr_real_base) + zend_map_ptr_static_size + CG(map_ptr_last);
20202057
memset(ptr, 0, (last - CG(map_ptr_last)) * sizeof(void*));
20212058
CG(map_ptr_last) = last;
20222059
}

‎Zend/zend_API.c

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2958,7 +2958,11 @@ ZEND_API zend_result zend_register_functions(zend_class_entry *scope, const zend
29582958
if (EG(active)) { // at run-time: this ought to only happen if registered with dl() or somehow temporarily at runtime
29592959
ZEND_MAP_PTR_INIT(internal_function->run_time_cache, zend_arena_calloc(&CG(arena), 1, zend_internal_run_time_cache_reserved_size()));
29602960
} else {
2961-
ZEND_MAP_PTR_NEW(internal_function->run_time_cache);
2961+
#if ZTS
2962+
ZEND_MAP_PTR_NEW_STATIC(internal_function->run_time_cache);
2963+
#else
2964+
ZEND_MAP_PTR_INIT(internal_function->run_time_cache, NULL);
2965+
#endif
29622966
}
29632967
if (ptr->flags) {
29642968
if (!(ptr->flags & ZEND_ACC_PPP_MASK)) {

‎Zend/zend_enum.c

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -421,7 +421,11 @@ static void zend_enum_register_func(zend_class_entry *ce, zend_known_string_id n
421421
if (EG(active)) { // at run-time
422422
ZEND_MAP_PTR_INIT(zif->run_time_cache, zend_arena_calloc(&CG(arena), 1, zend_internal_run_time_cache_reserved_size()));
423423
} else {
424-
ZEND_MAP_PTR_NEW(zif->run_time_cache);
424+
#if ZTS
425+
ZEND_MAP_PTR_NEW_STATIC(zif->run_time_cache);
426+
#else
427+
ZEND_MAP_PTR_INIT(zif->run_time_cache, NULL);
428+
#endif
425429
}
426430

427431
if (!zend_hash_add_ptr(&ce->function_table, name, zif)) {

‎Zend/zend_extensions.c

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -331,19 +331,22 @@ ZEND_API void zend_init_internal_run_time_cache(void) {
331331
functions += zend_hash_num_elements(&ce->function_table);
332332
} ZEND_HASH_FOREACH_END();
333333

334-
char *ptr = zend_arena_calloc(&CG(arena), functions, rt_size);
334+
size_t alloc_size = functions * rt_size;
335+
char *ptr = pemalloc(alloc_size, 1);
336+
337+
CG(internal_run_time_cache) = ptr;
338+
CG(internal_run_time_cache_size) = alloc_size;
339+
335340
zend_internal_function *zif;
336341
ZEND_HASH_MAP_FOREACH_PTR(CG(function_table), zif) {
337-
if (!ZEND_USER_CODE(zif->type) && ZEND_MAP_PTR_GET(zif->run_time_cache) == NULL)
338-
{
342+
if (!ZEND_USER_CODE(zif->type) && ZEND_MAP_PTR_GET(zif->run_time_cache) == NULL) {
339343
ZEND_MAP_PTR_SET(zif->run_time_cache, (void *)ptr);
340344
ptr += rt_size;
341345
}
342346
} ZEND_HASH_FOREACH_END();
343347
ZEND_HASH_MAP_FOREACH_PTR(CG(class_table), ce) {
344348
ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, zif) {
345-
if (!ZEND_USER_CODE(zif->type) && ZEND_MAP_PTR_GET(zif->run_time_cache) == NULL)
346-
{
349+
if (!ZEND_USER_CODE(zif->type) && ZEND_MAP_PTR_GET(zif->run_time_cache) == NULL) {
347350
ZEND_MAP_PTR_SET(zif->run_time_cache, (void *)ptr);
348351
ptr += rt_size;
349352
}
@@ -352,6 +355,12 @@ ZEND_API void zend_init_internal_run_time_cache(void) {
352355
}
353356
}
354357

358+
ZEND_API void zend_reset_internal_run_time_cache(void) {
359+
if (CG(internal_run_time_cache)) {
360+
memset(CG(internal_run_time_cache), 0, CG(internal_run_time_cache_size));
361+
}
362+
}
363+
355364
ZEND_API zend_extension *zend_get_extension(const char *extension_name)
356365
{
357366
zend_llist_element *element;

‎Zend/zend_extensions.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ void zend_shutdown_extensions(void);
149149

150150
ZEND_API size_t zend_internal_run_time_cache_reserved_size(void);
151151
ZEND_API void zend_init_internal_run_time_cache(void);
152+
ZEND_API void zend_reset_internal_run_time_cache(void);
152153

153154
BEGIN_EXTERN_C()
154155
ZEND_API zend_result zend_load_extension(const char *path);

‎Zend/zend_globals.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,9 @@ struct _zend_compiler_globals {
154154

155155
uint32_t rtd_key_counter;
156156

157+
void *internal_run_time_cache;
158+
uint32_t internal_run_time_cache_size;
159+
157160
zend_stack short_circuiting_opnums;
158161
#ifdef ZTS
159162
uint32_t copied_functions_count;

‎Zend/zend_map_ptr.h

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ typedef struct _zend_string zend_string;
4242
#define ZEND_MAP_PTR_NEW(ptr) do { \
4343
ZEND_MAP_PTR(ptr) = zend_map_ptr_new(); \
4444
} while (0)
45+
#define ZEND_MAP_PTR_NEW_STATIC(ptr) do { \
46+
ZEND_MAP_PTR(ptr) = zend_map_ptr_new_static(); \
47+
} while (0)
4548

4649
#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET
4750
# define ZEND_MAP_PTR_NEW_OFFSET() \
@@ -53,7 +56,7 @@ typedef struct _zend_string zend_string;
5356
ZEND_MAP_PTR_GET_IMM(ptr) : \
5457
((void*)(ZEND_MAP_PTR(ptr)))))
5558
# define ZEND_MAP_PTR_GET_IMM(ptr) \
56-
(*ZEND_MAP_PTR_OFFSET2PTR((uintptr_t)ZEND_MAP_PTR(ptr)))
59+
(*ZEND_MAP_PTR_OFFSET2PTR((intptr_t)ZEND_MAP_PTR(ptr)))
5760
# define ZEND_MAP_PTR_SET(ptr, val) do { \
5861
if (ZEND_MAP_PTR_IS_OFFSET(ptr)) { \
5962
ZEND_MAP_PTR_SET_IMM(ptr, val); \
@@ -62,11 +65,11 @@ typedef struct _zend_string zend_string;
6265
} \
6366
} while (0)
6467
# define ZEND_MAP_PTR_SET_IMM(ptr, val) do { \
65-
void **__p = ZEND_MAP_PTR_OFFSET2PTR((uintptr_t)ZEND_MAP_PTR(ptr)); \
68+
void **__p = ZEND_MAP_PTR_OFFSET2PTR((intptr_t)ZEND_MAP_PTR(ptr)); \
6669
*__p = (val); \
6770
} while (0)
6871
# define ZEND_MAP_PTR_BIASED_BASE(real_base) \
69-
((void*)(((uintptr_t)(real_base)) - 1))
72+
((void*)(((uintptr_t)(real_base)) + zend_map_ptr_static_size * sizeof(void *) - 1))
7073
#else
7174
# error "Unknown ZEND_MAP_PTR_KIND"
7275
#endif
@@ -75,9 +78,13 @@ BEGIN_EXTERN_C()
7578

7679
ZEND_API void zend_map_ptr_reset(void);
7780
ZEND_API void *zend_map_ptr_new(void);
81+
ZEND_API void *zend_map_ptr_new_static(void);
7882
ZEND_API void zend_map_ptr_extend(size_t last);
7983
ZEND_API void zend_alloc_ce_cache(zend_string *type_name);
8084

85+
ZEND_API extern size_t zend_map_ptr_static_last;
86+
ZEND_API extern size_t zend_map_ptr_static_size;
87+
8188
END_EXTERN_C()
8289

8390
#endif /* ZEND_MAP_PTR_H */

‎ext/ffi/ffi.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5393,6 +5393,11 @@ static zend_result ffi_fixup_temporaries(void) {
53935393
++zend_ffi_cast_fn.T;
53945394
++zend_ffi_type_fn.T;
53955395
}
5396+
#if !ZTS
5397+
ZEND_MAP_PTR(zend_ffi_new_fn.run_time_cache) = ZEND_MAP_PTR(((zend_internal_function *)zend_hash_str_find_ptr(&zend_ffi_ce->function_table, "new", sizeof("new")-1))->run_time_cache);
5398+
ZEND_MAP_PTR(zend_ffi_cast_fn.run_time_cache) = ZEND_MAP_PTR(((zend_internal_function *)zend_hash_str_find_ptr(&zend_ffi_ce->function_table, "cast", sizeof("cast")-1))->run_time_cache);
5399+
ZEND_MAP_PTR(zend_ffi_type_fn.run_time_cache) = ZEND_MAP_PTR(((zend_internal_function *)zend_hash_str_find_ptr(&zend_ffi_ce->function_table, "type", sizeof("type")-1))->run_time_cache);
5400+
#endif
53965401
if (prev_zend_post_startup_cb) {
53975402
return prev_zend_post_startup_cb();
53985403
}

‎ext/opcache/jit/zend_jit_ir.c

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4534,8 +4534,12 @@ static struct jit_observer_fcall_is_unobserved_data jit_observer_fcall_is_unobse
45344534
if (func && (func->common.fn_flags & ZEND_ACC_CLOSURE) == 0 && ZEND_MAP_PTR_IS_OFFSET(func->common.run_time_cache)) {
45354535
// JIT: ZEND_MAP_PTR_GET_IMM(func->common.runtime_cache)
45364536
run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CG(map_ptr_base)), (uintptr_t)ZEND_MAP_PTR(func->common.run_time_cache)));
4537+
#if !ZTS
4538+
} else if (func && rx == IS_UNUSED) { // happens for internal functions only
4539+
ZEND_ASSERT(!ZEND_USER_CODE(func->type));
4540+
run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_CONST_ADDR(func), offsetof(zend_op_array, run_time_cache__ptr)));
4541+
#endif
45374542
} else {
4538-
ZEND_ASSERT(rx != IR_UNUSED);
45394543
// Closures may be duplicated and have a different runtime cache. Use the regular run_time_cache access pattern for these
45404544
if (func && ZEND_USER_CODE(func->type)) { // not a closure and definitely not an internal function
45414545
run_time_cache = ir_LOAD_A(jit_CALL(rx, run_time_cache));

‎ext/xmlreader/php_xmlreader.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1276,6 +1276,10 @@ static zend_result xmlreader_fixup_temporaries(void) {
12761276
++xmlreader_open_fn.T;
12771277
++xmlreader_xml_fn.T;
12781278
}
1279+
#if !ZTS
1280+
ZEND_MAP_PTR(xmlreader_open_fn.run_time_cache) = ZEND_MAP_PTR(((zend_internal_function *)zend_hash_str_find_ptr(&xmlreader_class_entry->function_table, "open", sizeof("open")-1))->run_time_cache);
1281+
ZEND_MAP_PTR(xmlreader_xml_fn.run_time_cache) = ZEND_MAP_PTR(((zend_internal_function *)zend_hash_str_find_ptr(&xmlreader_class_entry->function_table, "xml", sizeof("xml")-1))->run_time_cache);
1282+
#endif
12791283
if (prev_zend_post_startup_cb) {
12801284
return prev_zend_post_startup_cb();
12811285
}

‎main/main.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2316,6 +2316,9 @@ zend_result php_module_startup(sapi_module_struct *sf, zend_module_entry *additi
23162316
/* freeze the list of observer fcall_init handlers */
23172317
zend_observer_post_startup();
23182318

2319+
/* freeze the list of persistent internal functions */
2320+
zend_init_internal_run_time_cache();
2321+
23192322
/* Extensions that add engine hooks after this point do so at their own peril */
23202323
zend_finalize_system_id();
23212324

0 commit comments

Comments
 (0)