Make internal run_time_cache a persistent allocation

We also add zend_map_ptr_static, so that we do not incur the overhead of constantly recreating the internal run_time_cache pointers on each request.
This mechanism might be extended for mutable_data of internal classes too.

Author: [email protected]

Zend/zend.c

@@ -742,6 +742,7 @@ static void compiler_globals_ctor(zend_compiler_globals *compiler_globals) /* {{
 		compiler_globals->map_ptr_real_base = base;
 		compiler_globals->map_ptr_base = ZEND_MAP_PTR_BIASED_BASE(base);
 		memset(base, 0, compiler_globals->map_ptr_last * sizeof(void*));
+		zend_init_internal_run_time_cache();
 	}
 }
 /* }}} */
@@ -785,6 +786,9 @@ static void compiler_globals_dtor(zend_compiler_globals *compiler_globals) /* {{
 		compiler_globals->map_ptr_base = ZEND_MAP_PTR_BIASED_BASE(NULL);
 		compiler_globals->map_ptr_size = 0;
 	}
+	if (compiler_globals->internal_run_time_cache) {
+		pefree(compiler_globals->internal_run_time_cache, 1);
+	}
 }
 /* }}} */
 
@@ -1083,6 +1087,7 @@ zend_result zend_post_startup(void) /* {{{ */
 
 	zend_compiler_globals *compiler_globals = ts_resource(compiler_globals_id);
 	zend_executor_globals *executor_globals = ts_resource(executor_globals_id);
+	global_map_ptr_last = compiler_globals->map_ptr_last;
 #endif
 
 	startup_done = true;
@@ -1113,6 +1118,9 @@ zend_result zend_post_startup(void) /* {{{ */
 	if (compiler_globals->map_ptr_real_base) {
 		free(compiler_globals->map_ptr_real_base);
 	}
+	if (compiler_globals->internal_run_time_cache) {
+		pefree(compiler_globals->internal_run_time_cache, 1);
+	}
 	compiler_globals->map_ptr_real_base = NULL;
 	compiler_globals->map_ptr_base = ZEND_MAP_PTR_BIASED_BASE(NULL);
 	if ((script_encoding_list = (zend_encoding **)compiler_globals->script_encoding_list)) {
@@ -1198,6 +1206,9 @@ void zend_shutdown(void) /* {{{ */
 		CG(script_encoding_list_size) = 0;
 	}
 #endif
+	zend_map_ptr_static_last = 0;
+	zend_map_ptr_static_size = 0;
+
 	zend_destroy_rsrc_list_dtors();
 
 	zend_unload_modules();
@@ -1297,9 +1308,9 @@ ZEND_API void zend_activate(void) /* {{{ */
 	init_executor();
 	startup_scanner();
 	if (CG(map_ptr_last)) {
-		memset(CG(map_ptr_real_base), 0, CG(map_ptr_last) * sizeof(void*));
+		memset((char *)CG(map_ptr_real_base) + zend_map_ptr_static_size * sizeof(void*), 0, (CG(map_ptr_last) - zend_map_ptr_static_size) * sizeof(void*));
 	}
-	zend_init_internal_run_time_cache();
+	zend_reset_internal_run_time_cache();
 	zend_observer_activate();
 }
 /* }}} */
@@ -1984,6 +1995,9 @@ void free_estring(char **str_p) /* {{{ */
 }
 /* }}} */
 
+ZEND_API size_t zend_map_ptr_static_size;
+ZEND_API size_t zend_map_ptr_static_last;
+
 ZEND_API void zend_map_ptr_reset(void)
 {
 	CG(map_ptr_last) = global_map_ptr_last;
@@ -2005,6 +2019,25 @@ ZEND_API void *zend_map_ptr_new(void)
 	return ZEND_MAP_PTR_PTR2OFFSET(ptr);
 }
 
+ZEND_API void *zend_map_ptr_new_static(void)
+{
+	void **ptr;
+
+	if (zend_map_ptr_static_last >= zend_map_ptr_static_size) {
+		CG(map_ptr_last) += 4096;
+		zend_map_ptr_static_size += 4096;
+		/* Grow map_ptr table */
+		CG(map_ptr_size) = ZEND_MM_ALIGNED_SIZE_EX(CG(map_ptr_last), 4096);
+		/* Note: there are no used non-static map_ptrs yet, hence we don't need to move the whole thing */
+		CG(map_ptr_real_base) = perealloc(CG(map_ptr_real_base), CG(map_ptr_size) * sizeof(void*), 1);
+		CG(map_ptr_base) = ZEND_MAP_PTR_BIASED_BASE(CG(map_ptr_real_base));
+	}
+	ptr = (void**)CG(map_ptr_real_base) + zend_map_ptr_static_last;
+	*ptr = NULL;
+	zend_map_ptr_static_last++;
+	return ZEND_MAP_PTR_PTR2OFFSET(ptr);
+}
+
 ZEND_API void zend_map_ptr_extend(size_t last)
 {
 	if (last > CG(map_ptr_last)) {

Zend/zend_API.c

@@ -2952,7 +2952,7 @@ ZEND_API zend_result zend_register_functions(zend_class_entry *scope, const zend
 		if (EG(active)) { // at run-time: this ought to only happen if registered with dl() or somehow temporarily at runtime
 			ZEND_MAP_PTR_INIT(internal_function->run_time_cache, zend_arena_calloc(&CG(arena), 1, zend_internal_run_time_cache_reserved_size()));
 		} else {
-			ZEND_MAP_PTR_NEW(internal_function->run_time_cache);
+			ZEND_MAP_PTR_NEW_STATIC(internal_function->run_time_cache);
 		}
 		if (ptr->flags) {
 			if (!(ptr->flags & ZEND_ACC_PPP_MASK)) {

Zend/zend_enum.c

@@ -421,7 +421,7 @@ static void zend_enum_register_func(zend_class_entry *ce, zend_known_string_id n
     if (EG(active)) { // at run-time
 		ZEND_MAP_PTR_INIT(zif->run_time_cache, zend_arena_calloc(&CG(arena), 1, zend_internal_run_time_cache_reserved_size()));
 	} else {
-		ZEND_MAP_PTR_NEW(zif->run_time_cache);
+		ZEND_MAP_PTR_NEW_STATIC(zif->run_time_cache);
 	}
 
 	if (!zend_hash_add_ptr(&ce->function_table, name, zif)) {

Zend/zend_extensions.c

@@ -331,19 +331,22 @@ ZEND_API void zend_init_internal_run_time_cache(void) {
 			functions += zend_hash_num_elements(&ce->function_table);
 		} ZEND_HASH_FOREACH_END();
 
-		char *ptr = zend_arena_calloc(&CG(arena), functions, rt_size);
+		size_t alloc_size = functions * rt_size;
+		char *ptr = pemalloc(alloc_size, 1);
+
+		CG(internal_run_time_cache) = ptr;
+		CG(internal_run_time_cache_size) = alloc_size;
+
 		zend_internal_function *zif;
 		ZEND_HASH_MAP_FOREACH_PTR(CG(function_table), zif) {
-			if (!ZEND_USER_CODE(zif->type) && ZEND_MAP_PTR_GET(zif->run_time_cache) == NULL)
-			{
+			if (!ZEND_USER_CODE(zif->type) && ZEND_MAP_PTR_GET(zif->run_time_cache) == NULL) {
 				ZEND_MAP_PTR_SET(zif->run_time_cache, (void *)ptr);
 				ptr += rt_size;
 			}
 		} ZEND_HASH_FOREACH_END();
 		ZEND_HASH_MAP_FOREACH_PTR(CG(class_table), ce) {
 			ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, zif) {
-				if (!ZEND_USER_CODE(zif->type) && ZEND_MAP_PTR_GET(zif->run_time_cache) == NULL)
-				{
+				if (!ZEND_USER_CODE(zif->type) && ZEND_MAP_PTR_GET(zif->run_time_cache) == NULL) {
 					ZEND_MAP_PTR_SET(zif->run_time_cache, (void *)ptr);
 					ptr += rt_size;
 				}
@@ -352,6 +355,12 @@ ZEND_API void zend_init_internal_run_time_cache(void) {
 	}
 }
 
+ZEND_API void zend_reset_internal_run_time_cache(void) {
+	if (CG(internal_run_time_cache)) {
+		memset(CG(internal_run_time_cache), 0, CG(internal_run_time_cache_size));
+	}
+}
+
 ZEND_API zend_extension *zend_get_extension(const char *extension_name)
 {
 	zend_llist_element *element;

Zend/zend_extensions.h

@@ -149,6 +149,7 @@ void zend_shutdown_extensions(void);
 
 ZEND_API size_t zend_internal_run_time_cache_reserved_size(void);
 ZEND_API void zend_init_internal_run_time_cache(void);
+ZEND_API void zend_reset_internal_run_time_cache(void);
 
 BEGIN_EXTERN_C()
 ZEND_API zend_result zend_load_extension(const char *path);

Zend/zend_globals.h

@@ -153,6 +153,9 @@ struct _zend_compiler_globals {
 
 	uint32_t rtd_key_counter;
 
+	void *internal_run_time_cache;
+	uint32_t internal_run_time_cache_size;
+
 	zend_stack short_circuiting_opnums;
 #ifdef ZTS
 	uint32_t copied_functions_count;

Zend/zend_map_ptr.h

@@ -42,18 +42,21 @@ typedef struct _zend_string zend_string;
 #define ZEND_MAP_PTR_NEW(ptr) do { \
 		ZEND_MAP_PTR(ptr) = zend_map_ptr_new(); \
 	} while (0)
+#define ZEND_MAP_PTR_NEW_STATIC(ptr) do { \
+		ZEND_MAP_PTR(ptr) = zend_map_ptr_new_static(); \
+	} while (0)
 
 #if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET
 # define ZEND_MAP_PTR_NEW_OFFSET() \
-	((uint32_t)(uintptr_t)zend_map_ptr_new())
+	((uint32_t)(intptr_t)zend_map_ptr_new())
 # define ZEND_MAP_PTR_IS_OFFSET(ptr) \
 	(((uintptr_t)ZEND_MAP_PTR(ptr)) & 1L)
 # define ZEND_MAP_PTR_GET(ptr) \
 	((ZEND_MAP_PTR_IS_OFFSET(ptr) ? \
 		ZEND_MAP_PTR_GET_IMM(ptr) : \
 		((void*)(ZEND_MAP_PTR(ptr)))))
 # define ZEND_MAP_PTR_GET_IMM(ptr) \
-	(*ZEND_MAP_PTR_OFFSET2PTR((uintptr_t)ZEND_MAP_PTR(ptr)))
+	(*ZEND_MAP_PTR_OFFSET2PTR((intptr_t)ZEND_MAP_PTR(ptr)))
 # define ZEND_MAP_PTR_SET(ptr, val) do { \
 		if (ZEND_MAP_PTR_IS_OFFSET(ptr)) { \
 			ZEND_MAP_PTR_SET_IMM(ptr, val); \
@@ -62,11 +65,11 @@ typedef struct _zend_string zend_string;
 		} \
 	} while (0)
 # define ZEND_MAP_PTR_SET_IMM(ptr, val) do { \
-		void **__p = ZEND_MAP_PTR_OFFSET2PTR((uintptr_t)ZEND_MAP_PTR(ptr)); \
+		void **__p = ZEND_MAP_PTR_OFFSET2PTR((intptr_t)ZEND_MAP_PTR(ptr)); \
 		*__p = (val); \
 	} while (0)
 # define ZEND_MAP_PTR_BIASED_BASE(real_base) \
-	((void*)(((uintptr_t)(real_base)) - 1))
+	((void*)(((uintptr_t)(real_base)) + zend_map_ptr_static_size * sizeof(void *) - 1))
 #else
 # error "Unknown ZEND_MAP_PTR_KIND"
 #endif
@@ -75,9 +78,13 @@ BEGIN_EXTERN_C()
 
 ZEND_API void  zend_map_ptr_reset(void);
 ZEND_API void *zend_map_ptr_new(void);
+ZEND_API void *zend_map_ptr_new_static(void);
 ZEND_API void  zend_map_ptr_extend(size_t last);
 ZEND_API void zend_alloc_ce_cache(zend_string *type_name);
 
+ZEND_API extern size_t zend_map_ptr_static_last;
+ZEND_API extern size_t zend_map_ptr_static_size;
+
 END_EXTERN_C()
 
 #endif /* ZEND_MAP_PTR_H */

main/main.c

@@ -2276,6 +2276,9 @@ zend_result php_module_startup(sapi_module_struct *sf, zend_module_entry *additi
 	/* freeze the list of observer fcall_init handlers */
 	zend_observer_post_startup();
 
+	/* freeze the list of persistent internal functions */
+	zend_init_internal_run_time_cache();
+
 	/* Extensions that add engine hooks after this point do so at their own peril */
 	zend_finalize_system_id();