Skip to content

Commit 95daf1a

Browse files
snehasishsnehasish
authored
Allow optimization of __size_returning_new variants. (#102258)
#101564 added support to TLI to detect variants of operator new which provide feedback on the actual size of memory allocated (http://wg21.link/P0901R5). This patch extends SimplifyLibCalls to handle hot cold hinting of these variants.
1 parent 56140a8 commit 95daf1a

File tree

6 files changed

+199
-7
lines changed

6 files changed

+199
-7
lines changed

‎llvm/include/llvm/Transforms/Utils/BuildLibCalls.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,13 @@ namespace llvm {
267267
IRBuilderBase &B,
268268
const TargetLibraryInfo *TLI,
269269
LibFunc NewFunc, uint8_t HotCold);
270+
Value *emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
271+
const TargetLibraryInfo *TLI,
272+
LibFunc NewFunc, uint8_t HotCold);
273+
Value *emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
274+
IRBuilderBase &B,
275+
const TargetLibraryInfo *TLI,
276+
LibFunc NewFunc, uint8_t HotCold);
270277
}
271278

272279
#endif

‎llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3692,7 +3692,7 @@ Instruction *InstCombinerImpl::tryOptimizeCall(CallInst *CI) {
36923692

36933693
// Skip optimizing notail and musttail calls so
36943694
// LibCallSimplifier::optimizeCall doesn't have to preserve those invariants.
3695-
// LibCallSimplifier::optimizeCall should try to preseve tail calls though.
3695+
// LibCallSimplifier::optimizeCall should try to preserve tail calls though.
36963696
if (CI->isMustTailCall() || CI->isNoTailCall())
36973697
return nullptr;
36983698

‎llvm/lib/Transforms/Instrumentation/MemProfiler.cpp

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "llvm/ADT/StringRef.h"
2121
#include "llvm/Analysis/MemoryBuiltins.h"
2222
#include "llvm/Analysis/MemoryProfileInfo.h"
23+
#include "llvm/Analysis/TargetLibraryInfo.h"
2324
#include "llvm/Analysis/ValueTracking.h"
2425
#include "llvm/IR/Constant.h"
2526
#include "llvm/IR/DataLayout.h"
@@ -753,8 +754,8 @@ stackFrameIncludesInlinedCallStack(ArrayRef<Frame> ProfileCallStack,
753754
return InlCallStackIter == InlinedCallStack.end();
754755
}
755756

756-
static bool isNewWithHotColdVariant(Function *Callee,
757-
const TargetLibraryInfo &TLI) {
757+
static bool isAllocationWithHotColdVariant(Function *Callee,
758+
const TargetLibraryInfo &TLI) {
758759
if (!Callee)
759760
return false;
760761
LibFunc Func;
@@ -769,6 +770,8 @@ static bool isNewWithHotColdVariant(Function *Callee,
769770
case LibFunc_ZnamRKSt9nothrow_t:
770771
case LibFunc_ZnamSt11align_val_t:
771772
case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
773+
case LibFunc_size_returning_new:
774+
case LibFunc_size_returning_new_aligned:
772775
return true;
773776
case LibFunc_Znwm12__hot_cold_t:
774777
case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
@@ -778,6 +781,8 @@ static bool isNewWithHotColdVariant(Function *Callee,
778781
case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
779782
case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
780783
case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
784+
case LibFunc_size_returning_new_hot_cold:
785+
case LibFunc_size_returning_new_aligned_hot_cold:
781786
return ClMemProfMatchHotColdNew;
782787
default:
783788
return false;
@@ -945,9 +950,8 @@ readMemprof(Module &M, Function &F, IndexedInstrProfReader *MemProfReader,
945950
// instruction's locations match the prefix Frame locations on an
946951
// allocation context with the same leaf.
947952
if (AllocInfoIter != LocHashToAllocInfo.end()) {
948-
// Only consider allocations via new, to reduce unnecessary metadata,
949-
// since those are the only allocations that will be targeted initially.
950-
if (!isNewWithHotColdVariant(CI->getCalledFunction(), TLI))
953+
// Only consider allocations which support hinting.
954+
if (!isAllocationWithHotColdVariant(CI->getCalledFunction(), TLI))
951955
continue;
952956
// We may match this instruction's location list to multiple MIB
953957
// contexts. Add them to a Trie specialized for trimming the contexts to

‎llvm/lib/Transforms/Utils/BuildLibCalls.cpp

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include "llvm/IR/CallingConv.h"
2020
#include "llvm/IR/Constants.h"
2121
#include "llvm/IR/DataLayout.h"
22+
#include "llvm/IR/DerivedTypes.h"
2223
#include "llvm/IR/Function.h"
2324
#include "llvm/IR/IRBuilder.h"
2425
#include "llvm/IR/Module.h"
@@ -1963,6 +1964,56 @@ Value *llvm::emitCalloc(Value *Num, Value *Size, IRBuilderBase &B,
19631964
return CI;
19641965
}
19651966

1967+
Value *llvm::emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
1968+
const TargetLibraryInfo *TLI,
1969+
LibFunc SizeFeedbackNewFunc,
1970+
uint8_t HotCold) {
1971+
Module *M = B.GetInsertBlock()->getModule();
1972+
if (!isLibFuncEmittable(M, TLI, SizeFeedbackNewFunc))
1973+
return nullptr;
1974+
1975+
StringRef Name = TLI->getName(SizeFeedbackNewFunc);
1976+
1977+
// __sized_ptr_t struct return type { void*, size_t }
1978+
StructType *SizedPtrT =
1979+
StructType::get(M->getContext(), {B.getPtrTy(), Num->getType()});
1980+
FunctionCallee Func =
1981+
M->getOrInsertFunction(Name, SizedPtrT, Num->getType(), B.getInt8Ty());
1982+
inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
1983+
CallInst *CI = B.CreateCall(Func, {Num, B.getInt8(HotCold)}, "sized_ptr");
1984+
1985+
if (const Function *F = dyn_cast<Function>(Func.getCallee()))
1986+
CI->setCallingConv(F->getCallingConv());
1987+
1988+
return CI;
1989+
}
1990+
1991+
Value *llvm::emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
1992+
IRBuilderBase &B,
1993+
const TargetLibraryInfo *TLI,
1994+
LibFunc SizeFeedbackNewFunc,
1995+
uint8_t HotCold) {
1996+
Module *M = B.GetInsertBlock()->getModule();
1997+
if (!isLibFuncEmittable(M, TLI, SizeFeedbackNewFunc))
1998+
return nullptr;
1999+
2000+
StringRef Name = TLI->getName(SizeFeedbackNewFunc);
2001+
2002+
// __sized_ptr_t struct return type { void*, size_t }
2003+
StructType *SizedPtrT =
2004+
StructType::get(M->getContext(), {B.getPtrTy(), Num->getType()});
2005+
FunctionCallee Func = M->getOrInsertFunction(Name, SizedPtrT, Num->getType(),
2006+
Align->getType(), B.getInt8Ty());
2007+
inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
2008+
CallInst *CI =
2009+
B.CreateCall(Func, {Num, Align, B.getInt8(HotCold)}, "sized_ptr");
2010+
2011+
if (const Function *F = dyn_cast<Function>(Func.getCallee()))
2012+
CI->setCallingConv(F->getCallingConv());
2013+
2014+
return CI;
2015+
}
2016+
19662017
Value *llvm::emitHotColdNew(Value *Num, IRBuilderBase &B,
19672018
const TargetLibraryInfo *TLI, LibFunc NewFunc,
19682019
uint8_t HotCold) {

‎llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "llvm/Analysis/ConstantFolding.h"
1919
#include "llvm/Analysis/Loads.h"
2020
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
21+
#include "llvm/Analysis/TargetLibraryInfo.h"
2122
#include "llvm/Analysis/ValueTracking.h"
2223
#include "llvm/IR/AttributeMask.h"
2324
#include "llvm/IR/DataLayout.h"
@@ -1745,7 +1746,7 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
17451746
// if cold or hot, and leave as-is for default handling if "notcold" aka warm.
17461747
// Note that in cases where we decide it is "notcold", it might be slightly
17471748
// better to replace the hinted call with a non hinted call, to avoid the
1748-
// extra paramter and the if condition check of the hint value in the
1749+
// extra parameter and the if condition check of the hint value in the
17491750
// allocator. This can be considered in the future.
17501751
switch (Func) {
17511752
case LibFunc_Znwm12__hot_cold_t:
@@ -1844,6 +1845,30 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
18441845
TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
18451846
HotCold);
18461847
break;
1848+
case LibFunc_size_returning_new:
1849+
if (HotCold != NotColdNewHintValue)
1850+
return emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
1851+
LibFunc_size_returning_new_hot_cold,
1852+
HotCold);
1853+
break;
1854+
case LibFunc_size_returning_new_hot_cold:
1855+
if (OptimizeExistingHotColdNew)
1856+
return emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
1857+
LibFunc_size_returning_new_hot_cold,
1858+
HotCold);
1859+
break;
1860+
case LibFunc_size_returning_new_aligned:
1861+
if (HotCold != NotColdNewHintValue)
1862+
return emitHotColdSizeReturningNewAligned(
1863+
CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1864+
LibFunc_size_returning_new_aligned_hot_cold, HotCold);
1865+
break;
1866+
case LibFunc_size_returning_new_aligned_hot_cold:
1867+
if (OptimizeExistingHotColdNew)
1868+
return emitHotColdSizeReturningNewAligned(
1869+
CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1870+
LibFunc_size_returning_new_aligned_hot_cold, HotCold);
1871+
break;
18471872
default:
18481873
return nullptr;
18491874
}
@@ -3759,6 +3784,7 @@ Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
37593784
Module *M = CI->getModule();
37603785
LibFunc Func;
37613786
Function *Callee = CI->getCalledFunction();
3787+
37623788
// Check for string/memory library functions.
37633789
if (TLI->getLibFunc(*Callee, Func) && isLibFuncEmittable(M, TLI, Func)) {
37643790
// Make sure we never change the calling convention.
@@ -3851,6 +3877,10 @@ Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
38513877
case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
38523878
case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
38533879
case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
3880+
case LibFunc_size_returning_new:
3881+
case LibFunc_size_returning_new_hot_cold:
3882+
case LibFunc_size_returning_new_aligned:
3883+
case LibFunc_size_returning_new_aligned_hot_cold:
38543884
return optimizeNew(CI, Builder, Func);
38553885
default:
38563886
break;

‎llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,94 @@ define void @array_new_align_nothrow_hot_cold() {
340340
ret void
341341
}
342342

343+
;; Check that operator __size_returning_new(unsigned long) converted to
344+
;; __size_returning_new(unsigned long, __hot_cold_t) with a hot or cold attribute.
345+
; HOTCOLD-LABEL: @size_returning_test()
346+
define void @size_returning_test() {
347+
;; Attribute cold converted to __hot_cold_t cold value.
348+
; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[COLD]])
349+
%call = call {ptr, i64} @__size_returning_new(i64 10) #3
350+
%p = extractvalue {ptr, i64} %call, 0
351+
call void @dummy(ptr %p)
352+
;; Attribute notcold has no effect.
353+
; HOTCOLD: @__size_returning_new(i64 10)
354+
%call1 = call {ptr, i64} @__size_returning_new(i64 10) #4
355+
%p1 = extractvalue {ptr, i64} %call1, 0
356+
call void @dummy(ptr %p1)
357+
;; Attribute hot converted to __hot_cold_t hot value.
358+
; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[HOT]])
359+
%call2 = call {ptr, i64} @__size_returning_new(i64 10) #5
360+
%p2 = extractvalue {ptr, i64} %call2, 0
361+
call void @dummy(ptr %p2)
362+
ret void
363+
}
364+
365+
;; Check that operator __size_returning_new_aligned(unsigned long, std::align_val_t) converted to
366+
;; __size_returning_new_aligned(unsigned long, std::align_val_t, __hot_cold_t) with a hot or cold attribute.
367+
; HOTCOLD-LABEL: @size_returning_aligned_test()
368+
define void @size_returning_aligned_test() {
369+
;; Attribute cold converted to __hot_cold_t cold value.
370+
; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[COLD]])
371+
%call = call {ptr, i64} @__size_returning_new_aligned(i64 10, i64 8) #3
372+
%p = extractvalue {ptr, i64} %call, 0
373+
call void @dummy(ptr %p)
374+
;; Attribute notcold has no effect.
375+
; HOTCOLD: @__size_returning_new_aligned(i64 10, i64 8)
376+
%call1 = call {ptr, i64} @__size_returning_new_aligned(i64 10, i64 8) #4
377+
%p1 = extractvalue {ptr, i64} %call1, 0
378+
call void @dummy(ptr %p1)
379+
;; Attribute hot converted to __hot_cold_t hot value.
380+
; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[HOT]])
381+
%call2 = call {ptr, i64} @__size_returning_new_aligned(i64 10, i64 8) #5
382+
%p2 = extractvalue {ptr, i64} %call2, 0
383+
call void @dummy(ptr %p2)
384+
ret void
385+
}
386+
387+
;; Check that __size_returning_new_hot_cold(unsigned long, __hot_cold_t)
388+
;; optionally has its hint updated.
389+
; HOTCOLD-LABEL: @size_returning_update_test()
390+
define void @size_returning_update_test() {
391+
;; Attribute cold converted to __hot_cold_t cold value.
392+
; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[PREVHINTCOLD]])
393+
%call = call {ptr, i64} @__size_returning_new_hot_cold(i64 10, i8 7) #3
394+
%p = extractvalue {ptr, i64} %call, 0
395+
call void @dummy(ptr %p)
396+
;; Attribute notcold converted to __hot_cold_t notcold value.
397+
; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[PREVHINTNOTCOLD]])
398+
%call1 = call {ptr, i64} @__size_returning_new_hot_cold(i64 10, i8 7) #4
399+
%p1 = extractvalue {ptr, i64} %call1, 0
400+
call void @dummy(ptr %p1)
401+
;; Attribute hot converted to __hot_cold_t hot value.
402+
; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[PREVHINTHOT]])
403+
%call2 = call {ptr, i64} @__size_returning_new_hot_cold(i64 10, i8 7) #5
404+
%p2 = extractvalue {ptr, i64} %call2, 0
405+
call void @dummy(ptr %p2)
406+
ret void
407+
}
408+
409+
;; Check that __size_returning_new_aligned_hot_cold(unsigned long, __hot_cold_t)
410+
;; optionally has its hint updated.
411+
; HOTCOLD-LABEL: @size_returning_aligned_update_test()
412+
define void @size_returning_aligned_update_test() {
413+
;; Attribute cold converted to __hot_cold_t cold value.
414+
; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[PREVHINTCOLD]])
415+
%call = call {ptr, i64} @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 7) #3
416+
%p = extractvalue {ptr, i64} %call, 0
417+
call void @dummy(ptr %p)
418+
;; Attribute notcold converted to __hot_cold_t notcold value.
419+
; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[PREVHINTNOTCOLD]])
420+
%call1 = call {ptr, i64} @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 7) #4
421+
%p1 = extractvalue {ptr, i64} %call1, 0
422+
call void @dummy(ptr %p1)
423+
;; Attribute hot converted to __hot_cold_t hot value.
424+
; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[PREVHINTHOT]])
425+
%call2 = call {ptr, i64} @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 7) #5
426+
%p2 = extractvalue {ptr, i64} %call2, 0
427+
call void @dummy(ptr %p2)
428+
ret void
429+
}
430+
343431
;; So that instcombine doesn't optimize out the call.
344432
declare void @dummy(ptr)
345433

@@ -360,6 +448,18 @@ declare ptr @_ZnamSt11align_val_t12__hot_cold_t(i64, i64, i8)
360448
declare ptr @_ZnamRKSt9nothrow_t12__hot_cold_t(i64, ptr, i8)
361449
declare ptr @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64, i64, ptr, i8)
362450

451+
452+
declare {ptr, i64} @__size_returning_new(i64)
453+
declare {ptr, i64} @__size_returning_new_hot_cold(i64, i8)
454+
declare {ptr, i64} @__size_returning_new_aligned(i64, i64)
455+
declare {ptr, i64} @__size_returning_new_aligned_hot_cold(i64, i64, i8)
456+
363457
attributes #0 = { builtin allocsize(0) "memprof"="cold" }
364458
attributes #1 = { builtin allocsize(0) "memprof"="notcold" }
365459
attributes #2 = { builtin allocsize(0) "memprof"="hot" }
460+
461+
;; Use separate attributes for __size_returning_new variants since they are not
462+
;; treated as builtins.
463+
attributes #3 = { "memprof" = "cold" }
464+
attributes #4 = { "memprof" = "notcold" }
465+
attributes #5 = { "memprof" = "hot" }

0 commit comments

Comments
 (0)