Skip to content

Allow optimization of __size_returning_new variants. #102258

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 15, 2024
Merged

Allow optimization of __size_returning_new variants. #102258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
39a1919
Allow optimization of __size_returning_new variants.
snehasish Aug 7, 2024
652f2ba
clang format.
snehasish Aug 7, 2024
0ae7c50
Address comments.
snehasish Aug 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,13 @@ namespace llvm {
IRBuilderBase &B,
const TargetLibraryInfo *TLI,
LibFunc NewFunc, uint8_t HotCold);
Value *emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
const TargetLibraryInfo *TLI,
LibFunc NewFunc, uint8_t HotCold);
Value *emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
IRBuilderBase &B,
const TargetLibraryInfo *TLI,
LibFunc NewFunc, uint8_t HotCold);
}

#endif
2 changes: 1 addition & 1 deletion llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3694,7 +3694,7 @@ Instruction *InstCombinerImpl::tryOptimizeCall(CallInst *CI) {

// Skip optimizing notail and musttail calls so
// LibCallSimplifier::optimizeCall doesn't have to preserve those invariants.
// LibCallSimplifier::optimizeCall should try to preseve tail calls though.
// LibCallSimplifier::optimizeCall should try to preserve tail calls though.
if (CI->isMustTailCall() || CI->isNoTailCall())
return nullptr;

Expand Down
14 changes: 9 additions & 5 deletions llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemoryProfileInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/DataLayout.h"
Expand Down Expand Up @@ -753,8 +754,8 @@ stackFrameIncludesInlinedCallStack(ArrayRef<Frame> ProfileCallStack,
return InlCallStackIter == InlinedCallStack.end();
}

static bool isNewWithHotColdVariant(Function *Callee,
const TargetLibraryInfo &TLI) {
static bool isAllocationWithHotColdVariant(Function *Callee,
const TargetLibraryInfo &TLI) {
if (!Callee)
return false;
LibFunc Func;
Expand All @@ -769,6 +770,8 @@ static bool isNewWithHotColdVariant(Function *Callee,
case LibFunc_ZnamRKSt9nothrow_t:
case LibFunc_ZnamSt11align_val_t:
case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
case LibFunc_size_returning_new:
case LibFunc_size_returning_new_aligned:
return true;
case LibFunc_Znwm12__hot_cold_t:
case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
Expand All @@ -778,6 +781,8 @@ static bool isNewWithHotColdVariant(Function *Callee,
case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
case LibFunc_size_returning_new_hot_cold:
case LibFunc_size_returning_new_aligned_hot_cold:
return ClMemProfMatchHotColdNew;
default:
return false;
Expand Down Expand Up @@ -945,9 +950,8 @@ readMemprof(Module &M, Function &F, IndexedInstrProfReader *MemProfReader,
// instruction's locations match the prefix Frame locations on an
// allocation context with the same leaf.
if (AllocInfoIter != LocHashToAllocInfo.end()) {
// Only consider allocations via new, to reduce unnecessary metadata,
// since those are the only allocations that will be targeted initially.
if (!isNewWithHotColdVariant(CI->getCalledFunction(), TLI))
// Only consider allocations which support hinting.
if (!isAllocationWithHotColdVariant(CI->getCalledFunction(), TLI))
continue;
// We may match this instruction's location list to multiple MIB
// contexts. Add them to a Trie specialized for trimming the contexts to
Expand Down
51 changes: 51 additions & 0 deletions llvm/lib/Transforms/Utils/BuildLibCalls.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Module.h"
Expand Down Expand Up @@ -1963,6 +1964,56 @@ Value *llvm::emitCalloc(Value *Num, Value *Size, IRBuilderBase &B,
return CI;
}

Value *llvm::emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
const TargetLibraryInfo *TLI,
LibFunc SizeFeedbackNewFunc,
uint8_t HotCold) {
Module *M = B.GetInsertBlock()->getModule();
if (!isLibFuncEmittable(M, TLI, SizeFeedbackNewFunc))
return nullptr;

StringRef Name = TLI->getName(SizeFeedbackNewFunc);

// __sized_ptr_t struct return type { void*, size_t }
StructType *SizedPtrT =
StructType::get(M->getContext(), {B.getPtrTy(), Num->getType()});
FunctionCallee Func =
M->getOrInsertFunction(Name, SizedPtrT, Num->getType(), B.getInt8Ty());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please use getOrInsertLibFunc instead.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When trying to do so I hit the assertion here: https://github.com/llvm/llvm-project/blob/main/llvm/lib/Transforms/Utils/BuildLibCalls.cpp#L1375-L1379

I think we'll need to change getOrInsertLibFunc to handle these cases if we want to use it. Given that the other transforms related to operator new don't use it I'll leave it as is. If you want we can follow up with a patch which addresses them together.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be good to follow up on this, as it may cause issues on s390x.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Filed #107321 to track so that I don't forget.

inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
CallInst *CI = B.CreateCall(Func, {Num, B.getInt8(HotCold)}, "sized_ptr");

if (const Function *F = dyn_cast<Function>(Func.getCallee()))
CI->setCallingConv(F->getCallingConv());

return CI;
}

Value *llvm::emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
IRBuilderBase &B,
const TargetLibraryInfo *TLI,
LibFunc SizeFeedbackNewFunc,
uint8_t HotCold) {
Module *M = B.GetInsertBlock()->getModule();
if (!isLibFuncEmittable(M, TLI, SizeFeedbackNewFunc))
return nullptr;

StringRef Name = TLI->getName(SizeFeedbackNewFunc);

// __sized_ptr_t struct return type { void*, size_t }
StructType *SizedPtrT =
StructType::get(M->getContext(), {B.getPtrTy(), Num->getType()});
FunctionCallee Func = M->getOrInsertFunction(Name, SizedPtrT, Num->getType(),
Align->getType(), B.getInt8Ty());
inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
CallInst *CI =
B.CreateCall(Func, {Num, Align, B.getInt8(HotCold)}, "sized_ptr");

if (const Function *F = dyn_cast<Function>(Func.getCallee()))
CI->setCallingConv(F->getCallingConv());

return CI;
}

Value *llvm::emitHotColdNew(Value *Num, IRBuilderBase &B,
const TargetLibraryInfo *TLI, LibFunc NewFunc,
uint8_t HotCold) {
Expand Down
32 changes: 31 additions & 1 deletion llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/AttributeMask.h"
#include "llvm/IR/DataLayout.h"
Expand Down Expand Up @@ -1745,7 +1746,7 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
// if cold or hot, and leave as-is for default handling if "notcold" aka warm.
// Note that in cases where we decide it is "notcold", it might be slightly
// better to replace the hinted call with a non hinted call, to avoid the
// extra paramter and the if condition check of the hint value in the
// extra parameter and the if condition check of the hint value in the
// allocator. This can be considered in the future.
switch (Func) {
case LibFunc_Znwm12__hot_cold_t:
Expand Down Expand Up @@ -1844,6 +1845,30 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
HotCold);
break;
case LibFunc_size_returning_new:
if (HotCold != NotColdNewHintValue)
return emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
LibFunc_size_returning_new_hot_cold,
HotCold);
break;
case LibFunc_size_returning_new_hot_cold:
if (OptimizeExistingHotColdNew)
return emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
LibFunc_size_returning_new_hot_cold,
HotCold);
break;
case LibFunc_size_returning_new_aligned:
if (HotCold != NotColdNewHintValue)
return emitHotColdSizeReturningNewAligned(
CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
LibFunc_size_returning_new_aligned_hot_cold, HotCold);
break;
case LibFunc_size_returning_new_aligned_hot_cold:
if (OptimizeExistingHotColdNew)
return emitHotColdSizeReturningNewAligned(
CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
LibFunc_size_returning_new_aligned_hot_cold, HotCold);
break;
default:
return nullptr;
}
Expand Down Expand Up @@ -3759,6 +3784,7 @@ Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
Module *M = CI->getModule();
LibFunc Func;
Function *Callee = CI->getCalledFunction();

// Check for string/memory library functions.
if (TLI->getLibFunc(*Callee, Func) && isLibFuncEmittable(M, TLI, Func)) {
// Make sure we never change the calling convention.
Expand Down Expand Up @@ -3851,6 +3877,10 @@ Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
case LibFunc_size_returning_new:
case LibFunc_size_returning_new_hot_cold:
case LibFunc_size_returning_new_aligned:
case LibFunc_size_returning_new_aligned_hot_cold:
return optimizeNew(CI, Builder, Func);
default:
break;
Expand Down
100 changes: 100 additions & 0 deletions llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,94 @@ define void @array_new_align_nothrow_hot_cold() {
ret void
}

;; Check that operator __size_returning_new(unsigned long) converted to
;; __size_returning_new(unsigned long, __hot_cold_t) with a hot or cold attribute.
; HOTCOLD-LABEL: @size_returning_test()
define void @size_returning_test() {
;; Attribute cold converted to __hot_cold_t cold value.
; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[COLD]])
%call = call {ptr, i64} @__size_returning_new(i64 10) #3
%p = extractvalue {ptr, i64} %call, 0
call void @dummy(ptr %p)
;; Attribute notcold has no effect.
; HOTCOLD: @__size_returning_new(i64 10)
%call1 = call {ptr, i64} @__size_returning_new(i64 10) #4
%p1 = extractvalue {ptr, i64} %call1, 0
call void @dummy(ptr %p1)
;; Attribute hot converted to __hot_cold_t hot value.
; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[HOT]])
%call2 = call {ptr, i64} @__size_returning_new(i64 10) #5
%p2 = extractvalue {ptr, i64} %call2, 0
call void @dummy(ptr %p2)
ret void
}

;; Check that operator __size_returning_new_aligned(unsigned long, std::align_val_t) converted to
;; __size_returning_new_aligned(unsigned long, std::align_val_t, __hot_cold_t) with a hot or cold attribute.
; HOTCOLD-LABEL: @size_returning_aligned_test()
define void @size_returning_aligned_test() {
;; Attribute cold converted to __hot_cold_t cold value.
; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[COLD]])
%call = call {ptr, i64} @__size_returning_new_aligned(i64 10, i64 8) #3
%p = extractvalue {ptr, i64} %call, 0
call void @dummy(ptr %p)
;; Attribute notcold has no effect.
; HOTCOLD: @__size_returning_new_aligned(i64 10, i64 8)
%call1 = call {ptr, i64} @__size_returning_new_aligned(i64 10, i64 8) #4
%p1 = extractvalue {ptr, i64} %call1, 0
call void @dummy(ptr %p1)
;; Attribute hot converted to __hot_cold_t hot value.
; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[HOT]])
%call2 = call {ptr, i64} @__size_returning_new_aligned(i64 10, i64 8) #5
%p2 = extractvalue {ptr, i64} %call2, 0
call void @dummy(ptr %p2)
ret void
}

;; Check that __size_returning_new_hot_cold(unsigned long, __hot_cold_t)
;; optionally has its hint updated.
; HOTCOLD-LABEL: @size_returning_update_test()
define void @size_returning_update_test() {
;; Attribute cold converted to __hot_cold_t cold value.
; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[PREVHINTCOLD]])
%call = call {ptr, i64} @__size_returning_new_hot_cold(i64 10, i8 7) #3
%p = extractvalue {ptr, i64} %call, 0
call void @dummy(ptr %p)
;; Attribute notcold converted to __hot_cold_t notcold value.
; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[PREVHINTNOTCOLD]])
%call1 = call {ptr, i64} @__size_returning_new_hot_cold(i64 10, i8 7) #4
%p1 = extractvalue {ptr, i64} %call1, 0
call void @dummy(ptr %p1)
;; Attribute hot converted to __hot_cold_t hot value.
; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[PREVHINTHOT]])
%call2 = call {ptr, i64} @__size_returning_new_hot_cold(i64 10, i8 7) #5
%p2 = extractvalue {ptr, i64} %call2, 0
call void @dummy(ptr %p2)
ret void
}

;; Check that __size_returning_new_aligned_hot_cold(unsigned long, __hot_cold_t)
;; optionally has its hint updated.
; HOTCOLD-LABEL: @size_returning_aligned_update_test()
define void @size_returning_aligned_update_test() {
;; Attribute cold converted to __hot_cold_t cold value.
; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[PREVHINTCOLD]])
%call = call {ptr, i64} @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 7) #3
%p = extractvalue {ptr, i64} %call, 0
call void @dummy(ptr %p)
;; Attribute notcold converted to __hot_cold_t notcold value.
; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[PREVHINTNOTCOLD]])
%call1 = call {ptr, i64} @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 7) #4
%p1 = extractvalue {ptr, i64} %call1, 0
call void @dummy(ptr %p1)
;; Attribute hot converted to __hot_cold_t hot value.
; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[PREVHINTHOT]])
%call2 = call {ptr, i64} @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 7) #5
%p2 = extractvalue {ptr, i64} %call2, 0
call void @dummy(ptr %p2)
ret void
}

;; So that instcombine doesn't optimize out the call.
declare void @dummy(ptr)

Expand All @@ -360,6 +448,18 @@ declare ptr @_ZnamSt11align_val_t12__hot_cold_t(i64, i64, i8)
declare ptr @_ZnamRKSt9nothrow_t12__hot_cold_t(i64, ptr, i8)
declare ptr @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64, i64, ptr, i8)


declare {ptr, i64} @__size_returning_new(i64)
declare {ptr, i64} @__size_returning_new_hot_cold(i64, i8)
declare {ptr, i64} @__size_returning_new_aligned(i64, i64)
declare {ptr, i64} @__size_returning_new_aligned_hot_cold(i64, i64, i8)

attributes #0 = { builtin allocsize(0) "memprof"="cold" }
attributes #1 = { builtin allocsize(0) "memprof"="notcold" }
attributes #2 = { builtin allocsize(0) "memprof"="hot" }

;; Use separate attributes for __size_returning_new variants since they are not
;; treated as builtins.
attributes #3 = { "memprof" = "cold" }
attributes #4 = { "memprof" = "notcold" }
attributes #5 = { "memprof" = "hot" }
Loading