Skip to content

Commit dfb903c

Browse files
yuxuanchen1997yuxuanchen1997
committed
[Clang] Introduce [[clang::coro_await_elidable]]
1 parent 56ee6a1 commit dfb903c

24 files changed

+375
-130
lines changed

‎clang/include/clang/AST/ExprCXX.h

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5082,18 +5082,19 @@ class CoroutineSuspendExpr : public Expr {
50825082
enum SubExpr { Operand, Common, Ready, Suspend, Resume, Count };
50835083

50845084
Stmt *SubExprs[SubExpr::Count];
5085-
OpaqueValueExpr *OpaqueValue = nullptr;
5085+
OpaqueValueExpr *CommonExprOpaqueValue = nullptr;
5086+
OpaqueValueExpr *InplaceCallOpaqueValue = nullptr;
50865087

50875088
public:
50885089
// These types correspond to the three C++ 'await_suspend' return variants
50895090
enum class SuspendReturnType { SuspendVoid, SuspendBool, SuspendHandle };
50905091

50915092
CoroutineSuspendExpr(StmtClass SC, SourceLocation KeywordLoc, Expr *Operand,
50925093
Expr *Common, Expr *Ready, Expr *Suspend, Expr *Resume,
5093-
OpaqueValueExpr *OpaqueValue)
5094+
OpaqueValueExpr *CommonExprOpaqueValue)
50945095
: Expr(SC, Resume->getType(), Resume->getValueKind(),
50955096
Resume->getObjectKind()),
5096-
KeywordLoc(KeywordLoc), OpaqueValue(OpaqueValue) {
5097+
KeywordLoc(KeywordLoc), CommonExprOpaqueValue(CommonExprOpaqueValue) {
50975098
SubExprs[SubExpr::Operand] = Operand;
50985099
SubExprs[SubExpr::Common] = Common;
50995100
SubExprs[SubExpr::Ready] = Ready;
@@ -5128,7 +5129,16 @@ class CoroutineSuspendExpr : public Expr {
51285129
}
51295130

51305131
/// getOpaqueValue - Return the opaque value placeholder.
5131-
OpaqueValueExpr *getOpaqueValue() const { return OpaqueValue; }
5132+
OpaqueValueExpr *getCommonExprOpaqueValue() const {
5133+
return CommonExprOpaqueValue;
5134+
}
5135+
5136+
OpaqueValueExpr *getInplaceCallOpaqueValue() const {
5137+
return InplaceCallOpaqueValue;
5138+
}
5139+
void setInplaceCallOpaqueValue(OpaqueValueExpr *E) {
5140+
InplaceCallOpaqueValue = E;
5141+
}
51325142

51335143
Expr *getReadyExpr() const {
51345144
return static_cast<Expr*>(SubExprs[SubExpr::Ready]);
@@ -5194,9 +5204,9 @@ class CoawaitExpr : public CoroutineSuspendExpr {
51945204
public:
51955205
CoawaitExpr(SourceLocation CoawaitLoc, Expr *Operand, Expr *Common,
51965206
Expr *Ready, Expr *Suspend, Expr *Resume,
5197-
OpaqueValueExpr *OpaqueValue, bool IsImplicit = false)
5207+
OpaqueValueExpr *CommonExprOpaqueValue, bool IsImplicit = false)
51985208
: CoroutineSuspendExpr(CoawaitExprClass, CoawaitLoc, Operand, Common,
5199-
Ready, Suspend, Resume, OpaqueValue) {
5209+
Ready, Suspend, Resume, CommonExprOpaqueValue) {
52005210
CoawaitBits.IsImplicit = IsImplicit;
52015211
}
52025212

@@ -5275,9 +5285,9 @@ class CoyieldExpr : public CoroutineSuspendExpr {
52755285
public:
52765286
CoyieldExpr(SourceLocation CoyieldLoc, Expr *Operand, Expr *Common,
52775287
Expr *Ready, Expr *Suspend, Expr *Resume,
5278-
OpaqueValueExpr *OpaqueValue)
5288+
OpaqueValueExpr *CommonExprOpaqueValue)
52795289
: CoroutineSuspendExpr(CoyieldExprClass, CoyieldLoc, Operand, Common,
5280-
Ready, Suspend, Resume, OpaqueValue) {}
5290+
Ready, Suspend, Resume, CommonExprOpaqueValue) {}
52815291
CoyieldExpr(SourceLocation CoyieldLoc, QualType Ty, Expr *Operand,
52825292
Expr *Common)
52835293
: CoroutineSuspendExpr(CoyieldExprClass, CoyieldLoc, Ty, Operand,

‎clang/include/clang/Basic/Attr.td

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1217,6 +1217,14 @@ def CoroDisableLifetimeBound : InheritableAttr {
12171217
let SimpleHandler = 1;
12181218
}
12191219

1220+
def CoroAwaitElidable : InheritableAttr {
1221+
let Spellings = [Clang<"coro_await_elidable">];
1222+
let Subjects = SubjectList<[CXXRecord]>;
1223+
let LangOpts = [CPlusPlus];
1224+
let Documentation = [CoroAwaitElidableDoc];
1225+
let SimpleHandler = 1;
1226+
}
1227+
12201228
// OSObject-based attributes.
12211229
def OSConsumed : InheritableParamAttr {
12221230
let Spellings = [Clang<"os_consumed">];

‎clang/include/clang/Basic/AttrDocs.td

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8108,6 +8108,24 @@ but do not pass them to the underlying coroutine or pass them by value.
81088108
}];
81098109
}
81108110

8111+
def CoroAwaitElidableDoc : Documentation {
8112+
let Category = DocCatDecl;
8113+
let Content = [{
8114+
The ``[[clang::coro_await_elidable]]`` is a class attribute which can be applied
8115+
to a coroutine return type.
8116+
8117+
When a coroutine function that returns such a type calls another coroutine function,
8118+
the compiler performs heap allocation elision when the following conditions are all met:
8119+
- callee coroutine function returns a type that is annotated with ``[[clang::coro_await_elidable]]``.
8120+
- In caller coroutine, the return value of the callee is a prvalue or an xvalue, and
8121+
- The temporary expression containing the callee coroutine object is immediately co_awaited.
8122+
8123+
The behavior is undefined if any of the following condition was met:
8124+
- the caller coroutine is destroyed earlier than the callee coroutine.
8125+
8126+
}];
8127+
}
8128+
81118129
def CountedByDocs : Documentation {
81128130
let Category = DocCatField;
81138131
let Content = [{
@@ -8267,4 +8285,3 @@ Declares that a function potentially allocates heap memory, and prevents any pot
82678285
of ``nonallocating`` by the compiler.
82688286
}];
82698287
}
8270-

‎clang/lib/CodeGen/CGBlocks.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1163,7 +1163,8 @@ llvm::Type *CodeGenModule::getGenericBlockLiteralType() {
11631163
}
11641164

11651165
RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
1166-
ReturnValueSlot ReturnValue) {
1166+
ReturnValueSlot ReturnValue,
1167+
llvm::CallBase **CallOrInvoke) {
11671168
const auto *BPT = E->getCallee()->getType()->castAs<BlockPointerType>();
11681169
llvm::Value *BlockPtr = EmitScalarExpr(E->getCallee());
11691170
llvm::Type *GenBlockTy = CGM.getGenericBlockLiteralType();
@@ -1220,7 +1221,7 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
12201221
CGCallee Callee(CGCalleeInfo(), Func);
12211222

12221223
// And call the block.
1223-
return EmitCall(FnInfo, Callee, ReturnValue, Args);
1224+
return EmitCall(FnInfo, Callee, ReturnValue, Args, CallOrInvoke);
12241225
}
12251226

12261227
Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable) {

‎clang/lib/CodeGen/CGCUDARuntime.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ CGCUDARuntime::~CGCUDARuntime() {}
2525

2626
RValue CGCUDARuntime::EmitCUDAKernelCallExpr(CodeGenFunction &CGF,
2727
const CUDAKernelCallExpr *E,
28-
ReturnValueSlot ReturnValue) {
28+
ReturnValueSlot ReturnValue,
29+
llvm::CallBase **CallOrInvoke) {
2930
llvm::BasicBlock *ConfigOKBlock = CGF.createBasicBlock("kcall.configok");
3031
llvm::BasicBlock *ContBlock = CGF.createBasicBlock("kcall.end");
3132

@@ -35,7 +36,7 @@ RValue CGCUDARuntime::EmitCUDAKernelCallExpr(CodeGenFunction &CGF,
3536

3637
eval.begin(CGF);
3738
CGF.EmitBlock(ConfigOKBlock);
38-
CGF.EmitSimpleCallExpr(E, ReturnValue);
39+
CGF.EmitSimpleCallExpr(E, ReturnValue, CallOrInvoke);
3940
CGF.EmitBranch(ContBlock);
4041

4142
CGF.EmitBlock(ContBlock);

‎clang/lib/CodeGen/CGCUDARuntime.h

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "llvm/IR/GlobalValue.h"
2222

2323
namespace llvm {
24+
class CallBase;
2425
class Function;
2526
class GlobalVariable;
2627
}
@@ -82,9 +83,10 @@ class CGCUDARuntime {
8283
CGCUDARuntime(CodeGenModule &CGM) : CGM(CGM) {}
8384
virtual ~CGCUDARuntime();
8485

85-
virtual RValue EmitCUDAKernelCallExpr(CodeGenFunction &CGF,
86-
const CUDAKernelCallExpr *E,
87-
ReturnValueSlot ReturnValue);
86+
virtual RValue
87+
EmitCUDAKernelCallExpr(CodeGenFunction &CGF, const CUDAKernelCallExpr *E,
88+
ReturnValueSlot ReturnValue,
89+
llvm::CallBase **CallOrInvoke = nullptr);
8890

8991
/// Emits a kernel launch stub.
9092
virtual void emitDeviceStub(CodeGenFunction &CGF, FunctionArgList &Args) = 0;

‎clang/lib/CodeGen/CGCXXABI.h

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -485,11 +485,11 @@ class CGCXXABI {
485485
llvm::PointerUnion<const CXXDeleteExpr *, const CXXMemberCallExpr *>;
486486

487487
/// Emit the ABI-specific virtual destructor call.
488-
virtual llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF,
489-
const CXXDestructorDecl *Dtor,
490-
CXXDtorType DtorType,
491-
Address This,
492-
DeleteOrMemberCallExpr E) = 0;
488+
virtual llvm::Value *
489+
EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor,
490+
CXXDtorType DtorType, Address This,
491+
DeleteOrMemberCallExpr E,
492+
llvm::CallBase **CallOrInvoke) = 0;
493493

494494
virtual void adjustCallArgsForDestructorThunk(CodeGenFunction &CGF,
495495
GlobalDecl GD,

‎clang/lib/CodeGen/CGClass.cpp

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2191,15 +2191,11 @@ static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
21912191
return true;
21922192
}
21932193

2194-
void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2195-
CXXCtorType Type,
2196-
bool ForVirtualBase,
2197-
bool Delegating,
2198-
Address This,
2199-
CallArgList &Args,
2200-
AggValueSlot::Overlap_t Overlap,
2201-
SourceLocation Loc,
2202-
bool NewPointerIsChecked) {
2194+
void CodeGenFunction::EmitCXXConstructorCall(
2195+
const CXXConstructorDecl *D, CXXCtorType Type, bool ForVirtualBase,
2196+
bool Delegating, Address This, CallArgList &Args,
2197+
AggValueSlot::Overlap_t Overlap, SourceLocation Loc,
2198+
bool NewPointerIsChecked, llvm::CallBase **CallOrInvoke) {
22032199
const CXXRecordDecl *ClassDecl = D->getParent();
22042200

22052201
if (!NewPointerIsChecked)
@@ -2247,7 +2243,7 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
22472243
const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
22482244
Args, D, Type, ExtraArgs.Prefix, ExtraArgs.Suffix, PassPrototypeArgs);
22492245
CGCallee Callee = CGCallee::forDirect(CalleePtr, GlobalDecl(D, Type));
2250-
EmitCall(Info, Callee, ReturnValueSlot(), Args, nullptr, false, Loc);
2246+
EmitCall(Info, Callee, ReturnValueSlot(), Args, CallOrInvoke, false, Loc);
22512247

22522248
// Generate vtable assumptions if we're constructing a complete object
22532249
// with a vtable. We don't do this for base subobjects for two reasons:

‎clang/lib/CodeGen/CGCoroutine.cpp

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,11 @@
1212

1313
#include "CGCleanup.h"
1414
#include "CodeGenFunction.h"
15-
#include "llvm/ADT/ScopeExit.h"
15+
#include "clang/AST/ExprCXX.h"
1616
#include "clang/AST/StmtCXX.h"
1717
#include "clang/AST/StmtVisitor.h"
18+
#include "llvm/ADT/ScopeExit.h"
19+
#include "llvm/IR/Intrinsics.h"
1820

1921
using namespace clang;
2022
using namespace CodeGen;
@@ -223,12 +225,22 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co
223225
CoroutineSuspendExpr const &S,
224226
AwaitKind Kind, AggValueSlot aggSlot,
225227
bool ignoreResult, bool forLValue) {
226-
auto *E = S.getCommonExpr();
228+
auto &Builder = CGF.Builder;
227229

228-
auto CommonBinder =
229-
CodeGenFunction::OpaqueValueMappingData::bind(CGF, S.getOpaqueValue(), E);
230-
auto UnbindCommonOnExit =
231-
llvm::make_scope_exit([&] { CommonBinder.unbind(CGF); });
230+
// If S.getInplaceCallOpaqueValue() is null, we don't have a nested opaque
231+
// value for common expression.
232+
std::optional<CodeGenFunction::OpaqueValueMapping> OperandMapping;
233+
if (auto *CallOV = S.getInplaceCallOpaqueValue()) {
234+
auto *CE = cast<CallExpr>(CallOV->getSourceExpr());
235+
llvm::CallBase *CallOrInvoke = nullptr;
236+
LValue CallResult = CGF.EmitCallExprLValue(CE, &CallOrInvoke);
237+
if (CallOrInvoke)
238+
CallOrInvoke->addFnAttr(llvm::Attribute::CoroMustElide);
239+
240+
OperandMapping.emplace(CGF, CallOV, CallResult);
241+
}
242+
CodeGenFunction::OpaqueValueMapping BindCommon(CGF,
243+
S.getCommonExprOpaqueValue());
232244

233245
auto Prefix = buildSuspendPrefixStr(Coro, Kind);
234246
BasicBlock *ReadyBlock = CGF.createBasicBlock(Prefix + Twine(".ready"));
@@ -241,7 +253,6 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co
241253
// Otherwise, emit suspend logic.
242254
CGF.EmitBlock(SuspendBlock);
243255

244-
auto &Builder = CGF.Builder;
245256
llvm::Function *CoroSave = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_save);
246257
auto *NullPtr = llvm::ConstantPointerNull::get(CGF.CGM.Int8PtrTy);
247258
auto *SaveCall = Builder.CreateCall(CoroSave, {NullPtr});
@@ -256,7 +267,8 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co
256267

257268
SmallVector<llvm::Value *, 3> SuspendIntrinsicCallArgs;
258269
SuspendIntrinsicCallArgs.push_back(
259-
CGF.getOrCreateOpaqueLValueMapping(S.getOpaqueValue()).getPointer(CGF));
270+
CGF.getOrCreateOpaqueLValueMapping(S.getCommonExprOpaqueValue())
271+
.getPointer(CGF));
260272

261273
SuspendIntrinsicCallArgs.push_back(CGF.CurCoro.Data->CoroBegin);
262274
SuspendIntrinsicCallArgs.push_back(SuspendWrapper);
@@ -455,7 +467,7 @@ CodeGenFunction::generateAwaitSuspendWrapper(Twine const &CoroName,
455467
Builder.CreateLoad(GetAddrOfLocalVar(&FrameDecl));
456468

457469
auto AwaiterBinder = CodeGenFunction::OpaqueValueMappingData::bind(
458-
*this, S.getOpaqueValue(), AwaiterLValue);
470+
*this, S.getCommonExprOpaqueValue(), AwaiterLValue);
459471

460472
auto *SuspendRet = EmitScalarExpr(S.getSuspendExpr());
461473

‎clang/lib/CodeGen/CGExpr.cpp

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5437,24 +5437,25 @@ RValue CodeGenFunction::EmitRValueForField(LValue LV,
54375437
//===--------------------------------------------------------------------===//
54385438

54395439
RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
5440-
ReturnValueSlot ReturnValue) {
5440+
ReturnValueSlot ReturnValue,
5441+
llvm::CallBase **CallOrInvoke) {
54415442
// Builtins never have block type.
54425443
if (E->getCallee()->getType()->isBlockPointerType())
5443-
return EmitBlockCallExpr(E, ReturnValue);
5444+
return EmitBlockCallExpr(E, ReturnValue, CallOrInvoke);
54445445

54455446
if (const auto *CE = dyn_cast<CXXMemberCallExpr>(E))
5446-
return EmitCXXMemberCallExpr(CE, ReturnValue);
5447+
return EmitCXXMemberCallExpr(CE, ReturnValue, CallOrInvoke);
54475448

54485449
if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(E))
5449-
return EmitCUDAKernelCallExpr(CE, ReturnValue);
5450+
return EmitCUDAKernelCallExpr(CE, ReturnValue, CallOrInvoke);
54505451

54515452
// A CXXOperatorCallExpr is created even for explicit object methods, but
54525453
// these should be treated like static function call.
54535454
if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(E))
54545455
if (const auto *MD =
54555456
dyn_cast_if_present<CXXMethodDecl>(CE->getCalleeDecl());
54565457
MD && MD->isImplicitObjectMemberFunction())
5457-
return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue);
5458+
return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue, CallOrInvoke);
54585459

54595460
CGCallee callee = EmitCallee(E->getCallee());
54605461

@@ -5467,14 +5468,17 @@ RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
54675468
return EmitCXXPseudoDestructorExpr(callee.getPseudoDestructorExpr());
54685469
}
54695470

5470-
return EmitCall(E->getCallee()->getType(), callee, E, ReturnValue);
5471+
return EmitCall(E->getCallee()->getType(), callee, E, ReturnValue,
5472+
/*Chain=*/nullptr, CallOrInvoke);
54715473
}
54725474

54735475
/// Emit a CallExpr without considering whether it might be a subclass.
54745476
RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E,
5475-
ReturnValueSlot ReturnValue) {
5477+
ReturnValueSlot ReturnValue,
5478+
llvm::CallBase **CallOrInvoke) {
54765479
CGCallee Callee = EmitCallee(E->getCallee());
5477-
return EmitCall(E->getCallee()->getType(), Callee, E, ReturnValue);
5480+
return EmitCall(E->getCallee()->getType(), Callee, E, ReturnValue,
5481+
/*Chain=*/nullptr, CallOrInvoke);
54785482
}
54795483

54805484
// Detect the unusual situation where an inline version is shadowed by a
@@ -5678,8 +5682,9 @@ LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
56785682
llvm_unreachable("bad evaluation kind");
56795683
}
56805684

5681-
LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
5682-
RValue RV = EmitCallExpr(E);
5685+
LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E,
5686+
llvm::CallBase **CallOrInvoke) {
5687+
RValue RV = EmitCallExpr(E, ReturnValueSlot(), CallOrInvoke);
56835688

56845689
if (!RV.isScalar())
56855690
return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
@@ -5802,9 +5807,11 @@ LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
58025807
AlignmentSource::Decl);
58035808
}
58045809

5805-
RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee,
5806-
const CallExpr *E, ReturnValueSlot ReturnValue,
5807-
llvm::Value *Chain) {
5810+
RValue CodeGenFunction::EmitCall(QualType CalleeType,
5811+
const CGCallee &OrigCallee, const CallExpr *E,
5812+
ReturnValueSlot ReturnValue,
5813+
llvm::Value *Chain,
5814+
llvm::CallBase **CallOrInvoke) {
58085815
// Get the actual function type. The callee type will always be a pointer to
58095816
// function type or a block pointer type.
58105817
assert(CalleeType->isFunctionPointerType() &&
@@ -6015,8 +6022,8 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee
60156022
Address(Handle, Handle->getType(), CGM.getPointerAlign()));
60166023
Callee.setFunctionPointer(Stub);
60176024
}
6018-
llvm::CallBase *CallOrInvoke = nullptr;
6019-
RValue Call = EmitCall(FnInfo, Callee, ReturnValue, Args, &CallOrInvoke,
6025+
llvm::CallBase *LocalCallOrInvoke = nullptr;
6026+
RValue Call = EmitCall(FnInfo, Callee, ReturnValue, Args, &LocalCallOrInvoke,
60206027
E == MustTailCall, E->getExprLoc());
60216028

60226029
// Generate function declaration DISuprogram in order to be used
@@ -6025,11 +6032,13 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee
60256032
if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
60266033
FunctionArgList Args;
60276034
QualType ResTy = BuildFunctionArgList(CalleeDecl, Args);
6028-
DI->EmitFuncDeclForCallSite(CallOrInvoke,
6035+
DI->EmitFuncDeclForCallSite(LocalCallOrInvoke,
60296036
DI->getFunctionType(CalleeDecl, ResTy, Args),
60306037
CalleeDecl);
60316038
}
60326039
}
6040+
if (CallOrInvoke)
6041+
*CallOrInvoke = LocalCallOrInvoke;
60336042

60346043
return Call;
60356044
}

0 commit comments

Comments
 (0)