Skip to content

Commit 038819a

Browse files
JDPailleuxJDPailleux
committed
[flang] Implement !DIR$ NOINLINE and FORCEINLINE directives
1 parent 92c93f5 commit 038819a

File tree

15 files changed

+286
-12
lines changed

15 files changed

+286
-12
lines changed

‎flang/docs/Directives.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@ A list of non-standard directives supported by Flang
5353
* `!dir$ novector` disabling vectorization on the following loop.
5454
* `!dir$ nounroll` disabling unrolling on the following loop.
5555
* `!dir$ nounroll_and_jam` disabling unrolling and jamming on the following loop.
56+
* `!dir$ inline` tells the compiler to attempt to inline routines if
57+
this directive is specified before a call statement or for all call function statements
58+
within a DO LOOP. This directive can be improved later to support other place(s) for
59+
inlining function calls.
60+
* `!dir$ noinline` works in the same way as the `inline` directive, but prevents
61+
any attempt of inlining by the compiler on a function call statement.
5662

5763
# Directive Details
5864

‎flang/include/flang/Evaluate/call.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,11 @@ class ProcedureRef {
254254
bool IsElemental() const { return proc_.IsElemental(); }
255255
bool hasAlternateReturns() const { return hasAlternateReturns_; }
256256

257+
bool hasNoInline() const { return noInline_; }
258+
void set_noInline(bool ni) { noInline_ = ni; }
259+
bool hasAlwaysInline() const { return alwaysInline_; }
260+
void set_alwaysInline(bool ai) { alwaysInline_ = ai; }
261+
257262
Expr<SomeType> *UnwrapArgExpr(int n) {
258263
if (static_cast<std::size_t>(n) < arguments_.size() && arguments_[n]) {
259264
return arguments_[n]->UnwrapExpr();
@@ -277,6 +282,8 @@ class ProcedureRef {
277282
ActualArguments arguments_;
278283
Chevrons chevrons_;
279284
bool hasAlternateReturns_;
285+
bool noInline_{false};
286+
bool alwaysInline_{false};
280287
};
281288

282289
template <typename A> class FunctionRef : public ProcedureRef {

‎flang/include/flang/Optimizer/Dialect/FIROps.td

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2490,6 +2490,8 @@ def fir_CallOp : fir_Op<"call",
24902490
OptionalAttr<DictArrayAttr>:$arg_attrs,
24912491
OptionalAttr<DictArrayAttr>:$res_attrs,
24922492
OptionalAttr<fir_FortranProcedureFlagsAttr>:$procedure_attrs,
2493+
OptionalAttr<UnitAttr>:$no_inline,
2494+
OptionalAttr<UnitAttr>:$always_inline,
24932495
DefaultValuedAttr<Arith_FastMathAttr,
24942496
"::mlir::arith::FastMathFlags::none">:$fastmath
24952497
);

‎flang/include/flang/Parser/dump-parse-tree.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,8 +204,10 @@ class ParseTreeDumper {
204204
NODE(parser, CompilerDirective)
205205
NODE(CompilerDirective, AssumeAligned)
206206
NODE(CompilerDirective, IgnoreTKR)
207+
NODE(CompilerDirective, ForceInline)
207208
NODE(CompilerDirective, LoopCount)
208209
NODE(CompilerDirective, NameValue)
210+
NODE(CompilerDirective, NoInline)
209211
NODE(CompilerDirective, Unrecognized)
210212
NODE(CompilerDirective, VectorAlways)
211213
NODE(CompilerDirective, Unroll)

‎flang/include/flang/Parser/parse-tree.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3354,6 +3354,8 @@ struct StmtFunctionStmt {
33543354
// !DIR$ NOVECTOR
33553355
// !DIR$ NOUNROLL
33563356
// !DIR$ NOUNROLL_AND_JAM
3357+
// !DIR$ FORCEINLINE
3358+
// !DIR$ NOINLINE
33573359
// !DIR$ <anything else>
33583360
struct CompilerDirective {
33593361
UNION_CLASS_BOILERPLATE(CompilerDirective);
@@ -3382,11 +3384,13 @@ struct CompilerDirective {
33823384
EMPTY_CLASS(NoVector);
33833385
EMPTY_CLASS(NoUnroll);
33843386
EMPTY_CLASS(NoUnrollAndJam);
3387+
EMPTY_CLASS(ForceInline);
3388+
EMPTY_CLASS(NoInline);
33853389
EMPTY_CLASS(Unrecognized);
33863390
CharBlock source;
33873391
std::variant<std::list<IgnoreTKR>, LoopCount, std::list<AssumeAligned>,
33883392
VectorAlways, std::list<NameValue>, Unroll, UnrollAndJam, Unrecognized,
3389-
NoVector, NoUnroll, NoUnrollAndJam>
3393+
NoVector, NoUnroll, NoUnrollAndJam, ForceInline, NoInline>
33903394
u;
33913395
};
33923396

‎flang/lib/Lower/Bridge.cpp

Lines changed: 113 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1828,6 +1828,23 @@ class FirConverter : public Fortran::lower::AbstractConverter {
18281828
setCurrentPosition(stmt.source);
18291829
assert(stmt.typedCall && "Call was not analyzed");
18301830
mlir::Value res{};
1831+
1832+
// Set 'no_inline' or 'always_inline' to true on the ProcedureRef.
1833+
// The NoInline and AlwaysInline attribute will be set in genProcedureRef
1834+
// later.
1835+
for (const auto *dir : eval.dirs) {
1836+
Fortran::common::visit(
1837+
Fortran::common::visitors{
1838+
[&](const Fortran::parser::CompilerDirective::ForceInline &) {
1839+
stmt.typedCall->set_alwaysInline(true);
1840+
},
1841+
[&](const Fortran::parser::CompilerDirective::NoInline &) {
1842+
stmt.typedCall->set_noInline(true);
1843+
},
1844+
[&](const auto &) {}},
1845+
dir->u);
1846+
}
1847+
18311848
if (lowerToHighLevelFIR()) {
18321849
std::optional<mlir::Type> resultType;
18331850
if (stmt.typedCall->hasAlternateReturns())
@@ -2053,6 +2070,47 @@ class FirConverter : public Fortran::lower::AbstractConverter {
20532070
// so no clean-up needs to be generated for these entities.
20542071
}
20552072

2073+
void attachInlineAttributes(
2074+
mlir::Operation &op,
2075+
const llvm::ArrayRef<const Fortran::parser::CompilerDirective *> &dirs) {
2076+
if (dirs.empty())
2077+
return;
2078+
2079+
for (mlir::Value operand : op.getOperands()) {
2080+
if (operand.getDefiningOp())
2081+
attachInlineAttributes(*operand.getDefiningOp(), dirs);
2082+
}
2083+
2084+
if (fir::CallOp callOp = mlir::dyn_cast<fir::CallOp>(op)) {
2085+
for (const auto *dir : dirs) {
2086+
Fortran::common::visit(
2087+
Fortran::common::visitors{
2088+
[&](const Fortran::parser::CompilerDirective::NoInline &) {
2089+
callOp.setNoInlineAttr(builder->getUnitAttr());
2090+
},
2091+
[&](const Fortran::parser::CompilerDirective::ForceInline &) {
2092+
callOp.setAlwaysInlineAttr(builder->getUnitAttr());
2093+
},
2094+
[&](const auto &) {}},
2095+
dir->u);
2096+
}
2097+
}
2098+
}
2099+
2100+
void attachAttributesToDoLoopOperations(
2101+
fir::DoLoopOp &doLoop,
2102+
llvm::SmallVectorImpl<const Fortran::parser::CompilerDirective *> &dirs) {
2103+
if (!doLoop.getOperation() || dirs.empty())
2104+
return;
2105+
2106+
for (mlir::Block &block : doLoop.getRegion()) {
2107+
for (mlir::Operation &op : block.getOperations()) {
2108+
if (!dirs.empty())
2109+
attachInlineAttributes(op, dirs);
2110+
}
2111+
}
2112+
}
2113+
20562114
/// Generate FIR for a DO construct. There are six variants:
20572115
/// - unstructured infinite and while loops
20582116
/// - structured and unstructured increment loops
@@ -2162,6 +2220,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
21622220

21632221
// This call may generate a branch in some contexts.
21642222
genFIR(endDoEval, unstructuredContext);
2223+
2224+
// Add attribute(s) on operations in fir::DoLoopOp if necessary
2225+
for (IncrementLoopInfo &info : incrementLoopNestInfo)
2226+
attachAttributesToDoLoopOperations(info.doLoop, doStmtEval.dirs);
21652227
}
21662228

21672229
/// Generate FIR to evaluate loop control values (lower, upper and step).
@@ -2935,6 +2997,26 @@ class FirConverter : public Fortran::lower::AbstractConverter {
29352997
e->dirs.push_back(&dir);
29362998
}
29372999

3000+
void
3001+
attachInliningDirectiveToStmt(const Fortran::parser::CompilerDirective &dir,
3002+
Fortran::lower::pft::Evaluation *e) {
3003+
while (e->isDirective())
3004+
e = e->lexicalSuccessor;
3005+
3006+
// If the successor is a statement or a do loop, the compiler
3007+
// will perform inlining.
3008+
if (e->isA<Fortran::parser::CallStmt>() ||
3009+
e->isA<Fortran::parser::NonLabelDoStmt>() ||
3010+
e->isA<Fortran::parser::AssignmentStmt>()) {
3011+
e->dirs.push_back(&dir);
3012+
} else {
3013+
mlir::Location loc = toLocation();
3014+
mlir::emitWarning(loc,
3015+
"Inlining directive not in front of loops, function"
3016+
"call or assignment.\n");
3017+
}
3018+
}
3019+
29383020
void genFIR(const Fortran::parser::CompilerDirective &dir) {
29393021
Fortran::lower::pft::Evaluation &eval = getEval();
29403022

@@ -2958,6 +3040,12 @@ class FirConverter : public Fortran::lower::AbstractConverter {
29583040
[&](const Fortran::parser::CompilerDirective::NoUnrollAndJam &) {
29593041
attachDirectiveToLoop(dir, &eval);
29603042
},
3043+
[&](const Fortran::parser::CompilerDirective::ForceInline &) {
3044+
attachInliningDirectiveToStmt(dir, &eval);
3045+
},
3046+
[&](const Fortran::parser::CompilerDirective::NoInline &) {
3047+
attachInliningDirectiveToStmt(dir, &eval);
3048+
},
29613049
[&](const auto &) {}},
29623050
dir.u);
29633051
}
@@ -4763,7 +4851,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
47634851

47644852
void genDataAssignment(
47654853
const Fortran::evaluate::Assignment &assign,
4766-
const Fortran::evaluate::ProcedureRef *userDefinedAssignment) {
4854+
const Fortran::evaluate::ProcedureRef *userDefinedAssignment,
4855+
const llvm::ArrayRef<const Fortran::parser::CompilerDirective *> &dirs =
4856+
{}) {
47674857
mlir::Location loc = getCurrentLocation();
47684858
fir::FirOpBuilder &builder = getFirOpBuilder();
47694859

@@ -4836,12 +4926,22 @@ class FirConverter : public Fortran::lower::AbstractConverter {
48364926
Fortran::lower::StatementContext localStmtCtx;
48374927
hlfir::Entity rhs = evaluateRhs(localStmtCtx);
48384928
hlfir::Entity lhs = evaluateLhs(localStmtCtx);
4839-
if (isCUDATransfer && !hasCUDAImplicitTransfer)
4929+
if (isCUDATransfer && !hasCUDAImplicitTransfer) {
48404930
genCUDADataTransfer(builder, loc, assign, lhs, rhs);
4841-
else
4931+
} else {
4932+
// If RHS or LHS have a CallOp in their expression, this operation will
4933+
// have the 'no_inline' or 'always_inline' attribute if there is a
4934+
// directive just before the assignement.
4935+
if (!dirs.empty()) {
4936+
if (rhs.getDefiningOp())
4937+
attachInlineAttributes(*rhs.getDefiningOp(), dirs);
4938+
if (lhs.getDefiningOp())
4939+
attachInlineAttributes(*lhs.getDefiningOp(), dirs);
4940+
}
48424941
builder.create<hlfir::AssignOp>(loc, rhs, lhs,
48434942
isWholeAllocatableAssignment,
48444943
keepLhsLengthInAllocatableAssignment);
4944+
}
48454945
if (hasCUDAImplicitTransfer && !isInDeviceContext) {
48464946
localSymbols.popScope();
48474947
for (mlir::Value temp : implicitTemps)
@@ -4909,16 +5009,21 @@ class FirConverter : public Fortran::lower::AbstractConverter {
49095009
}
49105010

49115011
/// Shared for both assignments and pointer assignments.
4912-
void genAssignment(const Fortran::evaluate::Assignment &assign) {
5012+
void
5013+
genAssignment(const Fortran::evaluate::Assignment &assign,
5014+
const llvm::ArrayRef<const Fortran::parser::CompilerDirective *>
5015+
&dirs = {}) {
49135016
mlir::Location loc = toLocation();
49145017
if (lowerToHighLevelFIR()) {
49155018
Fortran::common::visit(
49165019
Fortran::common::visitors{
49175020
[&](const Fortran::evaluate::Assignment::Intrinsic &) {
4918-
genDataAssignment(assign, /*userDefinedAssignment=*/nullptr);
5021+
genDataAssignment(assign, /*userDefinedAssignment=*/nullptr,
5022+
dirs);
49195023
},
49205024
[&](const Fortran::evaluate::ProcedureRef &procRef) {
4921-
genDataAssignment(assign, /*userDefinedAssignment=*/&procRef);
5025+
genDataAssignment(assign, /*userDefinedAssignment=*/&procRef,
5026+
dirs);
49225027
},
49235028
[&](const Fortran::evaluate::Assignment::BoundsSpec &lbExprs) {
49245029
if (isInsideHlfirForallOrWhere())
@@ -5323,7 +5428,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
53235428
}
53245429

53255430
void genFIR(const Fortran::parser::AssignmentStmt &stmt) {
5326-
genAssignment(*stmt.typedAssignment->v);
5431+
Fortran::lower::pft::Evaluation &eval = getEval();
5432+
genAssignment(*stmt.typedAssignment->v, eval.dirs);
53275433
}
53285434

53295435
void genFIR(const Fortran::parser::SyncAllStmt &stmt) {

‎flang/lib/Lower/ConvertCall.cpp

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -647,9 +647,15 @@ Fortran::lower::genCallOpAndResult(
647647
callResult = dispatch.getResult(0);
648648
} else {
649649
// Standard procedure call with fir.call.
650+
mlir::UnitAttr noinlineAttr, alwaysinlineAttr;
651+
if (caller.getCallDescription().hasNoInline())
652+
noinlineAttr = builder.getUnitAttr();
653+
else if (caller.getCallDescription().hasAlwaysInline())
654+
alwaysinlineAttr = builder.getUnitAttr();
650655
auto call = builder.create<fir::CallOp>(
651656
loc, funcType.getResults(), funcSymbolAttr, operands,
652-
/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, procAttrs);
657+
/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, procAttrs, noinlineAttr,
658+
alwaysinlineAttr);
653659

654660
callNumResults = call.getNumResults();
655661
if (callNumResults != 0)

‎flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,9 @@ struct DispatchOpConv : public OpConversionPattern<fir::DispatchOp> {
207207
args.append(dispatch.getArgs().begin(), dispatch.getArgs().end());
208208
rewriter.replaceOpWithNewOp<fir::CallOp>(
209209
dispatch, resTypes, nullptr, args, dispatch.getArgAttrsAttr(),
210-
dispatch.getResAttrsAttr(), dispatch.getProcedureAttrsAttr());
210+
dispatch.getResAttrsAttr(), dispatch.getProcedureAttrsAttr(),
211+
/*no_inline*/ mlir::UnitAttr{},
212+
/*alwais_inline*/ mlir::UnitAttr{});
211213
return mlir::success();
212214
}
213215

‎flang/lib/Parser/Fortran-parsers.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1314,6 +1314,10 @@ constexpr auto novector{"NOVECTOR" >> construct<CompilerDirective::NoVector>()};
13141314
constexpr auto nounroll{"NOUNROLL" >> construct<CompilerDirective::NoUnroll>()};
13151315
constexpr auto nounrollAndJam{
13161316
"NOUNROLL_AND_JAM" >> construct<CompilerDirective::NoUnrollAndJam>()};
1317+
constexpr auto forceinlineDir{
1318+
"FORCEINLINE" >> construct<CompilerDirective::ForceInline>()};
1319+
constexpr auto noinlineDir{
1320+
"NOINLINE" >> construct<CompilerDirective::NoInline>()};
13171321
TYPE_PARSER(beginDirective >> "DIR$ "_tok >>
13181322
sourced((construct<CompilerDirective>(ignore_tkr) ||
13191323
construct<CompilerDirective>(loopCount) ||
@@ -1324,6 +1328,8 @@ TYPE_PARSER(beginDirective >> "DIR$ "_tok >>
13241328
construct<CompilerDirective>(novector) ||
13251329
construct<CompilerDirective>(nounrollAndJam) ||
13261330
construct<CompilerDirective>(nounroll) ||
1331+
construct<CompilerDirective>(noinlineDir) ||
1332+
construct<CompilerDirective>(forceinlineDir) ||
13271333
construct<CompilerDirective>(
13281334
many(construct<CompilerDirective::NameValue>(
13291335
name, maybe(("="_tok || ":"_tok) >> digitString64))))) /

‎flang/lib/Parser/unparse.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1864,6 +1864,12 @@ class UnparseVisitor {
18641864
[&](const CompilerDirective::NoUnrollAndJam &) {
18651865
Word("!DIR$ NOUNROLL_AND_JAM");
18661866
},
1867+
[&](const CompilerDirective::ForceInline &) {
1868+
Word("!DIR$ FORCEINLINE");
1869+
},
1870+
[&](const CompilerDirective::NoInline &) {
1871+
Word("!DIR$ NOINLINE");
1872+
},
18671873
[&](const CompilerDirective::Unrecognized &) {
18681874
Word("!DIR$ ");
18691875
Word(x.source.ToString());

‎flang/lib/Semantics/canonicalize-directives.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,10 @@ static bool IsExecutionDirective(const parser::CompilerDirective &dir) {
6060
std::holds_alternative<parser::CompilerDirective::UnrollAndJam>(dir.u) ||
6161
std::holds_alternative<parser::CompilerDirective::NoVector>(dir.u) ||
6262
std::holds_alternative<parser::CompilerDirective::NoUnroll>(dir.u) ||
63-
std::holds_alternative<parser::CompilerDirective::NoUnrollAndJam>(dir.u);
63+
std::holds_alternative<parser::CompilerDirective::NoUnrollAndJam>(
64+
dir.u) ||
65+
std::holds_alternative<parser::CompilerDirective::ForceInline>(dir.u) ||
66+
std::holds_alternative<parser::CompilerDirective::NoInline>(dir.u);
6467
}
6568

6669
void CanonicalizationOfDirectives::Post(parser::SpecificationPart &spec) {

‎flang/lib/Semantics/resolve-names.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9576,7 +9576,9 @@ void ResolveNamesVisitor::Post(const parser::CompilerDirective &x) {
95769576
std::holds_alternative<parser::CompilerDirective::UnrollAndJam>(x.u) ||
95779577
std::holds_alternative<parser::CompilerDirective::NoVector>(x.u) ||
95789578
std::holds_alternative<parser::CompilerDirective::NoUnroll>(x.u) ||
9579-
std::holds_alternative<parser::CompilerDirective::NoUnrollAndJam>(x.u)) {
9579+
std::holds_alternative<parser::CompilerDirective::NoUnrollAndJam>(x.u) ||
9580+
std::holds_alternative<parser::CompilerDirective::ForceInline>(x.u) ||
9581+
std::holds_alternative<parser::CompilerDirective::NoInline>(x.u)) {
95809582
return;
95819583
}
95829584
if (const auto *tkr{

0 commit comments

Comments
 (0)