[PassBuilder][CodeGen] Add callback style pass buider

llvm/include/llvm/CodeGen/TargetPassConfig.h

@@ -190,8 +190,7 @@ class TargetPassConfig : public ImmutablePass {
 
   /// Returns pass name in `-stop-before` or `-stop-after`
   /// NOTE: New pass manager migration only
-  static Expected<StartStopInfo>
-  getStartStopInfo(PassInstrumentationCallbacks &PIC);
+  static Expected<StartStopInfo> getStartStopInfo();
 
   void setDisableVerify(bool Disable) { setOpt(DisableVerify, Disable); }
 

llvm/include/llvm/Passes/CodeGenPassBuilder.h

@@ -534,7 +534,7 @@ template <typename Derived, typename TargetMachineT>
 Error CodeGenPassBuilder<Derived, TargetMachineT>::buildPipeline(
     ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
     CodeGenFileType FileType) const {
-  auto StartStopInfo = TargetPassConfig::getStartStopInfo(*PIC);
+  auto StartStopInfo = TargetPassConfig::getStartStopInfo();
   if (!StartStopInfo)
     return StartStopInfo.takeError();
   setStartStopPasses(*StartStopInfo);

llvm/include/llvm/Passes/MachinePassRegistry.def

@@ -206,6 +206,7 @@ DUMMY_MACHINE_MODULE_PASS("mir-strip-debug", StripDebugMachineModulePass)
 #ifndef DUMMY_MACHINE_FUNCTION_PASS
 #define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME)
 #endif
+DUMMY_MACHINE_FUNCTION_PASS("bb-path-cloning", BasicBlockPathCloningPass)
 DUMMY_MACHINE_FUNCTION_PASS("bbsections-prepare", BasicBlockSectionsPass)
 DUMMY_MACHINE_FUNCTION_PASS("bbsections-profile-reader", BasicBlockSectionsProfileReaderPass)
 DUMMY_MACHINE_FUNCTION_PASS("block-placement", MachineBlockPlacementPass)
@@ -222,6 +223,8 @@ DUMMY_MACHINE_FUNCTION_PASS("fixup-statepoint-caller-saved", FixupStatepointCall
 DUMMY_MACHINE_FUNCTION_PASS("fs-profile-loader", MIRProfileLoaderNewPass)
 DUMMY_MACHINE_FUNCTION_PASS("funclet-layout", FuncletLayoutPass)
 DUMMY_MACHINE_FUNCTION_PASS("gc-empty-basic-blocks", GCEmptyBasicBlocksPass)
+DUMMY_MACHINE_FUNCTION_PASS("gc-machine-code-insersion",
+                            GCMachineCodeInsertionPass)
 DUMMY_MACHINE_FUNCTION_PASS("implicit-null-checks", ImplicitNullChecksPass)
 DUMMY_MACHINE_FUNCTION_PASS("init-undef-pass", InitUndefPass)
 DUMMY_MACHINE_FUNCTION_PASS("instruction-select", InstructionSelectPass)
@@ -266,6 +269,8 @@ DUMMY_MACHINE_FUNCTION_PASS("stack-frame-layout", StackFrameLayoutAnalysisPass)
 DUMMY_MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass)
 DUMMY_MACHINE_FUNCTION_PASS("stackmap-liveness", StackMapLivenessPass)
 DUMMY_MACHINE_FUNCTION_PASS("unpack-mi-bundles", UnpackMachineBundlesPass)
+DUMMY_MACHINE_FUNCTION_PASS("unreachable-mbb-elimination",
+                            UnreachableMachineBlockElimPass)
 DUMMY_MACHINE_FUNCTION_PASS("virtregrewriter", VirtRegRewriterPass)
 DUMMY_MACHINE_FUNCTION_PASS("xray-instrumentation", XRayInstrumentationPass)
 #undef DUMMY_MACHINE_FUNCTION_PASS

llvm/include/llvm/Passes/PassBuilder.h

@@ -15,14 +15,17 @@
 #ifndef LLVM_PASSES_PASSBUILDER_H
 #define LLVM_PASSES_PASSBUILDER_H
 
+#include "llvm/ADT/StringSet.h"
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/RegAllocCommon.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Passes/OptimizationLevel.h"
+#include "llvm/Support/CodeGen.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/PGOOptions.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/CGPassBuilderOption.h"
 #include "llvm/Transforms/IPO/Inliner.h"
 #include "llvm/Transforms/IPO/ModuleInliner.h"
 #include "llvm/Transforms/Scalar/LoopPassManager.h"
@@ -34,6 +37,7 @@ class StringRef;
 class AAManager;
 class TargetMachine;
 class ModuleSummaryIndex;
+class MCContext;
 template <typename T> class IntrusiveRefCntPtr;
 namespace vfs {
 class FileSystem;
@@ -106,6 +110,7 @@ class PassBuilder {
   TargetMachine *TM;
   PipelineTuningOptions PTO;
   std::optional<PGOOptions> PGOOpt;
+  CGPassBuilderOption CGPBO;
   PassInstrumentationCallbacks *PIC;
 
 public:
@@ -308,6 +313,24 @@ class PassBuilder {
   /// TargetMachine::registerDefaultAliasAnalyses().
   AAManager buildDefaultAAPipeline();
 
+  /// Build CodeGen pass pipeline.
+  ///
+  /// {{@
+  Expected<ModulePassManager>
+  buildDefaultCodeGenPipeline(raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
+                              CodeGenFileType FileType, MCContext &Ctx);
+  Error buildDefaultCodeGenPipeline(ModulePassManager &MPM,
+                                    raw_pwrite_stream &Out,
+                                    raw_pwrite_stream *DwoOut,
+                                    CodeGenFileType FileType, MCContext &Ctx);
+  Error addRegAllocPass(MachineFunctionPassManager &MFPM,
+                        StringRef Filter = "all");
+  // TODO: Add method to build MC emission pipeline.
+  template <typename... PassTs> void disablePass() {
+    (DisabledPasses.insert(PassTs::name()), ...);
+  }
+  /// @}}
+
   /// Parse a textual pass pipeline description into a \c
   /// ModulePassManager.
   ///
@@ -523,6 +546,133 @@ class PassBuilder {
     FullLinkTimeOptimizationLastEPCallbacks.push_back(C);
   }
 
+  /// Register target specific callbacks to extend codegen pipeline.
+  /// {{@
+
+  /// If target want its own pipeline, use this callback.
+  void setCustomCodeGenPipelineBuilderCallback(
+      const std::function<Error(ModulePassManager &, raw_pwrite_stream &,
+                                raw_pwrite_stream *, CodeGenFileType,
+                                MCContext &)>
+          C) {
+    CustomCodeGenPipelineBuilderCallback = C;
+  }
+
+  void registerCodeGenIREarlyEPCallback(
+      const std::function<void(ModulePassManager &)> C) {
+    CodeGenIREarlyEPCallbacks.push_back(C);
+  }
+
+  void registerGCLoweringEPCallback(
+      const std::function<void(FunctionPassManager &)> C) {
+    GCLoweringEPCallbacks.push_back(C);
+  }
+
+  void registerISelPrepareEPCallback(
+      const std::function<void(ModulePassManager &)> &C) {
+    ISelPrepareEPCallbacks.push_back(C);
+  }
+
+  void registerMachineSSAOptimizationEarlyEPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    MachineSSAOptimizationEarlyEPCallbacks.push_back(C);
+  }
+
+  void registerILPOptsEPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    ILPOptsEPCallbacks.push_back(C);
+  }
+
+  void registerMachineSSAOptimizationLastEPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    MachineSSAOptimizationLastEPCallbacks.push_back(C);
+  }
+
+  void registerPreRegAllocEPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    PreRegAllocEPCallbacks.push_back(C);
+  }
+
+  void registerPostRegAllocEPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    PostRegAllocEPCallbacks.push_back(C);
+  }
+
+  void registerPreRegBankSelectEPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    PreRegBankSelectEPCallbacks.push_back(C);
+  }
+
+  void registerPreGlobalInstructionSelectEPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    PreGlobalInstructionSelectEPCallbacks.push_back(C);
+  }
+
+  void registerPostGlobalInstructionSelectEPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    PostGlobalInstructionSelectEPCallbacks.push_back(C);
+  }
+
+  void registerMachineLateOptimizationEPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    MachineLateOptimizationEPCallbacks.push_back(C);
+  }
+
+  void registerPreSched2EPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    PreSched2EPCallbacks.push_back(C);
+  }
+
+  void registerPostRewriteEPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    PostRewriteEPCallbacks.push_back(C);
+  }
+
+  void registerPreEmitEPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    PreEmitEPCallbacks.push_back(C);
+  }
+
+  void registerPostBBSectionsEPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    PostBBSectionsEPCallbacks.push_back(C);
+  }
+
+  void registerMIEmitEPCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    MIEmitEPCallbacks.push_back(C);
+  }
+
+  void setAddInstSelectorCallback(
+      const std::function<void(MachineFunctionPassManager &)> &C) {
+    AddInstSelectorCallback = C;
+  }
+
+  void setCodeGenPreparePassesCallback(
+      const std::function<void(ModulePassManager &)> C) {
+    AddCodeGenPreparePassesCallback = C;
+  }
+
+  void setRegAllocFastCallback(
+      const std::function<Error(MachineFunctionPassManager &)> &C) {
+    AddRegAllocFastCallback = C;
+  }
+
+  void setRegAllocOptimizedCallback(
+      const std::function<Error(MachineFunctionPassManager &)> &C) {
+    AddRegAllocOptimizedCallback = C;
+  }
+  ///@}}
+
+  /// Building block callbacks for codegen pipeline.
+  void addDefaultCodeGenPreparePasses(ModulePassManager &MPM);
+  Error addDefaultRegAllocFastPasses(MachineFunctionPassManager &MFPM);
+  Error addDefaultRegAllocOptimizedPasses(MachineFunctionPassManager &MFPM);
+
+  // New pass manager migration methods, don't use them
+  // outside llvm!
+  CGPassBuilderOption &getCGPBO() { return CGPBO; }
+
   /// Register a callback for parsing an AliasAnalysis Name to populate
   /// the given AAManager \p AA
   void registerParseAACallback(
@@ -647,6 +797,28 @@ class PassBuilder {
                                                     OptimizationLevel Level,
                                                     ThinOrFullLTOPhase Phase);
 
+  void invokeCodeGenIREarlyEPCallbacks(ModulePassManager &MPM);
+  void invokeGCLoweringEPCallbacks(FunctionPassManager &FPM);
+  void invokeISelPrepareEPCallbacks(ModulePassManager &MPM);
+  void invokeMachineSSAOptimizationEarlyEPCallbacks(
+      MachineFunctionPassManager &MFPM);
+  void
+  invokeMachineSSAOptimizationLastEPCallbacks(MachineFunctionPassManager &MFPM);
+  void invokePreRegAllocEPCallbacks(MachineFunctionPassManager &MFPM);
+  void invokePostRegAllocEPCallbacks(MachineFunctionPassManager &MFPM);
+  void invokePreRegBankSelectEPCallbacks(MachineFunctionPassManager &MFPM);
+  void
+  invokePreGlobalInstructionSelectEPCallbacks(MachineFunctionPassManager &MFPM);
+  void invokePostGlobalInstructionSelectEPCallbacks(
+      MachineFunctionPassManager &MFPM);
+  void invokeILPOptsEPCallbacks(MachineFunctionPassManager &MFPM);
+  void
+  invokeMachineLateOptimizationEPCallbacks(MachineFunctionPassManager &MFPM);
+  void invokePreEmitEPCallbacks(MachineFunctionPassManager &MFPM);
+  void invokePostBBSectionsEPCallbacks(MachineFunctionPassManager &MFPM);
+  void invokeMIEmitEPCallbacks(MachineFunctionPassManager &MFPM);
+  void invokePreSched2EPCallbacks(MachineFunctionPassManager &MFPM);
+
   static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
     if (!Name.consume_front(PassName))
       return false;
@@ -711,6 +883,21 @@ class PassBuilder {
   void addVectorPasses(OptimizationLevel Level, FunctionPassManager &FPM,
                        bool IsFullLTO);
 
+  Error addExceptionHandlingPasses(FunctionPassManager &FPM);
+
+  Error addInstructionSelectorPasses(MachineFunctionPassManager &MFPM);
+
+  void addMachineSSAOptimizationPasses(MachineFunctionPassManager &MFPM);
+
+  Error addMachinePasses(ModulePassManager &MPM, FunctionPassManager &FPM,
+                         MachineFunctionPassManager &MFPM);
+
+  Error addRegisterAllocatorPasses(MachineFunctionPassManager &MFPM);
+
+  Error parseRegAllocOption(StringRef Text);
+
+  bool isOptimizedRegAlloc() const;
+
   static std::optional<std::vector<PipelineElement>>
   parsePipelineText(StringRef Text);
 
@@ -779,6 +966,55 @@ class PassBuilder {
               2>
       PipelineEarlySimplificationEPCallbacks;
 
+  // CodeGen extension point callbacks
+  std::function<Error(ModulePassManager &, raw_pwrite_stream &,
+                      raw_pwrite_stream *, CodeGenFileType, MCContext &)>
+      CustomCodeGenPipelineBuilderCallback;
+
+  SmallVector<std::function<void(ModulePassManager &)>, 2>
+      CodeGenIREarlyEPCallbacks;
+  SmallVector<std::function<void(FunctionPassManager &)>, 2>
+      GCLoweringEPCallbacks;
+  SmallVector<std::function<void(ModulePassManager &)>, 2>
+      ISelPrepareEPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      MachineSSAOptimizationEarlyEPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      MachineSSAOptimizationLastEPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      PreRegAllocEPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      PostRegAllocEPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      PreRegBankSelectEPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      PreGlobalInstructionSelectEPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      PostGlobalInstructionSelectEPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      ILPOptsEPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      MachineLateOptimizationEPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      PreSched2EPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      PostRewriteEPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      PreEmitEPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      PostBBSectionsEPCallbacks;
+  SmallVector<std::function<void(MachineFunctionPassManager &)>, 2>
+      MIEmitEPCallbacks;
+
+  std::function<void(ModulePassManager &)> AddCodeGenPreparePassesCallback;
+  std::function<void(MachineFunctionPassManager &)> AddInstSelectorCallback;
+  std::function<Error(MachineFunctionPassManager &)> AddRegAllocFastCallback;
+  std::function<Error(MachineFunctionPassManager &)>
+      AddRegAllocOptimizedCallback;
+  StringSet<> DisabledPasses;
+  StringMap<MachineFunctionPassManager> RegAllocPasses;
+  // TODO: Add methods in LLVMTargetMachine so we can get rid of it.
+
   SmallVector<std::function<void(ModuleAnalysisManager &)>, 2>
       ModuleAnalysisRegistrationCallbacks;
   SmallVector<std::function<bool(StringRef, ModulePassManager &,

llvm/include/llvm/Target/CGPassBuilderOption.h

@@ -42,6 +42,9 @@ struct CGPassBuilderOption {
   bool PrintLSR = false;
   bool DisableMergeICmps = false;
   bool DisablePartialLibcallInlining = false;
+  bool DisableReplaceWithVecLib = false;
+  bool DisableLayoutFSProfileLoader = false;
+  bool DisablePrologEpilogInserterPass = false;
   bool DisableConstantHoisting = false;
   bool DisableSelectOptimize = true;
   bool DisableAtExitBasedGlobalDtorLowering = false;

llvm/lib/CodeGen/TargetPassConfig.cpp

@@ -188,11 +188,11 @@ static cl::opt<bool> DisableLayoutFSProfileLoader(
     "disable-layout-fsprofile-loader", cl::init(false), cl::Hidden,
     cl::desc("Disable MIRProfileLoader before BlockPlacement"));
 // Specify FSProfile file name.
-static cl::opt<std::string>
+cl::opt<std::string>
     FSProfileFile("fs-profile-file", cl::init(""), cl::value_desc("filename"),
                   cl::desc("Flow Sensitive profile file name."), cl::Hidden);
 // Specify Remapping file for FSProfile.
-static cl::opt<std::string> FSRemappingFile(
+cl::opt<std::string> FSRemappingFile(
     "fs-remapping-file", cl::init(""), cl::value_desc("filename"),
     cl::desc("Flow Sensitive profile remapping file name."), cl::Hidden);
 
@@ -500,6 +500,8 @@ CGPassBuilderOption llvm::getCGPassBuilderOption() {
   SET_BOOLEAN_OPTION(DisableMergeICmps)
   SET_BOOLEAN_OPTION(DisableLSR)
   SET_BOOLEAN_OPTION(DisableConstantHoisting)
+  SET_BOOLEAN_OPTION(DisableReplaceWithVecLib)
+  SET_BOOLEAN_OPTION(DisableLayoutFSProfileLoader)
   SET_BOOLEAN_OPTION(DisableCGP)
   SET_BOOLEAN_OPTION(DisablePartialLibcallInlining)
   SET_BOOLEAN_OPTION(DisableSelectOptimize)
@@ -542,8 +544,7 @@ void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC,
   });
 }
 
-Expected<TargetPassConfig::StartStopInfo>
-TargetPassConfig::getStartStopInfo(PassInstrumentationCallbacks &PIC) {
+Expected<TargetPassConfig::StartStopInfo> TargetPassConfig::getStartStopInfo() {
   auto [StartBefore, StartBeforeInstanceNum] =
       getPassNameAndInstanceNum(StartBeforeOpt);
   auto [StartAfter, StartAfterInstanceNum] =

llvm/lib/Passes/CMakeLists.txt

@@ -3,6 +3,7 @@ add_llvm_component_library(LLVMPasses
   OptimizationLevel.cpp
   PassBuilder.cpp
   PassBuilderBindings.cpp
+  PassBuilderCodeGen.cpp
   PassBuilderPipelines.cpp
   PassPlugin.cpp
   StandardInstrumentations.cpp