Cleaned up after experiments.

Author: JonPsson

llvm/include/llvm/CodeGen/MachineCombinerPattern.h

@@ -176,6 +176,15 @@ enum class MachineCombinerPattern {
   FMSUB,
   FNMSUB,
 
+  // SystemZ patterns.  (EXPERIMENTAL)
+  FMA2_P1P0,
+  FMA2_P0P1,
+  FMA2,
+  FMA1_Add_L,
+  FMA1_Add_R,
+  FMA3,     // These are inspired by PPC
+  FMA2_Add, //
+
   // X86 VNNI
   DPWSSD,
 

llvm/include/llvm/CodeGen/TargetInstrInfo.h

@@ -1698,7 +1698,7 @@ class TargetInstrInfo : public MCInstrInfo {
   /// instruction that defines FoldAsLoadDefReg, and the function returns
   /// the machine instruction generated due to folding.
   virtual MachineInstr *optimizeLoadInstr(MachineInstr &MI,
-                                          const MachineRegisterInfo *MRI,
+                                          MachineRegisterInfo *MRI,
                                           Register &FoldAsLoadDefReg,
                                           MachineInstr *&DefMI) const {
     return nullptr;

llvm/lib/CodeGen/MachineCombiner.cpp

@@ -155,9 +155,7 @@ MachineCombiner::getOperandDef(const MachineOperand &MO) {
   // We need a virtual register definition.
   if (MO.isReg() && MO.getReg().isVirtual())
     DefInstr = MRI->getUniqueVRegDef(MO.getReg());
-  // PHI's have no depth etc.
-  if (DefInstr && DefInstr->isPHI())
-    DefInstr = nullptr;
+  // (PATCH PROPOSED for PHIs: https://github.com/llvm/llvm-project/pull/82025)
   return DefInstr;
 }
 
@@ -317,6 +315,13 @@ static CombinerObjective getCombinerObjective(MachineCombinerPattern P) {
   case MachineCombinerPattern::FMADD_XA:
   case MachineCombinerPattern::FMSUB:
   case MachineCombinerPattern::FNMSUB:
+  case MachineCombinerPattern::FMA2_P1P0:
+  case MachineCombinerPattern::FMA2_P0P1:
+  case MachineCombinerPattern::FMA2:
+  case MachineCombinerPattern::FMA1_Add_L:
+  case MachineCombinerPattern::FMA1_Add_R:
+  case MachineCombinerPattern::FMA3:
+  case MachineCombinerPattern::FMA2_Add:
     return CombinerObjective::MustReduceDepth;
   case MachineCombinerPattern::REASSOC_XY_BCA:
   case MachineCombinerPattern::REASSOC_XY_BAC:

llvm/lib/Target/SystemZ/CMakeLists.txt

@@ -20,6 +20,7 @@ add_llvm_target(SystemZCodeGen
   SystemZConstantPoolValue.cpp
   SystemZCopyPhysRegs.cpp
   SystemZElimCompare.cpp
+  SystemZFinalizeReassociation.cpp
   SystemZFrameLowering.cpp
   SystemZHazardRecognizer.cpp
   SystemZISelDAGToDAG.cpp

llvm/lib/Target/SystemZ/SystemZ.h

@@ -195,12 +195,14 @@ FunctionPass *createSystemZShortenInstPass(SystemZTargetMachine &TM);
 FunctionPass *createSystemZLongBranchPass(SystemZTargetMachine &TM);
 FunctionPass *createSystemZLDCleanupPass(SystemZTargetMachine &TM);
 FunctionPass *createSystemZCopyPhysRegsPass(SystemZTargetMachine &TM);
+FunctionPass *createSystemZFinalizeReassociationPass(SystemZTargetMachine &TM);
 FunctionPass *createSystemZPostRewritePass(SystemZTargetMachine &TM);
 FunctionPass *createSystemZTDCPass();
 
 void initializeSystemZCopyPhysRegsPass(PassRegistry &);
 void initializeSystemZDAGToDAGISelPass(PassRegistry &);
 void initializeSystemZElimComparePass(PassRegistry &);
+void initializeSystemZFinalizeReassociationPass(PassRegistry &);
 void initializeSystemZLDCleanupPass(PassRegistry &);
 void initializeSystemZLongBranchPass(PassRegistry &);
 void initializeSystemZPostRewritePass(PassRegistry &);

llvm/lib/Target/SystemZ/SystemZFinalizeReassociation.cpp

@@ -0,0 +1,127 @@
+//===---- SystemZFinalizeReassociation.cpp - Finalize FP reassociation ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass is the last step of the process of enabling reassociation with
+// the MachineCombiner. These are the steps involved:
+//
+// 1. Instruction selection: Disable reg/mem folding for any operations that
+//    are reassociable since MachineCombiner will not succeed
+//    otherwise. Instead select a reg/reg pseudo that pretends to clobber CC.
+//
+// 2. MachineCombiner: Performs reassociation with the reg/reg instructions.
+//
+// 3. PeepholeOptimizer: fold loads into reg/mem instructions after
+//    reassociation. The reg/mem opcode sets CC which is why the special
+//    reg/reg pseudo is needed.
+//
+// 4. Convert any remaining pseudos into the target opcodes that do not
+//    clobber CC (this pass).
+//
+//===----------------------------------------------------------------------===//
+
+#include "SystemZMachineFunctionInfo.h"
+#include "SystemZTargetMachine.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/Target/TargetMachine.h"
+
+using namespace llvm;
+
+namespace {
+
+class SystemZFinalizeReassociation : public MachineFunctionPass {
+public:
+  static char ID;
+  SystemZFinalizeReassociation()
+    : MachineFunctionPass(ID), TII(nullptr), MRI(nullptr) {
+    initializeSystemZFinalizeReassociationPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+private:
+
+  bool visitMBB(MachineBasicBlock &MBB);
+
+  const SystemZInstrInfo *TII;
+  MachineRegisterInfo *MRI;
+};
+
+char SystemZFinalizeReassociation::ID = 0;
+
+} // end anonymous namespace
+
+INITIALIZE_PASS(SystemZFinalizeReassociation, "systemz-finalize-reassoc",
+                "SystemZ Finalize Reassociation", false, false)
+
+FunctionPass *llvm::
+createSystemZFinalizeReassociationPass(SystemZTargetMachine &TM) {
+  return new SystemZFinalizeReassociation();
+}
+
+void SystemZFinalizeReassociation::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.setPreservesCFG();
+  MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+bool SystemZFinalizeReassociation::visitMBB(MachineBasicBlock &MBB) {
+  bool Changed = false;
+  for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
+    unsigned PseudoOpcode = MI.getOpcode();
+    unsigned TargetOpcode =
+      PseudoOpcode == SystemZ::WFADB_CCPseudo    ? SystemZ::WFADB
+      : PseudoOpcode == SystemZ::WFASB_CCPseudo  ? SystemZ::WFASB
+      : PseudoOpcode == SystemZ::WFSDB_CCPseudo  ? SystemZ::WFSDB
+      : PseudoOpcode == SystemZ::WFSSB_CCPseudo  ? SystemZ::WFSSB
+      : PseudoOpcode == SystemZ::WFMDB_CCPseudo  ? SystemZ::WFMDB
+      : PseudoOpcode == SystemZ::WFMSB_CCPseudo  ? SystemZ::WFMSB
+      : PseudoOpcode == SystemZ::WFMADB_CCPseudo ? SystemZ::WFMADB
+      : PseudoOpcode == SystemZ::WFMASB_CCPseudo ? SystemZ::WFMASB
+      : 0;
+    if (TargetOpcode) {
+      // PeepholeOptimizer will not fold any loads across basic blocks, which
+      // however seems beneficial, so do it here:
+      bool Folded = false;
+      for (unsigned Op = 1; Op <= 2; ++Op) {
+        Register Reg = MI.getOperand(Op).getReg();
+        if (MachineInstr *DefMI = MRI->getVRegDef(Reg))
+          if (TII->optimizeLoadInstr(MI, MRI, Reg, DefMI)) {
+            MI.eraseFromParent();
+            DefMI->eraseFromParent();
+            MRI->markUsesInDebugValueAsUndef(Reg);
+            Folded = true;
+            break;
+          }
+      }
+
+      if (!Folded) {
+        MI.setDesc(TII->get(TargetOpcode));
+        int CCIdx = MI.findRegisterDefOperandIdx(SystemZ::CC);
+        MI.removeOperand(CCIdx);
+      }
+      Changed = true;
+    }
+  }
+  return Changed;
+}
+
+bool SystemZFinalizeReassociation::runOnMachineFunction(MachineFunction &F) {
+  TII = F.getSubtarget<SystemZSubtarget>().getInstrInfo();
+  MRI = &F.getRegInfo();
+
+  bool Modified = false;
+  for (auto &MBB : F)
+    Modified |= visitMBB(MBB);
+
+  return Modified;
+}

llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp

@@ -347,6 +347,13 @@ class SystemZDAGToDAGISel : public SelectionDAGISel {
   // Try to expand a boolean SELECT_CCMASK using an IPM sequence.
   SDValue expandSelectBoolean(SDNode *Node);
 
+  // Return true if the flags of N and the subtarget allows for reassociation.
+  bool isReassociable(SDNode *N) const {
+    return N->getFlags().hasAllowReassociation() &&
+           N->getFlags().hasNoSignedZeros() &&
+           Subtarget->hasVector();
+  }
+
 public:
   static char ID;
 

llvm/lib/Target/SystemZ/SystemZInstrFP.td

@@ -430,8 +430,10 @@ let Uses = [FPC], mayRaiseFPException = 1,
     def ADBR : BinaryRRE<"adbr", 0xB31A, any_fadd, FP64,  FP64>;
     def AXBR : BinaryRRE<"axbr", 0xB34A, any_fadd, FP128, FP128>;
   }
-  defm AEB : BinaryRXEAndPseudo<"aeb", 0xED0A, any_fadd, FP32, load, 4>;
-  defm ADB : BinaryRXEAndPseudo<"adb", 0xED1A, any_fadd, FP64, load, 8>;
+  defm AEB : BinaryRXEAndPseudo<"aeb", 0xED0A, z_any_fadd_noreassoc, FP32,
+                                load, 4>;
+  defm ADB : BinaryRXEAndPseudo<"adb", 0xED1A, z_any_fadd_noreassoc, FP64,
+                                load, 8>;
 }
 
 // Subtraction.
@@ -441,8 +443,10 @@ let Uses = [FPC], mayRaiseFPException = 1,
   def SDBR : BinaryRRE<"sdbr", 0xB31B, any_fsub, FP64,  FP64>;
   def SXBR : BinaryRRE<"sxbr", 0xB34B, any_fsub, FP128, FP128>;
 
-  defm SEB : BinaryRXEAndPseudo<"seb",  0xED0B, any_fsub, FP32, load, 4>;
-  defm SDB : BinaryRXEAndPseudo<"sdb",  0xED1B, any_fsub, FP64, load, 8>;
+  defm SEB : BinaryRXEAndPseudo<"seb",  0xED0B, z_any_fsub_noreassoc, FP32,
+                                load, 4>;
+  defm SDB : BinaryRXEAndPseudo<"sdb",  0xED1B, z_any_fsub_noreassoc, FP64,
+                                load, 8>;
 }
 
 // Multiplication.
@@ -452,8 +456,10 @@ let Uses = [FPC], mayRaiseFPException = 1 in {
     def MDBR  : BinaryRRE<"mdbr",  0xB31C, any_fmul, FP64,  FP64>;
     def MXBR  : BinaryRRE<"mxbr",  0xB34C, any_fmul, FP128, FP128>;
   }
-  defm MEEB : BinaryRXEAndPseudo<"meeb", 0xED17, any_fmul, FP32, load, 4>;
-  defm MDB  : BinaryRXEAndPseudo<"mdb",  0xED1C, any_fmul, FP64, load, 8>;
+  defm MEEB : BinaryRXEAndPseudo<"meeb", 0xED17, z_any_fmul_noreassoc, FP32,
+                                 load, 4>;
+  defm MDB  : BinaryRXEAndPseudo<"mdb",  0xED1C, z_any_fmul_noreassoc, FP64,
+                                 load, 8>;
 }
 
 // f64 multiplication of two FP32 registers.
@@ -495,8 +501,10 @@ let Uses = [FPC], mayRaiseFPException = 1 in {
   def MAEBR : TernaryRRD<"maebr", 0xB30E, z_any_fma, FP32, FP32>;
   def MADBR : TernaryRRD<"madbr", 0xB31E, z_any_fma, FP64, FP64>;
 
-  defm MAEB : TernaryRXFAndPseudo<"maeb", 0xED0E, z_any_fma, FP32, FP32, load, 4>;
-  defm MADB : TernaryRXFAndPseudo<"madb", 0xED1E, z_any_fma, FP64, FP64, load, 8>;
+  defm MAEB : TernaryRXFAndPseudo<"maeb", 0xED0E, z_any_fma_noreassoc, FP32,
+                                  FP32, load, 4>;
+  defm MADB : TernaryRXFAndPseudo<"madb", 0xED1E, z_any_fma_noreassoc, FP64,
+                                  FP64, load, 8>;
 }
 
 // Fused multiply-subtract.

llvm/lib/Target/SystemZ/SystemZInstrFormats.td

@@ -5536,3 +5536,33 @@ multiclass StringRRE<string mnemonic, bits<16> opcode,
                       [(set GR64:$end, (operator GR64:$start1, GR64:$start2,
                                                  GR32:$char))]>;
 }
+
+multiclass BinaryVRRcAndCCPseudo<string mnemonic, bits<16> opcode,
+                                 SDPatternOperator operator,
+                                 SDPatternOperator reassoc_operator,
+                                 TypedReg tr1, TypedReg tr2, bits<4> type = 0,
+                                 bits<4> m5 = 0, bits<4> m6 = 0,
+                                 string fp_mnemonic = ""> {
+  def "" : BinaryVRRc<mnemonic, opcode, operator, tr1, tr2, type, m5, m6,
+                      fp_mnemonic>;
+  let Defs = [CC], AddedComplexity = 1 in // Win over "".
+    def _CCPseudo : Pseudo<(outs tr1.op:$V1), (ins tr2.op:$V2, tr2.op:$V3),
+                           [(set (tr1.vt tr1.op:$V1),
+                                 (reassoc_operator (tr2.vt tr2.op:$V2),
+                                                   (tr2.vt tr2.op:$V3)))]>;
+}
+
+multiclass TernaryVRReAndCCPseudo<string mnemonic, bits<16> opcode,
+                                  SDPatternOperator operator,
+                                  SDPatternOperator reassoc_operator,
+                                  TypedReg tr1, TypedReg tr2, bits<4> m5 = 0,
+                                  bits<4> type = 0, string fp_mnemonic = ""> {
+  def "" : TernaryVRRe<mnemonic, opcode, operator, tr1, tr2, m5, type,
+                       fp_mnemonic>;
+  let Defs = [CC], AddedComplexity = 1 in // Win over "".
+    def _CCPseudo : Pseudo<(outs tr1.op:$V1),
+                           (ins tr2.op:$V2, tr2.op:$V3, tr1.op:$V4),
+            [(set (tr1.vt tr1.op:$V1), (reassoc_operator (tr2.vt tr2.op:$V2),
+                                                         (tr2.vt tr2.op:$V3),
+                                                         (tr1.vt tr1.op:$V4)))]>;
+}