Try to use DataDependenceGraph in VectorPredicationPass.

llvm/include/llvm/Analysis/DDGPrinter.h

@@ -87,6 +87,32 @@ private:
 
 using DDGDotGraphTraits = DOTGraphTraits<const DataDependenceGraph *>;
 
+template <>
+struct DOTGraphTraits<DataDependenceGraph *> : public DefaultDOTGraphTraits {
+
+  DOTGraphTraits(bool IsSimple = false) : DefaultDOTGraphTraits(IsSimple) {}
+
+  std::string getGraphName(DataDependenceGraph *G) {
+    assert(G && "expected a valid pointer to the graph.");
+    return "DDG for '" + std::string(G->getName()) + "'";
+  }
+
+  std::string getGraphProperties(DataDependenceGraph *G) {
+    return "\tnode [style=filled,colorscheme=paired12]\n";
+  }
+
+  std::string getNodeAttributes(DDGNode *Node, DataDependenceGraph *Graph);
+  std::string getNodeLabel(DDGNode *Node, DataDependenceGraph *Graph);
+  std::string getEdgeAttributes(DDGNode *Node,
+                                GraphTraits<DDGNode *>::ChildIteratorType I,
+                                DataDependenceGraph *G);
+
+private:
+  unsigned LastUsedColorIdx = 0;
+  unsigned MaxColorIdx = 12;
+  DenseMap<const BasicBlock *, unsigned> ColorsMap;
+};
+
 } // namespace llvm
 
 #endif // LLVM_ANALYSIS_DDGPRINTER_H

llvm/include/llvm/Transforms/Vectorize/VectorPredication.h

@@ -8,41 +8,24 @@ namespace llvm {
 
 using InstToMaskEVLMap = DenseMap<Instruction *, std::pair<Value *, Value *>>;
 
-struct BlockData {
+class VectorPredicationPass : public PassInfoMixin<VectorPredicationPass> {
+private:
   // Vector that stores all vector predicated memory writing operations found in
-  // the basic block. If after phase 1 is empty, then the basic block can be
-  // skipped by following phases.
+  // the function.
   SmallVector<Instruction *> MemoryWritingVPInstructions;
 
-  // Store all instructions of the basic block (in the same order as they are
-  // found), assigning to each the list of users. Skip PHIs and terminators.
-  MapVector<Instruction *, SmallPtrSet<Instruction *, 4>> TopologicalGraph;
-
-  // Map each full-length vector operation eligible to be transformed to a
-  // vector predication one with the (mask,evl) pair of its first vector
+  // Map each full-length vector operation, eligible to be transformed to a
+  // vector predication one, with the {mask,evl} pair of its first vector
   // predicated memory writing operation user.
   InstToMaskEVLMap VecOpsToTransform;
 
-  // Ordered list representing the reverse order of how the basic block has to
-  // be transformed due to the new vector predicated instructions.
-  SmallVector<Instruction *> NewBBReverseOrder;
-
-  BlockData() = default;
-};
-
-class VectorPredicationPass : public PassInfoMixin<VectorPredicationPass> {
-private:
   // List of instructions to be replaced by the new VP operations and that later
   // should be removed, if possible.
   DenseMap<Instruction *, Value *> OldInstructionsToRemove;
 
-  void analyseBasicBlock(BasicBlock &BB, BlockData &BBInfo);
-  void findCandidateVectorOperations(BasicBlock &BB, BlockData &BBInfo);
-  void addNewUsersToMasksAndEVLs(BasicBlock &BB, BlockData &BBInfo);
-  void buildNewBasicBlockSchedule(BasicBlock &BB, BlockData &BBInfo);
-  void emitNewBasicBlockSchedule(BasicBlock &BB, BlockData &BBInfo);
-  void transformCandidateVectorOperations(BasicBlock &BB, BlockData &BBInfo);
-
+  void analyseFunction(Function &F);
+  void findCandidateVectorOperations();
+  void transformCandidateVectorOperations();
   void removeOldInstructions();
 
 public:

llvm/lib/Analysis/DDGPrinter.cpp

@@ -148,3 +148,67 @@ std::string DDGDotGraphTraits::getVerboseEdgeAttributes(
   OS << "]\"";
   return OS.str();
 }
+
+std::string DOTGraphTraits<DataDependenceGraph *>::getNodeAttributes(
+    DDGNode *Node, DataDependenceGraph *Graph) {
+  std::string Str;
+  raw_string_ostream OS(Str);
+
+  if (auto *SimpleNode = dyn_cast<SimpleDDGNode>(Node)) {
+    unsigned ColorIdx = 0;
+    for (auto *II : SimpleNode->getInstructions()) {
+      auto ColorIdxIt = ColorsMap.find(II->getParent());
+      if (ColorIdxIt == ColorsMap.end()) {
+        ColorIdxIt =
+            ColorsMap
+                .insert(std::make_pair(II->getParent(), ++LastUsedColorIdx))
+                .first;
+      }
+      if (ColorIdx == 0)
+        ColorIdx = ColorIdxIt->second;
+      else if (ColorIdx != ColorIdxIt->second)
+        break;
+    }
+    if (ColorIdx > 0 && ColorIdx <= MaxColorIdx)
+      OS << "color=" << ColorIdx;
+  }
+
+  return OS.str();
+}
+
+std::string DOTGraphTraits<DataDependenceGraph *>::getNodeLabel(
+    DDGNode *Node, DataDependenceGraph *Graph) {
+  std::string Str;
+  raw_string_ostream OS(Str);
+  OS << "Kind:" << Node->getKind() << "\n";
+  if (auto *SimpleNode = dyn_cast<SimpleDDGNode>(Node))
+    for (auto *II : SimpleNode->getInstructions())
+      OS << *II << "\n";
+  else if (auto *PiNode = dyn_cast<PiBlockDDGNode>(Node))
+    OS << "with\n" << PiNode->getNodes().size() << " nodes\n";
+  else if (!isa<RootDDGNode>(Node))
+    llvm_unreachable("Unimplemented type of node");
+  return OS.str();
+}
+
+std::string DOTGraphTraits<DataDependenceGraph *>::getEdgeAttributes(
+    DDGNode *Node, GraphTraits<DDGNode *>::ChildIteratorType I,
+    DataDependenceGraph *G) {
+  std::string Str;
+  raw_string_ostream OS(Str);
+
+  DDGEdge *Edge = *I.getCurrent();
+  DDGEdge::EdgeKind Kind = Edge->getKind();
+  OS << "style=";
+  if (Kind == DDGEdge::EdgeKind::Rooted)
+    OS << "dotted";
+  else if (Kind == DDGEdge::EdgeKind::MemoryDependence)
+    OS << "dashed, color=blue";
+  else if (Kind == DDGEdge::EdgeKind::Unknown)
+    OS << "invis";
+  else
+    OS << "bold";
+  OS << "";
+
+  return OS.str();
+}

llvm/lib/Transforms/Vectorize/VectorPredication.cpp

 #include "llvm/Transforms/Vectorize/VectorPredication.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/DDG.h"
+#include "llvm/Analysis/DDGPrinter.h"
+#include "llvm/Analysis/DependenceAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/VectorBuilder.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/GraphWriter.h"
 #include "llvm/Transforms/Utils/Local.h"
 
 #define DEBUG_TYPE "vector-predication"
@@ -12,54 +21,37 @@ STATISTIC(Transforms, "Number of full-length -> evl vector transformation.");
 
 using namespace llvm;
 
-// Map each instruction to its uses and save all memory writing vector
-// predicated instructions found in the basic block.
-void VectorPredicationPass::analyseBasicBlock(BasicBlock &BB,
-                                              BlockData &BBInfo) {
-  for (Instruction &I : BB) {
-    if (isa<PHINode>(I) || I.isTerminator())
-      continue;
+namespace {
+cl::opt<bool> PrintDDG("print-ddg", cl::Hidden, cl::init(false),
+                       cl::desc("Create dot file for the DDG"));
+} // namespace
 
-    SmallPtrSet<Instruction *, 4> IUsers;
-    for (User *IU : I.users()) {
-      assert(isa<Instruction>(IU) && "Unexpected behaviour.");
-      auto *IUInst = cast<Instruction>(IU);
-      if (IUInst->getParent() != I.getParent())
-        continue;
-      if (isa<PHINode>(IUInst) || IUInst->isTerminator())
-        continue;
-
-      IUsers.insert(IUInst);
-    }
-    BBInfo.TopologicalGraph.insert({&I, IUsers});
-
-    if (auto *CI = dyn_cast<CallInst>(&I)) {
-      if (auto *CF = CI->getCalledFunction()) {
-        Intrinsic::ID ID = CF->getIntrinsicID();
+// Register all memory writing VP instructions found in the function.
+void VectorPredicationPass::analyseFunction(Function &F) {
+  for (BasicBlock &BB : F) {
+    for (Instruction &I : BB) {
+      if (auto *CI = dyn_cast<CallInst>(&I)) {
+        Intrinsic::ID ID = CI->getIntrinsicID();
         if (ID == Intrinsic::vp_store || ID == Intrinsic::vp_scatter) {
-          BBInfo.MemoryWritingVPInstructions.push_back(&I);
+          MemoryWritingVPInstructions.push_back(&I);
         }
       }
     }
   }
 }
 
-static void findCandidateVectorOperation(BasicBlock &BB, Value *Op, Value *Mask,
-                                         Value *EVL,
+static void findCandidateVectorOperation(Value *Op, Value *Mask, Value *EVL,
                                          InstToMaskEVLMap &VecOpsToTransform) {
   auto *OpInst = dyn_cast<Instruction>(Op);
   if (!OpInst)
     return;
 
-  if (OpInst->getParent() != &BB)
-    return;
-
   Intrinsic::ID VPID = VPIntrinsic::getForOpcode(OpInst->getOpcode());
   if (VPID == Intrinsic::not_intrinsic)
     return;
 
   // If the instruction is already present in the map, it means it was already
-  // visited starting from a previous memory wrtiting vp operation.
+  // visited starting from a previous memory writing vp operation.
   if (!VecOpsToTransform
            .insert(std::make_pair(OpInst, std::make_pair(Mask, EVL)))
            .second) {
@@ -87,114 +79,54 @@ static void findCandidateVectorOperation(BasicBlock &BB, Value *Op, Value *Mask,
   switch (VPID) {
   default:
     for (auto *OpVal : OpInst->operand_values())
-      findCandidateVectorOperation(BB, OpVal, Mask, EVL, VecOpsToTransform);
+      findCandidateVectorOperation(OpVal, Mask, EVL, VecOpsToTransform);
     break;
   case Intrinsic::vp_select: {
     Value *Cond = OpInst->getOperand(0);
     if (Cond->getType()->isVectorTy())
-      findCandidateVectorOperation(BB, Cond, nullptr, EVL, VecOpsToTransform);
+      findCandidateVectorOperation(Cond, nullptr, EVL, VecOpsToTransform);
 
     // TODO: if the condition argument is a vector, we could backpropagate it
     // as mask for the true branch and its negation as mask for the false one.
     // WARNING: when creating the negation of the condition, we must ensure it
     // dominates all uses.
-    findCandidateVectorOperation(BB, OpInst->getOperand(1), nullptr, EVL,
+    findCandidateVectorOperation(OpInst->getOperand(1), nullptr, EVL,
                                  VecOpsToTransform);
-    findCandidateVectorOperation(BB, OpInst->getOperand(2), nullptr, EVL,
+    findCandidateVectorOperation(OpInst->getOperand(2), nullptr, EVL,
                                  VecOpsToTransform);
     break;
   }
   }
 }
 
-// For each vector predicated memory writing operation of the basic block, go
-// back to the stored vector defining instruction and verify it is a vector
-// operation. Add it to the list of instructions to be transformed into vector
-// predicated ones, then recursively repeat the process for its vector
-// arguments.
-void VectorPredicationPass::findCandidateVectorOperations(BasicBlock &BB,
-                                                          BlockData &BBInfo) {
-  if (BBInfo.MemoryWritingVPInstructions.empty())
+// For each VP memory writing operation, go back to the stored vector defining
+// instruction and verify it is a vector operation. Add it to the list of
+// instructions to be transformed into vector predicated ones, then recursively
+// repeat the process for its vector arguments.
+void VectorPredicationPass::findCandidateVectorOperations() {
+  if (MemoryWritingVPInstructions.empty())
     return;
 
-  for (Instruction *I : BBInfo.MemoryWritingVPInstructions) {
-    assert(I->getParent() == &BB && "This is not the right basic block");
+  for (Instruction *I : MemoryWritingVPInstructions) {
     auto *VPI = cast<VPIntrinsic>(I);
     Value *StoredOperand = VPI->getMemoryDataParam();
     Value *MaskOperand = VPI->getMaskParam();
     Value *EVLOperand = VPI->getVectorLengthParam();
     // First, visit the mask operand (assigning an allones mask to this branch)
     // and only then visit the stored operand.
-    findCandidateVectorOperation(BB, MaskOperand, nullptr, EVLOperand,
-                                 BBInfo.VecOpsToTransform);
-    findCandidateVectorOperation(BB, StoredOperand, MaskOperand, EVLOperand,
-                                 BBInfo.VecOpsToTransform);
-  }
-}
-
-// Add the candidates as users of the mask and evl linked to each of them.
-void VectorPredicationPass::addNewUsersToMasksAndEVLs(BasicBlock &BB,
-                                                      BlockData &BBInfo) {
-  if (BBInfo.VecOpsToTransform.empty())
-    return;
-
-  for (auto [K, V] : BBInfo.VecOpsToTransform) {
-    if (auto *MaskInst = dyn_cast_if_present<Instruction>(V.first))
-      BBInfo.TopologicalGraph[MaskInst].insert(K);
-    if (auto *EVLInst = dyn_cast<Instruction>(V.second))
-      BBInfo.TopologicalGraph[EVLInst].insert(K);
-  }
-}
-
-// Topologically sort, preserving as much as possible the original order.
-void VectorPredicationPass::buildNewBasicBlockSchedule(BasicBlock &BB,
-                                                       BlockData &BBInfo) {
-  if (BBInfo.VecOpsToTransform.empty())
-    return;
-
-  while (!BBInfo.TopologicalGraph.empty()) {
-    Instruction *Inst = nullptr;
-    for (auto B = BBInfo.TopologicalGraph.rbegin(),
-              E = BBInfo.TopologicalGraph.rend();
-         B != E; B++) {
-      if (B->second.empty()) {
-        Inst = B->first;
-        break;
-      }
-    }
-    assert(Inst && "Failed to empty topological graph!");
-
-    BBInfo.NewBBReverseOrder.push_back(Inst);
-    BBInfo.TopologicalGraph.erase(Inst);
-
-    for (auto B = BBInfo.TopologicalGraph.begin(),
-              E = BBInfo.TopologicalGraph.end();
-         B != E; B++) {
-      B->second.erase(Inst);
-    }
-  }
-}
-
-// Modify the basic block based on the topological order generated.
-void VectorPredicationPass::emitNewBasicBlockSchedule(BasicBlock &BB,
-                                                      BlockData &BBInfo) {
-  if (BBInfo.VecOpsToTransform.empty())
-    return;
-
-  Instruction *InsertPoint = BB.getTerminator();
-  for (Instruction *I : BBInfo.NewBBReverseOrder) {
-    I->moveBefore(InsertPoint);
-    InsertPoint = I;
+    findCandidateVectorOperation(MaskOperand, nullptr, EVLOperand,
+                                 VecOpsToTransform);
+    findCandidateVectorOperation(StoredOperand, MaskOperand, EVLOperand,
+                                 VecOpsToTransform);
   }
 }
 
 // Transform candidates to vector predicated instructions.
-void VectorPredicationPass::transformCandidateVectorOperations(
-    BasicBlock &BB, BlockData &BBInfo) {
-  if (BBInfo.VecOpsToTransform.empty())
+void VectorPredicationPass::transformCandidateVectorOperations() {
+  if (VecOpsToTransform.empty())
     return;
 
-  for (auto [I, P] : BBInfo.VecOpsToTransform) {
+  for (auto [I, P] : VecOpsToTransform) {
     Value *Mask, *EVL;
     std::tie(Mask, EVL) = P;
 
@@ -251,6 +183,8 @@ void VectorPredicationPass::removeOldInstructions() {
     if (isInstructionTriviallyDead(I))
       I->eraseFromParent();
   }
+
+  OldInstructionsToRemove.clear();
 }
 
 PreservedAnalyses VectorPredicationPass::run(Function &F,
@@ -258,20 +192,34 @@ PreservedAnalyses VectorPredicationPass::run(Function &F,
   assert(OldInstructionsToRemove.empty() &&
          "Map should be cleared at the end of each run of the pass.");
 
-  for (BasicBlock &BB : F) {
-    BlockData BBInfo;
+  analyseFunction(F);
+  findCandidateVectorOperations();
+  // TODO: before transformation, create DDG and use it to check if adding a new
+  // edge creates a cycle.
+  transformCandidateVectorOperations();
+  removeOldInstructions();
 
-    analyseBasicBlock(BB, BBInfo);
-    findCandidateVectorOperations(BB, BBInfo);
-    addNewUsersToMasksAndEVLs(BB, BBInfo);
-    buildNewBasicBlockSchedule(BB, BBInfo);
-    emitNewBasicBlockSchedule(BB, BBInfo);
-    transformCandidateVectorOperations(BB, BBInfo);
+  // Retrieve DependenceInfo for the function.
+  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfo TLI(TLII);
+  AAResults AA(TLI);
+  AssumptionCache AC(F);
+  DominatorTree DT(F);
+  LoopInfo LI(DT);
+  ScalarEvolution SE(F, TLI, AC, DT, LI);
+  DependenceInfo DI(&F, &AA, &SE, &LI);
+
+  DataDependenceGraph DDG(F, DI);
+  if (PrintDDG) {
+    std::string Filename = ("ddg." + F.getName() + ".dot").str();
+    std::error_code EC;
+    raw_fd_ostream File(Filename, EC, sys::fs::OF_Text);
+    if (!EC)
+      WriteGraph(File, &DDG);
+    else
+      errs() << "Error opening file for writing!\n";
   }
 
-  removeOldInstructions();
-  OldInstructionsToRemove.clear();
-
   // TODO: think about which analysis are preserved.
   return PreservedAnalyses::none();
 }