Skip to content
Snippets Groups Projects
Commit e0befe16 authored by Lorenzo Albano's avatar Lorenzo Albano
Browse files

Try to use DataDependenceGraph in VectorPredicationPass.

parent feb66a09
Branches use-ddg
No related tags found
No related merge requests found
......@@ -87,6 +87,32 @@ private:
using DDGDotGraphTraits = DOTGraphTraits<const DataDependenceGraph *>;
template <>
struct DOTGraphTraits<DataDependenceGraph *> : public DefaultDOTGraphTraits {
DOTGraphTraits(bool IsSimple = false) : DefaultDOTGraphTraits(IsSimple) {}
std::string getGraphName(DataDependenceGraph *G) {
assert(G && "expected a valid pointer to the graph.");
return "DDG for '" + std::string(G->getName()) + "'";
}
std::string getGraphProperties(DataDependenceGraph *G) {
return "\tnode [style=filled,colorscheme=paired12]\n";
}
std::string getNodeAttributes(DDGNode *Node, DataDependenceGraph *Graph);
std::string getNodeLabel(DDGNode *Node, DataDependenceGraph *Graph);
std::string getEdgeAttributes(DDGNode *Node,
GraphTraits<DDGNode *>::ChildIteratorType I,
DataDependenceGraph *G);
private:
unsigned LastUsedColorIdx = 0;
unsigned MaxColorIdx = 12;
DenseMap<const BasicBlock *, unsigned> ColorsMap;
};
} // namespace llvm
#endif // LLVM_ANALYSIS_DDGPRINTER_H
......@@ -8,41 +8,24 @@ namespace llvm {
using InstToMaskEVLMap = DenseMap<Instruction *, std::pair<Value *, Value *>>;
struct BlockData {
class VectorPredicationPass : public PassInfoMixin<VectorPredicationPass> {
private:
// Vector that stores all vector predicated memory writing operations found in
// the basic block. If after phase 1 is empty, then the basic block can be
// skipped by following phases.
// the function.
SmallVector<Instruction *> MemoryWritingVPInstructions;
// Store all instructions of the basic block (in the same order as they are
// found), assigning to each the list of users. Skip PHIs and terminators.
MapVector<Instruction *, SmallPtrSet<Instruction *, 4>> TopologicalGraph;
// Map each full-length vector operation eligible to be transformed to a
// vector predication one with the (mask,evl) pair of its first vector
// Map each full-length vector operation, eligible to be transformed to a
// vector predication one, with the {mask,evl} pair of its first vector
// predicated memory writing operation user.
InstToMaskEVLMap VecOpsToTransform;
// Ordered list representing the reverse order of how the basic block has to
// be transformed due to the new vector predicated instructions.
SmallVector<Instruction *> NewBBReverseOrder;
BlockData() = default;
};
class VectorPredicationPass : public PassInfoMixin<VectorPredicationPass> {
private:
// List of instructions to be replaced by the new VP operations and that later
// should be removed, if possible.
DenseMap<Instruction *, Value *> OldInstructionsToRemove;
void analyseBasicBlock(BasicBlock &BB, BlockData &BBInfo);
void findCandidateVectorOperations(BasicBlock &BB, BlockData &BBInfo);
void addNewUsersToMasksAndEVLs(BasicBlock &BB, BlockData &BBInfo);
void buildNewBasicBlockSchedule(BasicBlock &BB, BlockData &BBInfo);
void emitNewBasicBlockSchedule(BasicBlock &BB, BlockData &BBInfo);
void transformCandidateVectorOperations(BasicBlock &BB, BlockData &BBInfo);
void analyseFunction(Function &F);
void findCandidateVectorOperations();
void transformCandidateVectorOperations();
void removeOldInstructions();
public:
......
......@@ -148,3 +148,67 @@ std::string DDGDotGraphTraits::getVerboseEdgeAttributes(
OS << "]\"";
return OS.str();
}
std::string DOTGraphTraits<DataDependenceGraph *>::getNodeAttributes(
DDGNode *Node, DataDependenceGraph *Graph) {
std::string Str;
raw_string_ostream OS(Str);
if (auto *SimpleNode = dyn_cast<SimpleDDGNode>(Node)) {
unsigned ColorIdx = 0;
for (auto *II : SimpleNode->getInstructions()) {
auto ColorIdxIt = ColorsMap.find(II->getParent());
if (ColorIdxIt == ColorsMap.end()) {
ColorIdxIt =
ColorsMap
.insert(std::make_pair(II->getParent(), ++LastUsedColorIdx))
.first;
}
if (ColorIdx == 0)
ColorIdx = ColorIdxIt->second;
else if (ColorIdx != ColorIdxIt->second)
break;
}
if (ColorIdx > 0 && ColorIdx <= MaxColorIdx)
OS << "color=" << ColorIdx;
}
return OS.str();
}
std::string DOTGraphTraits<DataDependenceGraph *>::getNodeLabel(
DDGNode *Node, DataDependenceGraph *Graph) {
std::string Str;
raw_string_ostream OS(Str);
OS << "Kind:" << Node->getKind() << "\n";
if (auto *SimpleNode = dyn_cast<SimpleDDGNode>(Node))
for (auto *II : SimpleNode->getInstructions())
OS << *II << "\n";
else if (auto *PiNode = dyn_cast<PiBlockDDGNode>(Node))
OS << "with\n" << PiNode->getNodes().size() << " nodes\n";
else if (!isa<RootDDGNode>(Node))
llvm_unreachable("Unimplemented type of node");
return OS.str();
}
std::string DOTGraphTraits<DataDependenceGraph *>::getEdgeAttributes(
DDGNode *Node, GraphTraits<DDGNode *>::ChildIteratorType I,
DataDependenceGraph *G) {
std::string Str;
raw_string_ostream OS(Str);
DDGEdge *Edge = *I.getCurrent();
DDGEdge::EdgeKind Kind = Edge->getKind();
OS << "style=";
if (Kind == DDGEdge::EdgeKind::Rooted)
OS << "dotted";
else if (Kind == DDGEdge::EdgeKind::MemoryDependence)
OS << "dashed, color=blue";
else if (Kind == DDGEdge::EdgeKind::Unknown)
OS << "invis";
else
OS << "bold";
OS << "";
return OS.str();
}
#include "llvm/Transforms/Vectorize/VectorPredication.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/DDG.h"
#include "llvm/Analysis/DDGPrinter.h"
#include "llvm/Analysis/DependenceAnalysis.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/VectorBuilder.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/GraphWriter.h"
#include "llvm/Transforms/Utils/Local.h"
#define DEBUG_TYPE "vector-predication"
......@@ -12,54 +21,37 @@ STATISTIC(Transforms, "Number of full-length -> evl vector transformation.");
using namespace llvm;
// Map each instruction to its uses and save all memory writing vector
// predicated instructions found in the basic block.
void VectorPredicationPass::analyseBasicBlock(BasicBlock &BB,
BlockData &BBInfo) {
for (Instruction &I : BB) {
if (isa<PHINode>(I) || I.isTerminator())
continue;
namespace {
cl::opt<bool> PrintDDG("print-ddg", cl::Hidden, cl::init(false),
cl::desc("Create dot file for the DDG"));
} // namespace
SmallPtrSet<Instruction *, 4> IUsers;
for (User *IU : I.users()) {
assert(isa<Instruction>(IU) && "Unexpected behaviour.");
auto *IUInst = cast<Instruction>(IU);
if (IUInst->getParent() != I.getParent())
continue;
if (isa<PHINode>(IUInst) || IUInst->isTerminator())
continue;
IUsers.insert(IUInst);
}
BBInfo.TopologicalGraph.insert({&I, IUsers});
if (auto *CI = dyn_cast<CallInst>(&I)) {
if (auto *CF = CI->getCalledFunction()) {
Intrinsic::ID ID = CF->getIntrinsicID();
// Register all memory writing VP instructions found in the function.
void VectorPredicationPass::analyseFunction(Function &F) {
for (BasicBlock &BB : F) {
for (Instruction &I : BB) {
if (auto *CI = dyn_cast<CallInst>(&I)) {
Intrinsic::ID ID = CI->getIntrinsicID();
if (ID == Intrinsic::vp_store || ID == Intrinsic::vp_scatter) {
BBInfo.MemoryWritingVPInstructions.push_back(&I);
MemoryWritingVPInstructions.push_back(&I);
}
}
}
}
}
static void findCandidateVectorOperation(BasicBlock &BB, Value *Op, Value *Mask,
Value *EVL,
static void findCandidateVectorOperation(Value *Op, Value *Mask, Value *EVL,
InstToMaskEVLMap &VecOpsToTransform) {
auto *OpInst = dyn_cast<Instruction>(Op);
if (!OpInst)
return;
if (OpInst->getParent() != &BB)
return;
Intrinsic::ID VPID = VPIntrinsic::getForOpcode(OpInst->getOpcode());
if (VPID == Intrinsic::not_intrinsic)
return;
// If the instruction is already present in the map, it means it was already
// visited starting from a previous memory wrtiting vp operation.
// visited starting from a previous memory writing vp operation.
if (!VecOpsToTransform
.insert(std::make_pair(OpInst, std::make_pair(Mask, EVL)))
.second) {
......@@ -87,114 +79,54 @@ static void findCandidateVectorOperation(BasicBlock &BB, Value *Op, Value *Mask,
switch (VPID) {
default:
for (auto *OpVal : OpInst->operand_values())
findCandidateVectorOperation(BB, OpVal, Mask, EVL, VecOpsToTransform);
findCandidateVectorOperation(OpVal, Mask, EVL, VecOpsToTransform);
break;
case Intrinsic::vp_select: {
Value *Cond = OpInst->getOperand(0);
if (Cond->getType()->isVectorTy())
findCandidateVectorOperation(BB, Cond, nullptr, EVL, VecOpsToTransform);
findCandidateVectorOperation(Cond, nullptr, EVL, VecOpsToTransform);
// TODO: if the condition argument is a vector, we could backpropagate it
// as mask for the true branch and its negation as mask for the false one.
// WARNING: when creating the negation of the condition, we must ensure it
// dominates all uses.
findCandidateVectorOperation(BB, OpInst->getOperand(1), nullptr, EVL,
findCandidateVectorOperation(OpInst->getOperand(1), nullptr, EVL,
VecOpsToTransform);
findCandidateVectorOperation(BB, OpInst->getOperand(2), nullptr, EVL,
findCandidateVectorOperation(OpInst->getOperand(2), nullptr, EVL,
VecOpsToTransform);
break;
}
}
}
// For each vector predicated memory writing operation of the basic block, go
// back to the stored vector defining instruction and verify it is a vector
// operation. Add it to the list of instructions to be transformed into vector
// predicated ones, then recursively repeat the process for its vector
// arguments.
void VectorPredicationPass::findCandidateVectorOperations(BasicBlock &BB,
BlockData &BBInfo) {
if (BBInfo.MemoryWritingVPInstructions.empty())
// For each VP memory writing operation, go back to the stored vector defining
// instruction and verify it is a vector operation. Add it to the list of
// instructions to be transformed into vector predicated ones, then recursively
// repeat the process for its vector arguments.
void VectorPredicationPass::findCandidateVectorOperations() {
if (MemoryWritingVPInstructions.empty())
return;
for (Instruction *I : BBInfo.MemoryWritingVPInstructions) {
assert(I->getParent() == &BB && "This is not the right basic block");
for (Instruction *I : MemoryWritingVPInstructions) {
auto *VPI = cast<VPIntrinsic>(I);
Value *StoredOperand = VPI->getMemoryDataParam();
Value *MaskOperand = VPI->getMaskParam();
Value *EVLOperand = VPI->getVectorLengthParam();
// First, visit the mask operand (assigning an allones mask to this branch)
// and only then visit the stored operand.
findCandidateVectorOperation(BB, MaskOperand, nullptr, EVLOperand,
BBInfo.VecOpsToTransform);
findCandidateVectorOperation(BB, StoredOperand, MaskOperand, EVLOperand,
BBInfo.VecOpsToTransform);
}
}
// Add the candidates as users of the mask and evl linked to each of them.
void VectorPredicationPass::addNewUsersToMasksAndEVLs(BasicBlock &BB,
BlockData &BBInfo) {
if (BBInfo.VecOpsToTransform.empty())
return;
for (auto [K, V] : BBInfo.VecOpsToTransform) {
if (auto *MaskInst = dyn_cast_if_present<Instruction>(V.first))
BBInfo.TopologicalGraph[MaskInst].insert(K);
if (auto *EVLInst = dyn_cast<Instruction>(V.second))
BBInfo.TopologicalGraph[EVLInst].insert(K);
}
}
// Topologically sort, preserving as much as possible the original order.
void VectorPredicationPass::buildNewBasicBlockSchedule(BasicBlock &BB,
BlockData &BBInfo) {
if (BBInfo.VecOpsToTransform.empty())
return;
while (!BBInfo.TopologicalGraph.empty()) {
Instruction *Inst = nullptr;
for (auto B = BBInfo.TopologicalGraph.rbegin(),
E = BBInfo.TopologicalGraph.rend();
B != E; B++) {
if (B->second.empty()) {
Inst = B->first;
break;
}
}
assert(Inst && "Failed to empty topological graph!");
BBInfo.NewBBReverseOrder.push_back(Inst);
BBInfo.TopologicalGraph.erase(Inst);
for (auto B = BBInfo.TopologicalGraph.begin(),
E = BBInfo.TopologicalGraph.end();
B != E; B++) {
B->second.erase(Inst);
}
}
}
// Modify the basic block based on the topological order generated.
void VectorPredicationPass::emitNewBasicBlockSchedule(BasicBlock &BB,
BlockData &BBInfo) {
if (BBInfo.VecOpsToTransform.empty())
return;
Instruction *InsertPoint = BB.getTerminator();
for (Instruction *I : BBInfo.NewBBReverseOrder) {
I->moveBefore(InsertPoint);
InsertPoint = I;
findCandidateVectorOperation(MaskOperand, nullptr, EVLOperand,
VecOpsToTransform);
findCandidateVectorOperation(StoredOperand, MaskOperand, EVLOperand,
VecOpsToTransform);
}
}
// Transform candidates to vector predicated instructions.
void VectorPredicationPass::transformCandidateVectorOperations(
BasicBlock &BB, BlockData &BBInfo) {
if (BBInfo.VecOpsToTransform.empty())
void VectorPredicationPass::transformCandidateVectorOperations() {
if (VecOpsToTransform.empty())
return;
for (auto [I, P] : BBInfo.VecOpsToTransform) {
for (auto [I, P] : VecOpsToTransform) {
Value *Mask, *EVL;
std::tie(Mask, EVL) = P;
......@@ -251,6 +183,8 @@ void VectorPredicationPass::removeOldInstructions() {
if (isInstructionTriviallyDead(I))
I->eraseFromParent();
}
OldInstructionsToRemove.clear();
}
PreservedAnalyses VectorPredicationPass::run(Function &F,
......@@ -258,20 +192,34 @@ PreservedAnalyses VectorPredicationPass::run(Function &F,
assert(OldInstructionsToRemove.empty() &&
"Map should be cleared at the end of each run of the pass.");
for (BasicBlock &BB : F) {
BlockData BBInfo;
analyseFunction(F);
findCandidateVectorOperations();
// TODO: before transformation, create DDG and use it to check if adding a new
// edge creates a cycle.
transformCandidateVectorOperations();
removeOldInstructions();
analyseBasicBlock(BB, BBInfo);
findCandidateVectorOperations(BB, BBInfo);
addNewUsersToMasksAndEVLs(BB, BBInfo);
buildNewBasicBlockSchedule(BB, BBInfo);
emitNewBasicBlockSchedule(BB, BBInfo);
transformCandidateVectorOperations(BB, BBInfo);
// Retrieve DependenceInfo for the function.
TargetLibraryInfoImpl TLII;
TargetLibraryInfo TLI(TLII);
AAResults AA(TLI);
AssumptionCache AC(F);
DominatorTree DT(F);
LoopInfo LI(DT);
ScalarEvolution SE(F, TLI, AC, DT, LI);
DependenceInfo DI(&F, &AA, &SE, &LI);
DataDependenceGraph DDG(F, DI);
if (PrintDDG) {
std::string Filename = ("ddg." + F.getName() + ".dot").str();
std::error_code EC;
raw_fd_ostream File(Filename, EC, sys::fs::OF_Text);
if (!EC)
WriteGraph(File, &DDG);
else
errs() << "Error opening file for writing!\n";
}
removeOldInstructions();
OldInstructionsToRemove.clear();
// TODO: think about which analysis are preserved.
return PreservedAnalyses::none();
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment