Newer
Older
} else if (HasUse && HasDef &&
vrm.getStackSlot(NewVReg) == VirtRegMap::NO_STACK_SLOT) {
// If this interval hasn't been assigned a stack slot (because earlier
// def is a deleted remat def), do it now.
assert(Slot != VirtRegMap::NO_STACK_SLOT);
vrm.assignVirt2StackSlot(NewVReg, Slot);
// Re-matting an instruction with virtual register use. Add the
// register as an implicit use on the use MI.
if (DefIsReMat && ImpUse)
MI->addOperand(MachineOperand::CreateReg(ImpUse, false, true));
// create a new register interval for this spill / remat.
LiveInterval &nI = getOrCreateInterval(NewVReg);
if (CreatedNewVReg) {
NewLIs.push_back(&nI);
MBBVRegsMap.insert(std::make_pair(MI->getParent()->getNumber(), NewVReg));
if (TrySplit)
vrm.setIsSplitFromReg(NewVReg, li.reg);
}
if (CreatedNewVReg) {
LiveRange LR(getLoadIndex(index), getUseIndex(index)+1,
nI.getNextValue(~0U, 0, VNInfoAllocator));
DOUT << " +" << LR;
nI.addRange(LR);
} else {
// Extend the split live interval to this def / use.
unsigned End = getUseIndex(index)+1;
LiveRange LR(nI.ranges[nI.ranges.size()-1].end, End,
nI.getValNumInfo(nI.getNumValNums()-1));
DOUT << " +" << LR;
nI.addRange(LR);
}
}
if (HasDef) {
LiveRange LR(getDefIndex(index), getStoreIndex(index),
nI.getNextValue(~0U, 0, VNInfoAllocator));
DOUT << " +" << LR;
nI.addRange(LR);
}
Evan Cheng
committed
return CanFold;
bool LiveIntervals::anyKillInMBBAfterIdx(const LiveInterval &li,
const VNInfo *VNI,
MachineBasicBlock *MBB, unsigned Idx) const {
for (unsigned j = 0, ee = VNI->kills.size(); j != ee; ++j) {
unsigned KillIdx = VNI->kills[j];
if (KillIdx > Idx && KillIdx < End)
return true;
static const VNInfo *findDefinedVNInfo(const LiveInterval &li, unsigned DefIdx) {
const VNInfo *VNI = NULL;
for (LiveInterval::const_vni_iterator i = li.vni_begin(),
e = li.vni_end(); i != e; ++i)
if ((*i)->def == DefIdx) {
VNI = *i;
break;
}
return VNI;
}
/// RewriteInfo - Keep track of machine instrs that will be rewritten
/// during spilling.
struct RewriteInfo {
unsigned Index;
MachineInstr *MI;
bool HasUse;
bool HasDef;
RewriteInfo(unsigned i, MachineInstr *mi, bool u, bool d)
: Index(i), MI(mi), HasUse(u), HasDef(d) {}
};
struct RewriteInfoCompare {
bool operator()(const RewriteInfo &LHS, const RewriteInfo &RHS) const {
return LHS.Index < RHS.Index;
}
};
rewriteInstructionsForSpills(const LiveInterval &li, bool TrySplit,
MachineInstr *ReMatOrigDefMI, MachineInstr *ReMatDefMI,
unsigned Slot, int LdSlot,
bool isLoad, bool isLoadSS, bool DefIsReMat, bool CanDelete,
Evan Cheng
committed
VirtRegMap &vrm,
const TargetRegisterClass* rc,
SmallVector<int, 4> &ReMatIds,
std::map<unsigned, std::vector<SRInfo> > &SpillIdxes,
BitVector &RestoreMBBs,
std::map<unsigned, std::vector<SRInfo> > &RestoreIdxes,
std::map<unsigned,unsigned> &MBBVRegsMap,
Evan Cheng
committed
bool AllCanFold = true;
unsigned start = getBaseIndex(I->start);
unsigned end = getBaseIndex(I->end-1) + InstrSlots::NUM;
// First collect all the def / use in this live range that will be rewritten.
// Make sure they are sorted according instruction index.
std::vector<RewriteInfo> RewriteMIs;
Evan Cheng
committed
for (MachineRegisterInfo::reg_iterator ri = mri_->reg_begin(li.reg),
re = mri_->reg_end(); ri != re; ) {
MachineInstr *MI = &(*ri);
MachineOperand &O = ri.getOperand();
++ri;
unsigned index = getInstructionIndex(MI);
if (index < start || index >= end)
continue;
RewriteMIs.push_back(RewriteInfo(index, MI, O.isUse(), O.isDef()));
}
std::sort(RewriteMIs.begin(), RewriteMIs.end(), RewriteInfoCompare());
unsigned ImpUse = DefIsReMat ? getReMatImplicitUse(li, ReMatDefMI) : 0;
// Now rewrite the defs and uses.
for (unsigned i = 0, e = RewriteMIs.size(); i != e; ) {
RewriteInfo &rwi = RewriteMIs[i];
++i;
unsigned index = rwi.Index;
bool MIHasUse = rwi.HasUse;
bool MIHasDef = rwi.HasDef;
MachineInstr *MI = rwi.MI;
// If MI def and/or use the same register multiple times, then there
// are multiple entries.
while (i != e && RewriteMIs[i].MI == MI) {
assert(RewriteMIs[i].Index == index);
bool isUse = RewriteMIs[i].HasUse;
if (isUse) ++NumUses;
MIHasUse |= isUse;
MIHasDef |= RewriteMIs[i].HasDef;
++i;
}
if (ImpUse && MI != ReMatDefMI) {
// Re-matting an instruction with virtual register use. Update the
// register interval's spill weight.
unsigned loopDepth = loopInfo->getLoopDepth(MI->getParent());
LiveInterval &ImpLi = getInterval(ImpUse);
ImpLi.weight +=
getSpillWeight(false, true, loopDepth) * NumUses / ImpLi.getSize();
}
unsigned MBBId = MBB->getNumber();
Evan Cheng
committed
unsigned ThisVReg = 0;
std::map<unsigned,unsigned>::const_iterator NVI = MBBVRegsMap.find(MBBId);
if (NVI != MBBVRegsMap.end()) {
Evan Cheng
committed
ThisVReg = NVI->second;
// One common case:
// x = use
// ...
// ...
// def = ...
// = use
// It's better to start a new interval to avoid artifically
// extend the new interval.
if (MIHasDef && !MIHasUse) {
MBBVRegsMap.erase(MBB->getNumber());
Evan Cheng
committed
ThisVReg = 0;
}
}
Evan Cheng
committed
bool IsNew = ThisVReg == 0;
if (IsNew) {
// This ends the previous live interval. If all of its def / use
// can be folded, give it a low spill weight.
if (NewVReg && TrySplit && AllCanFold) {
LiveInterval &nI = getOrCreateInterval(NewVReg);
nI.weight /= 10.0F;
}
AllCanFold = true;
}
NewVReg = ThisVReg;
Evan Cheng
committed
bool CanFold = rewriteInstructionForSpills(li, I->valno, TrySplit,
Evan Cheng
committed
index, end, MI, ReMatOrigDefMI, ReMatDefMI,
Slot, LdSlot, isLoad, isLoadSS, DefIsReMat,
Evan Cheng
committed
CanDelete, vrm, rc, ReMatIds, loopInfo, NewVReg,
ImpUse, HasDef, HasUse, MBBVRegsMap, NewLIs);
Evan Cheng
committed
AllCanFold &= CanFold;
// Update weight of spill interval.
LiveInterval &nI = getOrCreateInterval(NewVReg);
// The spill weight is now infinity as it cannot be spilled again.
nI.weight = HUGE_VALF;
continue;
}
// Keep track of the last def and first use in each MBB.
if (HasDef) {
if (MI != ReMatOrigDefMI || !CanDelete) {
bool HasKill = false;
if (!HasUse)
HasKill = anyKillInMBBAfterIdx(li, I->valno, MBB, getDefIndex(index));
else {
// If this is a two-address code, then this index starts a new VNInfo.
const VNInfo *VNI = findDefinedVNInfo(li, getDefIndex(index));
if (VNI)
HasKill = anyKillInMBBAfterIdx(li, VNI, MBB, getDefIndex(index));
std::map<unsigned, std::vector<SRInfo> >::iterator SII =
SpillIdxes.find(MBBId);
if (!HasKill) {
if (SII == SpillIdxes.end()) {
std::vector<SRInfo> S;
S.push_back(SRInfo(index, NewVReg, true));
SpillIdxes.insert(std::make_pair(MBBId, S));
} else if (SII->second.back().vreg != NewVReg) {
SII->second.push_back(SRInfo(index, NewVReg, true));
} else if ((int)index > SII->second.back().index) {
// If there is an earlier def and this is a two-address
// instruction, then it's not possible to fold the store (which
// would also fold the load).
SRInfo &Info = SII->second.back();
Info.index = index;
Info.canFold = !HasUse;
SpillMBBs.set(MBBId);
} else if (SII != SpillIdxes.end() &&
SII->second.back().vreg == NewVReg &&
(int)index > SII->second.back().index) {
// There is an earlier def that's not killed (must be two-address).
// The spill is no longer needed.
SII->second.pop_back();
if (SII->second.empty()) {
SpillIdxes.erase(MBBId);
SpillMBBs.reset(MBBId);
}
if (HasUse) {
std::map<unsigned, std::vector<SRInfo> >::iterator SII =
SpillIdxes.find(MBBId);
if (SII != SpillIdxes.end() &&
SII->second.back().vreg == NewVReg &&
(int)index > SII->second.back().index)
// Use(s) following the last def, it's not safe to fold the spill.
SII->second.back().canFold = false;
std::map<unsigned, std::vector<SRInfo> >::iterator RII =
RestoreIdxes.find(MBBId);
if (RII != RestoreIdxes.end() && RII->second.back().vreg == NewVReg)
// If we are splitting live intervals, only fold if it's the first
// use and there isn't another use later in the MBB.
RII->second.back().canFold = false;
else if (IsNew) {
// Only need a reload if there isn't an earlier def / use.
if (RII == RestoreIdxes.end()) {
std::vector<SRInfo> Infos;
Infos.push_back(SRInfo(index, NewVReg, true));
RestoreIdxes.insert(std::make_pair(MBBId, Infos));
} else {
RII->second.push_back(SRInfo(index, NewVReg, true));
}
RestoreMBBs.set(MBBId);
}
// Update spill weight.
unsigned loopDepth = loopInfo->getLoopDepth(MBB);
nI.weight += getSpillWeight(HasDef, HasUse, loopDepth);
Evan Cheng
committed
if (NewVReg && TrySplit && AllCanFold) {
// If all of its def / use can be folded, give it a low spill weight.
LiveInterval &nI = getOrCreateInterval(NewVReg);
nI.weight /= 10.0F;
}
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
bool LiveIntervals::alsoFoldARestore(int Id, int index, unsigned vr,
BitVector &RestoreMBBs,
std::map<unsigned,std::vector<SRInfo> > &RestoreIdxes) {
if (!RestoreMBBs[Id])
return false;
std::vector<SRInfo> &Restores = RestoreIdxes[Id];
for (unsigned i = 0, e = Restores.size(); i != e; ++i)
if (Restores[i].index == index &&
Restores[i].vreg == vr &&
Restores[i].canFold)
return true;
return false;
}
void LiveIntervals::eraseRestoreInfo(int Id, int index, unsigned vr,
BitVector &RestoreMBBs,
std::map<unsigned,std::vector<SRInfo> > &RestoreIdxes) {
if (!RestoreMBBs[Id])
return;
std::vector<SRInfo> &Restores = RestoreIdxes[Id];
for (unsigned i = 0, e = Restores.size(); i != e; ++i)
if (Restores[i].index == index && Restores[i].vreg)
Restores[i].index = -1;
}
const MachineLoopInfo *loopInfo, VirtRegMap &vrm) {
// Since this is called after the analysis is done we don't know if
// LiveVariables is available
lv_ = getAnalysisToUpdate<LiveVariables>();
assert(li.weight != HUGE_VALF &&
"attempt to spill already spilled interval!");
DOUT << "\t\t\t\tadding intervals for spills for interval: ";
// Each bit specify whether it a spill is required in the MBB.
BitVector SpillMBBs(mf_->getNumBlockIDs());
std::map<unsigned, std::vector<SRInfo> > SpillIdxes;
BitVector RestoreMBBs(mf_->getNumBlockIDs());
std::map<unsigned, std::vector<SRInfo> > RestoreIdxes;
std::map<unsigned,unsigned> MBBVRegsMap;
Evan Cheng
committed
const TargetRegisterClass* rc = mri_->getRegClass(li.reg);
unsigned NumValNums = li.getNumValNums();
SmallVector<MachineInstr*, 4> ReMatDefs;
ReMatDefs.resize(NumValNums, NULL);
SmallVector<MachineInstr*, 4> ReMatOrigDefs;
ReMatOrigDefs.resize(NumValNums, NULL);
SmallVector<int, 4> ReMatIds;
ReMatIds.resize(NumValNums, VirtRegMap::MAX_STACK_SLOT);
BitVector ReMatDelete(NumValNums);
unsigned Slot = VirtRegMap::MAX_STACK_SLOT;
// Spilling a split live interval. It cannot be split any further. Also,
// it's also guaranteed to be a single val# / range interval.
if (vrm.getPreSplitReg(li.reg)) {
vrm.setIsSplitFromReg(li.reg, 0);
// Unset the split kill marker on the last use.
unsigned KillIdx = vrm.getKillPoint(li.reg);
if (KillIdx) {
MachineInstr *KillMI = getInstructionFromIndex(KillIdx);
assert(KillMI && "Last use disappeared?");
int KillOp = KillMI->findRegisterUseOperandIdx(li.reg, true);
assert(KillOp != -1 && "Last use disappeared?");
KillMI->getOperand(KillOp).setIsKill(false);
Evan Cheng
committed
vrm.removeKillPoint(li.reg);
bool DefIsReMat = vrm.isReMaterialized(li.reg);
Slot = vrm.getStackSlot(li.reg);
assert(Slot != VirtRegMap::MAX_STACK_SLOT);
MachineInstr *ReMatDefMI = DefIsReMat ?
vrm.getReMaterializedMI(li.reg) : NULL;
int LdSlot = 0;
bool isLoadSS = DefIsReMat && tii_->isLoadFromStackSlot(ReMatDefMI, LdSlot);
bool isLoad = isLoadSS ||
(DefIsReMat && (ReMatDefMI->getDesc().isSimpleLoad()));
bool IsFirstRange = true;
for (LiveInterval::Ranges::const_iterator
I = li.ranges.begin(), E = li.ranges.end(); I != E; ++I) {
// If this is a split live interval with multiple ranges, it means there
// are two-address instructions that re-defined the value. Only the
// first def can be rematerialized!
if (IsFirstRange) {
// Note ReMatOrigDefMI has already been deleted.
rewriteInstructionsForSpills(li, false, I, NULL, ReMatDefMI,
Slot, LdSlot, isLoad, isLoadSS, DefIsReMat,
Evan Cheng
committed
false, vrm, rc, ReMatIds, loopInfo,
SpillMBBs, SpillIdxes, RestoreMBBs, RestoreIdxes,
MBBVRegsMap, NewLIs);
} else {
rewriteInstructionsForSpills(li, false, I, NULL, 0,
Slot, 0, false, false, false,
Evan Cheng
committed
false, vrm, rc, ReMatIds, loopInfo,
SpillMBBs, SpillIdxes, RestoreMBBs, RestoreIdxes,
MBBVRegsMap, NewLIs);
}
IsFirstRange = false;
}
return NewLIs;
}
bool TrySplit = SplitAtBB && !intervalIsInOneMBB(li);
if (SplitLimit != -1 && (int)numSplits >= SplitLimit)
TrySplit = false;
if (TrySplit)
++numSplits;
bool NeedStackSlot = false;
for (LiveInterval::const_vni_iterator i = li.vni_begin(), e = li.vni_end();
i != e; ++i) {
const VNInfo *VNI = *i;
unsigned VN = VNI->id;
unsigned DefIdx = VNI->def;
if (DefIdx == ~1U)
continue; // Dead val#.
// Is the def for the val# rematerializable?
MachineInstr *ReMatDefMI = (DefIdx == ~0u)
? 0 : getInstructionFromIndex(DefIdx);
Evan Cheng
committed
bool dummy;
if (ReMatDefMI && isReMaterializable(li, VNI, ReMatDefMI, dummy)) {
// Original def may be modified so we have to make a copy here. vrm must
// delete these!
ReMatDefs[VN] = ReMatDefMI = ReMatDefMI->clone();
if (VNI->hasPHIKill) {
// A kill is a phi node, not all of its uses can be rematerialized.
CanDelete = false;
// Need a stack slot if there is any live range where uses cannot be
// rematerialized.
NeedStackSlot = true;
}
if (CanDelete)
ReMatDelete.set(VN);
} else {
// Need a stack slot if there is any live range where uses cannot be
// rematerialized.
NeedStackSlot = true;
}
}
// One stack slot per live interval.
if (NeedStackSlot && vrm.getPreSplitReg(li.reg) == 0)
Slot = vrm.assignVirt2StackSlot(li.reg);
// Create new intervals and rewrite defs and uses.
for (LiveInterval::Ranges::const_iterator
I = li.ranges.begin(), E = li.ranges.end(); I != E; ++I) {
MachineInstr *ReMatDefMI = ReMatDefs[I->valno->id];
MachineInstr *ReMatOrigDefMI = ReMatOrigDefs[I->valno->id];
bool DefIsReMat = ReMatDefMI != NULL;
bool CanDelete = ReMatDelete[I->valno->id];
int LdSlot = 0;
bool isLoadSS = DefIsReMat && tii_->isLoadFromStackSlot(ReMatDefMI, LdSlot);
(DefIsReMat && ReMatDefMI->getDesc().isSimpleLoad());
rewriteInstructionsForSpills(li, TrySplit, I, ReMatOrigDefMI, ReMatDefMI,
Slot, LdSlot, isLoad, isLoadSS, DefIsReMat,
Evan Cheng
committed
CanDelete, vrm, rc, ReMatIds, loopInfo,
SpillMBBs, SpillIdxes, RestoreMBBs, RestoreIdxes,
MBBVRegsMap, NewLIs);
// Insert spills / restores if we are splitting.
if (!TrySplit)
return NewLIs;
SmallPtrSet<LiveInterval*, 4> AddedKill;
SmallVector<unsigned, 2> Ops;
if (NeedStackSlot) {
int Id = SpillMBBs.find_first();
while (Id != -1) {
std::vector<SRInfo> &spills = SpillIdxes[Id];
for (unsigned i = 0, e = spills.size(); i != e; ++i) {
int index = spills[i].index;
unsigned VReg = spills[i].vreg;
LiveInterval &nI = getOrCreateInterval(VReg);
bool isReMat = vrm.isReMaterialized(VReg);
MachineInstr *MI = getInstructionFromIndex(index);
bool CanFold = false;
bool FoundUse = false;
Ops.clear();
Evan Cheng
committed
if (spills[i].canFold) {
CanFold = true;
for (unsigned j = 0, ee = MI->getNumOperands(); j != ee; ++j) {
MachineOperand &MO = MI->getOperand(j);
if (!MO.isRegister() || MO.getReg() != VReg)
continue;
Ops.push_back(j);
if (MO.isDef())
Evan Cheng
committed
continue;
if (isReMat ||
(!FoundUse && !alsoFoldARestore(Id, index, VReg,
RestoreMBBs, RestoreIdxes))) {
// MI has two-address uses of the same register. If the use
// isn't the first and only use in the BB, then we can't fold
// it. FIXME: Move this to rewriteInstructionsForSpills.
CanFold = false;
Evan Cheng
committed
break;
}
FoundUse = true;
}
}
// Fold the store into the def if possible.
Evan Cheng
committed
bool Folded = false;
if (CanFold && !Ops.empty()) {
if (tryFoldMemoryOperand(MI, vrm, NULL, index, Ops, true, Slot,VReg)){
Evan Cheng
committed
Folded = true;
// Also folded uses, do not issue a load.
eraseRestoreInfo(Id, index, VReg, RestoreMBBs, RestoreIdxes);
nI.removeRange(getLoadIndex(index), getUseIndex(index)+1);
}
nI.removeRange(getDefIndex(index), getStoreIndex(index));
Evan Cheng
committed
}
// Else tell the spiller to issue a spill.
if (!Folded) {
LiveRange *LR = &nI.ranges[nI.ranges.size()-1];
bool isKill = LR->end == getStoreIndex(index);
vrm.addSpillPoint(VReg, isKill, MI);
if (isKill)
AddedKill.insert(&nI);
}
Id = SpillMBBs.find_next(Id);
}
int Id = RestoreMBBs.find_first();
while (Id != -1) {
std::vector<SRInfo> &restores = RestoreIdxes[Id];
for (unsigned i = 0, e = restores.size(); i != e; ++i) {
int index = restores[i].index;
if (index == -1)
continue;
unsigned VReg = restores[i].vreg;
LiveInterval &nI = getOrCreateInterval(VReg);
MachineInstr *MI = getInstructionFromIndex(index);
bool CanFold = false;
Ops.clear();
Evan Cheng
committed
if (restores[i].canFold) {
CanFold = true;
for (unsigned j = 0, ee = MI->getNumOperands(); j != ee; ++j) {
MachineOperand &MO = MI->getOperand(j);
if (!MO.isRegister() || MO.getReg() != VReg)
continue;
if (MO.isDef()) {
// If this restore were to be folded, it would have been folded
// already.
CanFold = false;
Ops.push_back(j);
// Fold the load into the use if possible.
Evan Cheng
committed
bool Folded = false;
if (CanFold && !Ops.empty()) {
if (!vrm.isReMaterialized(VReg))
Folded = tryFoldMemoryOperand(MI, vrm, NULL,index,Ops,true,Slot,VReg);
else {
MachineInstr *ReMatDefMI = vrm.getReMaterializedMI(VReg);
int LdSlot = 0;
bool isLoadSS = tii_->isLoadFromStackSlot(ReMatDefMI, LdSlot);
// If the rematerializable def is a load, also try to fold it.
if (isLoadSS || ReMatDefMI->getDesc().isSimpleLoad())
Folded = tryFoldMemoryOperand(MI, vrm, ReMatDefMI, index,
Ops, isLoadSS, LdSlot, VReg);
Evan Cheng
committed
unsigned ImpUse = getReMatImplicitUse(li, ReMatDefMI);
if (ImpUse) {
// Re-matting an instruction with virtual register use. Add the
// register as an implicit use on the use MI and update the register
// interval's spill weight.
unsigned loopDepth = loopInfo->getLoopDepth(MI->getParent());
LiveInterval &ImpLi = getInterval(ImpUse);
ImpLi.weight +=
getSpillWeight(false, true, loopDepth) / ImpLi.getSize();
Evan Cheng
committed
MI->addOperand(MachineOperand::CreateReg(ImpUse, false, true));
}
}
// If folding is not possible / failed, then tell the spiller to issue a
// load / rematerialization for us.
if (Folded)
nI.removeRange(getLoadIndex(index), getUseIndex(index)+1);
vrm.addRestorePoint(VReg, MI);
Id = RestoreMBBs.find_next(Id);
// Finalize intervals: add kills, finalize spill weights, and filter out
// dead intervals.
std::vector<LiveInterval*> RetNewLIs;
for (unsigned i = 0, e = NewLIs.size(); i != e; ++i) {
LiveInterval *LI = NewLIs[i];
if (!LI->empty()) {
LI->weight /= LI->getSize();
if (!AddedKill.count(LI)) {
LiveRange *LR = &LI->ranges[LI->ranges.size()-1];
unsigned LastUseIdx = getBaseIndex(LR->end);
MachineInstr *LastUse = getInstructionFromIndex(LastUseIdx);
int UseIdx = LastUse->findRegisterUseOperandIdx(LI->reg);
assert(UseIdx != -1);
Evan Cheng
committed
if (LastUse->getOperand(UseIdx).isImplicit() ||
LastUse->getDesc().getOperandConstraint(UseIdx,TOI::TIED_TO) == -1){
LastUse->getOperand(UseIdx).setIsKill();
vrm.addKillPoint(LI->reg, LastUseIdx);
Evan Cheng
committed
}
RetNewLIs.push_back(LI);
}
}