Newer
Older
//===-- llvm/CodeGen/VirtRegMap.cpp - Virtual Register Map ----------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements the VirtRegMap class.
//
// It also contains implementations of the the Spiller interface, which, given a
// virtual register map and a machine function, eliminates all virtual
// references by replacing them with physical register references - adding spill
// code as necessary.
//
//===----------------------------------------------------------------------===//
#define DEBUG_TYPE "spiller"
#include "VirtRegMap.h"
#include "llvm/Function.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
Evan Cheng
committed
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetInstrInfo.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Compiler.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallSet.h"
Chris Lattner
committed
#include <algorithm>
using namespace llvm;
Evan Cheng
committed
STATISTIC(NumSpills , "Number of register spills");
Evan Cheng
committed
STATISTIC(NumReMats , "Number of re-materialization");
STATISTIC(NumDRM , "Number of re-materializable defs elided");
STATISTIC(NumStores , "Number of stores added");
STATISTIC(NumLoads , "Number of loads added");
STATISTIC(NumReused , "Number of values reused");
STATISTIC(NumDSE , "Number of dead stores elided");
STATISTIC(NumDCE , "Number of copies elided");
STATISTIC(NumDSS , "Number of dead spill slots removed");
STATISTIC(NumCommutes, "Number of instructions commuted");
enum SpillerName { simple, local };
}
static cl::opt<SpillerName>
SpillerOpt("spiller",
cl::desc("Spiller to use: (default: local)"),
cl::Prefix,
cl::values(clEnumVal(simple, "simple spiller"),
clEnumVal(local, "local spiller"),
clEnumValEnd),
cl::init(local));
//===----------------------------------------------------------------------===//
// VirtRegMap implementation
//===----------------------------------------------------------------------===//
Chris Lattner
committed
VirtRegMap::VirtRegMap(MachineFunction &mf)
: TII(*mf.getTarget().getInstrInfo()), MF(mf),
Virt2PhysMap(NO_PHYS_REG), Virt2StackSlotMap(NO_STACK_SLOT),
Virt2SplitKillMap(0), ReMatMap(NULL), ReMatId(MAX_STACK_SLOT+1),
LowSpillSlot(NO_STACK_SLOT), HighSpillSlot(NO_STACK_SLOT) {
SpillSlotToUsesMap.resize(8);
Evan Cheng
committed
ImplicitDefed.resize(MF.getRegInfo().getLastVirtReg()+1-
TargetRegisterInfo::FirstVirtualRegister);
Chris Lattner
committed
grow();
}
void VirtRegMap::grow() {
unsigned LastVirtReg = MF.getRegInfo().getLastVirtReg();
Virt2PhysMap.grow(LastVirtReg);
Virt2StackSlotMap.grow(LastVirtReg);
Virt2ReMatIdMap.grow(LastVirtReg);
Evan Cheng
committed
Virt2SplitKillMap.grow(LastVirtReg);
ReMatMap.grow(LastVirtReg);
Evan Cheng
committed
ImplicitDefed.resize(LastVirtReg-TargetRegisterInfo::FirstVirtualRegister+1);
}
int VirtRegMap::assignVirt2StackSlot(unsigned virtReg) {
assert(TargetRegisterInfo::isVirtualRegister(virtReg));
Chris Lattner
committed
assert(Virt2StackSlotMap[virtReg] == NO_STACK_SLOT &&
"attempt to assign stack slot to already spilled register");
const TargetRegisterClass* RC = MF.getRegInfo().getRegClass(virtReg);
int SS = MF.getFrameInfo()->CreateStackObject(RC->getSize(),
RC->getAlignment());
if (LowSpillSlot == NO_STACK_SLOT)
LowSpillSlot = SS;
if (HighSpillSlot == NO_STACK_SLOT || SS > HighSpillSlot)
HighSpillSlot = SS;
unsigned Idx = SS-LowSpillSlot;
while (Idx >= SpillSlotToUsesMap.size())
SpillSlotToUsesMap.resize(SpillSlotToUsesMap.size()*2);
Virt2StackSlotMap[virtReg] = SS;
++NumSpills;
}
void VirtRegMap::assignVirt2StackSlot(unsigned virtReg, int SS) {
assert(TargetRegisterInfo::isVirtualRegister(virtReg));
Chris Lattner
committed
assert(Virt2StackSlotMap[virtReg] == NO_STACK_SLOT &&
"attempt to assign stack slot to already spilled register");
assert((SS >= 0 ||
(SS >= MF.getFrameInfo()->getObjectIndexBegin())) &&
"illegal fixed frame index");
}
int VirtRegMap::assignVirtReMatId(unsigned virtReg) {
assert(TargetRegisterInfo::isVirtualRegister(virtReg));
assert(Virt2ReMatIdMap[virtReg] == NO_STACK_SLOT &&
"attempt to assign re-mat id to already spilled register");
Virt2ReMatIdMap[virtReg] = ReMatId;
void VirtRegMap::assignVirtReMatId(unsigned virtReg, int id) {
assert(TargetRegisterInfo::isVirtualRegister(virtReg));
assert(Virt2ReMatIdMap[virtReg] == NO_STACK_SLOT &&
"attempt to assign re-mat id to already spilled register");
Virt2ReMatIdMap[virtReg] = id;
}
Evan Cheng
committed
int VirtRegMap::getEmergencySpillSlot(const TargetRegisterClass *RC) {
std::map<const TargetRegisterClass*, int>::iterator I =
EmergencySpillSlots.find(RC);
if (I != EmergencySpillSlots.end())
return I->second;
int SS = MF.getFrameInfo()->CreateStackObject(RC->getSize(),
RC->getAlignment());
if (LowSpillSlot == NO_STACK_SLOT)
LowSpillSlot = SS;
if (HighSpillSlot == NO_STACK_SLOT || SS > HighSpillSlot)
HighSpillSlot = SS;
EmergencySpillSlots[RC] = SS;
Evan Cheng
committed
return SS;
}
void VirtRegMap::addSpillSlotUse(int FI, MachineInstr *MI) {
if (!MF.getFrameInfo()->isFixedObjectIndex(FI)) {
// If FI < LowSpillSlot, this stack reference was produced by
// instruction selection and is not a spill
if (FI >= LowSpillSlot) {
assert(FI >= 0 && "Spill slot index should not be negative!");
assert((unsigned)FI-LowSpillSlot < SpillSlotToUsesMap.size()
&& "Invalid spill slot");
SpillSlotToUsesMap[FI-LowSpillSlot].insert(MI);
}
Chris Lattner
committed
void VirtRegMap::virtFolded(unsigned VirtReg, MachineInstr *OldMI,
MachineInstr *NewMI, ModRef MRInfo) {
Chris Lattner
committed
// Move previous memory references folded to new instruction.
MI2VirtMapTy::iterator IP = MI2VirtMap.lower_bound(NewMI);
for (MI2VirtMapTy::iterator I = MI2VirtMap.lower_bound(OldMI),
Chris Lattner
committed
E = MI2VirtMap.end(); I != E && I->first == OldMI; ) {
MI2VirtMap.insert(IP, std::make_pair(NewMI, I->second));
MI2VirtMap.erase(I++);
}
// add new memory reference
Chris Lattner
committed
MI2VirtMap.insert(IP, std::make_pair(NewMI, std::make_pair(VirtReg, MRInfo)));
}
void VirtRegMap::virtFolded(unsigned VirtReg, MachineInstr *MI, ModRef MRInfo) {
MI2VirtMapTy::iterator IP = MI2VirtMap.lower_bound(MI);
MI2VirtMap.insert(IP, std::make_pair(MI, std::make_pair(VirtReg, MRInfo)));
}
void VirtRegMap::RemoveMachineInstrFromMaps(MachineInstr *MI) {
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (!MO.isFI())
continue;
int FI = MO.getIndex();
if (MF.getFrameInfo()->isFixedObjectIndex(FI))
continue;
// This stack reference was produced by instruction selection and
// is not a spill
if (FI < LowSpillSlot)
continue;
assert((unsigned)FI-LowSpillSlot < SpillSlotToUsesMap.size()
SpillSlotToUsesMap[FI-LowSpillSlot].erase(MI);
}
MI2VirtMap.erase(MI);
SpillPt2VirtMap.erase(MI);
RestorePt2VirtMap.erase(MI);
Evan Cheng
committed
EmergencySpillMap.erase(MI);
Chris Lattner
committed
void VirtRegMap::print(std::ostream &OS) const {
const TargetRegisterInfo* TRI = MF.getTarget().getRegisterInfo();
Chris Lattner
committed
OS << "********** REGISTER MAP **********\n";
for (unsigned i = TargetRegisterInfo::FirstVirtualRegister,
e = MF.getRegInfo().getLastVirtReg(); i <= e; ++i) {
Chris Lattner
committed
if (Virt2PhysMap[i] != (unsigned)VirtRegMap::NO_PHYS_REG)
OS << "[reg" << i << " -> " << TRI->getName(Virt2PhysMap[i])
Bill Wendling
committed
<< "]\n";
}
for (unsigned i = TargetRegisterInfo::FirstVirtualRegister,
e = MF.getRegInfo().getLastVirtReg(); i <= e; ++i)
Chris Lattner
committed
if (Virt2StackSlotMap[i] != VirtRegMap::NO_STACK_SLOT)
OS << "[reg" << i << " -> fi#" << Virt2StackSlotMap[i] << "]\n";
OS << '\n';
}
void VirtRegMap::dump() const {
print(cerr);
//===----------------------------------------------------------------------===//
// Simple Spiller Implementation
//===----------------------------------------------------------------------===//
Alkis Evlogimenos
committed
Spiller::~Spiller() {}
Alkis Evlogimenos
committed
namespace {
struct VISIBILITY_HIDDEN SimpleSpiller : public Spiller {
bool runOnMachineFunction(MachineFunction& mf, VirtRegMap &VRM);
};
}
bool SimpleSpiller::runOnMachineFunction(MachineFunction &MF, VirtRegMap &VRM) {
DOUT << "********** REWRITE MACHINE CODE **********\n";
DOUT << "********** Function: " << MF.getFunction()->getName() << '\n';
Chris Lattner
committed
const TargetMachine &TM = MF.getTarget();
Owen Anderson
committed
const TargetInstrInfo &TII = *TM.getInstrInfo();
const TargetRegisterInfo &TRI = *TM.getRegisterInfo();
Owen Anderson
committed
Chris Lattner
committed
// LoadedRegs - Keep track of which vregs are loaded, so that we only load
// each vreg once (in the case where a spilled vreg is used by multiple
// operands). This is always smaller than the number of operands to the
// current machine instr, so it should be small.
std::vector<unsigned> LoadedRegs;
for (MachineFunction::iterator MBBI = MF.begin(), E = MF.end();
MBBI != E; ++MBBI) {
DOUT << MBBI->getBasicBlock()->getName() << ":\n";
MachineBasicBlock &MBB = *MBBI;
for (MachineBasicBlock::iterator MII = MBB.begin(),
E = MBB.end(); MII != E; ++MII) {
MachineInstr &MI = *MII;
for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI.getOperand(i);
if (MO.isReg() && MO.getReg()) {
if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
unsigned VirtReg = MO.getReg();
unsigned SubIdx = MO.getSubReg();
unsigned PhysReg = VRM.getPhys(VirtReg);
unsigned RReg = SubIdx ? TRI.getSubReg(PhysReg, SubIdx) : PhysReg;
if (!VRM.isAssignedReg(VirtReg)) {
int StackSlot = VRM.getStackSlot(VirtReg);
Chris Lattner
committed
const TargetRegisterClass* RC =
MF.getRegInfo().getRegClass(VirtReg);
if (MO.isUse() &&
std::find(LoadedRegs.begin(), LoadedRegs.end(), VirtReg)
== LoadedRegs.end()) {
Owen Anderson
committed
TII.loadRegFromStackSlot(MBB, &MI, PhysReg, StackSlot, RC);
MachineInstr *LoadMI = prior(MII);
VRM.addSpillSlotUse(StackSlot, LoadMI);
LoadedRegs.push_back(VirtReg);
++NumLoads;
if (MO.isDef()) {
Owen Anderson
committed
TII.storeRegToStackSlot(MBB, next(MII), PhysReg, true,
StackSlot, RC);
MachineInstr *StoreMI = next(MII);
VRM.addSpillSlotUse(StackSlot, StoreMI);
++NumStores;
}
}
MF.getRegInfo().setPhysRegUsed(RReg);
MI.getOperand(i).setReg(RReg);
} else {
MF.getRegInfo().setPhysRegUsed(MO.getReg());
}
}
DOUT << '\t' << MI;
Chris Lattner
committed
LoadedRegs.clear();
}
}
return true;
}
//===----------------------------------------------------------------------===//
// Local Spiller Implementation
//===----------------------------------------------------------------------===//
namespace {
/// LocalSpiller - This spiller does a simple pass over the machine basic
/// block to attempt to keep spills in registers as much as possible for
/// blocks that have low register pressure (the vreg may be spilled due to
/// register pressure in other blocks).
class VISIBILITY_HIDDEN LocalSpiller : public Spiller {
MachineRegisterInfo *RegInfo;
const TargetInstrInfo *TII;
DenseMap<MachineInstr*, unsigned> DistanceMap;
public:
bool runOnMachineFunction(MachineFunction &MF, VirtRegMap &VRM) {
RegInfo = &MF.getRegInfo();
TRI = MF.getTarget().getRegisterInfo();
TII = MF.getTarget().getInstrInfo();
DOUT << "\n**** Local spiller rewriting function '"
<< MF.getFunction()->getName() << "':\n";
DOUT << "**** Machine Instrs (NOTE! Does not include spills and reloads!)"
" ****\n";
for (MachineFunction::iterator MBB = MF.begin(), E = MF.end();
MBB != E; ++MBB)
RewriteMBB(*MBB, VRM);
// Mark unused spill slots.
MachineFrameInfo *MFI = MF.getFrameInfo();
int SS = VRM.getLowSpillSlot();
if (SS != VirtRegMap::NO_STACK_SLOT)
for (int e = VRM.getHighSpillSlot(); SS <= e; ++SS)
if (!VRM.isSpillSlotUsed(SS)) {
MFI->RemoveStackObject(SS);
++NumDSS;
}
return true;
}
private:
void TransferDeadness(MachineBasicBlock *MBB, unsigned CurDist,
unsigned Reg, BitVector &RegKills,
std::vector<MachineOperand*> &KillOps);
bool PrepForUnfoldOpti(MachineBasicBlock &MBB,
MachineBasicBlock::iterator &MII,
std::vector<MachineInstr*> &MaybeDeadStores,
AvailableSpills &Spills, BitVector &RegKills,
std::vector<MachineOperand*> &KillOps,
VirtRegMap &VRM);
Evan Cheng
committed
bool CommuteToFoldReload(MachineBasicBlock &MBB,
MachineBasicBlock::iterator &MII,
unsigned VirtReg, unsigned SrcReg, int SS,
BitVector &RegKills,
std::vector<MachineOperand*> &KillOps,
const TargetRegisterInfo *TRI,
VirtRegMap &VRM);
void SpillRegToStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator &MII,
int Idx, unsigned PhysReg, int StackSlot,
const TargetRegisterClass *RC,
bool isAvailable, MachineInstr *&LastStore,
AvailableSpills &Spills,
SmallSet<MachineInstr*, 4> &ReMatDefs,
BitVector &RegKills,
std::vector<MachineOperand*> &KillOps,
void RewriteMBB(MachineBasicBlock &MBB, VirtRegMap &VRM);
};
}
Chris Lattner
committed
/// AvailableSpills - As the local spiller is scanning and rewriting an MBB from
/// top down, keep track of which spills slots or remat are available in each
/// register.
///
/// Note that not all physregs are created equal here. In particular, some
/// physregs are reloads that we are allowed to clobber or ignore at any time.
/// Other physregs are values that the register allocated program is using that
/// we cannot CHANGE, but we can read if we like. We keep track of this on a
/// per-stack-slot / remat id basis as the low bit in the value of the
/// SpillSlotsAvailable entries. The predicate 'canClobberPhysReg()' checks
/// this bit and addAvailable sets it if.
namespace {
class VISIBILITY_HIDDEN AvailableSpills {
Chris Lattner
committed
const TargetInstrInfo *TII;
// SpillSlotsOrReMatsAvailable - This map keeps track of all of the spilled
// or remat'ed virtual register values that are still available, due to being
// loaded or stored to, but not invalidated yet.
std::map<int, unsigned> SpillSlotsOrReMatsAvailable;
Chris Lattner
committed
// PhysRegsAvailable - This is the inverse of SpillSlotsOrReMatsAvailable,
// indicating which stack slot values are currently held by a physreg. This
// is used to invalidate entries in SpillSlotsOrReMatsAvailable when a
// physreg is modified.
Chris Lattner
committed
std::multimap<unsigned, int> PhysRegsAvailable;
void disallowClobberPhysRegOnly(unsigned PhysReg);
Chris Lattner
committed
void ClobberPhysRegOnly(unsigned PhysReg);
public:
AvailableSpills(const TargetRegisterInfo *tri, const TargetInstrInfo *tii)
: TRI(tri), TII(tii) {
Chris Lattner
committed
}
const TargetRegisterInfo *getRegInfo() const { return TRI; }
/// getSpillSlotOrReMatPhysReg - If the specified stack slot or remat is
/// available in a physical register, return that PhysReg, otherwise
/// return 0.
unsigned getSpillSlotOrReMatPhysReg(int Slot) const {
std::map<int, unsigned>::const_iterator I =
SpillSlotsOrReMatsAvailable.find(Slot);
if (I != SpillSlotsOrReMatsAvailable.end()) {
return I->second >> 1; // Remove the CanClobber bit.
}
Chris Lattner
committed
return 0;
}
/// addAvailable - Mark that the specified stack slot / remat is available in
/// the specified physreg. If CanClobber is true, the physreg can be modified
/// at any time without changing the semantics of the program.
void addAvailable(int SlotOrReMat, MachineInstr *MI, unsigned Reg,
bool CanClobber = true) {
// If this stack slot is thought to be available in some other physreg,
// remove its record.
ModifyStackSlotOrReMat(SlotOrReMat);
PhysRegsAvailable.insert(std::make_pair(Reg, SlotOrReMat));
SpillSlotsOrReMatsAvailable[SlotOrReMat]= (Reg << 1) | (unsigned)CanClobber;
Chris Lattner
committed
if (SlotOrReMat > VirtRegMap::MAX_STACK_SLOT)
DOUT << "Remembering RM#" << SlotOrReMat-VirtRegMap::MAX_STACK_SLOT-1;
DOUT << "Remembering SS#" << SlotOrReMat;
DOUT << " in physreg " << TRI->getName(Reg) << "\n";
Chris Lattner
committed
}
/// canClobberPhysReg - Return true if the spiller is allowed to change the
/// value of the specified stackslot register if it desires. The specified
/// stack slot must be available in a physreg for this query to make sense.
bool canClobberPhysReg(int SlotOrReMat) const {
assert(SpillSlotsOrReMatsAvailable.count(SlotOrReMat) &&
"Value not available!");
return SpillSlotsOrReMatsAvailable.find(SlotOrReMat)->second & 1;
}
/// disallowClobberPhysReg - Unset the CanClobber bit of the specified
/// stackslot register. The register is still available but is no longer
/// allowed to be modifed.
void disallowClobberPhysReg(unsigned PhysReg);
Chris Lattner
committed
/// ClobberPhysReg - This is called when the specified physreg changes
/// value. We use this to invalidate any info about stuff that lives in
Chris Lattner
committed
/// it and any of its aliases.
void ClobberPhysReg(unsigned PhysReg);
/// ModifyStackSlotOrReMat - This method is called when the value in a stack
/// slot changes. This removes information about which register the previous
/// value for this slot lives in (as the previous value is dead now).
void ModifyStackSlotOrReMat(int SlotOrReMat);
Chris Lattner
committed
};
Chris Lattner
committed
/// disallowClobberPhysRegOnly - Unset the CanClobber bit of the specified
/// stackslot register. The register is still available but is no longer
/// allowed to be modifed.
void AvailableSpills::disallowClobberPhysRegOnly(unsigned PhysReg) {
std::multimap<unsigned, int>::iterator I =
PhysRegsAvailable.lower_bound(PhysReg);
while (I != PhysRegsAvailable.end() && I->first == PhysReg) {
int SlotOrReMat = I->second;
assert((SpillSlotsOrReMatsAvailable[SlotOrReMat] >> 1) == PhysReg &&
SpillSlotsOrReMatsAvailable[SlotOrReMat] &= ~1;
DOUT << "PhysReg " << TRI->getName(PhysReg)
<< " copied, it is available for use but can no longer be modified\n";
}
}
/// disallowClobberPhysReg - Unset the CanClobber bit of the specified
/// stackslot register and its aliases. The register and its aliases may
/// still available but is no longer allowed to be modifed.
void AvailableSpills::disallowClobberPhysReg(unsigned PhysReg) {
for (const unsigned *AS = TRI->getAliasSet(PhysReg); *AS; ++AS)
disallowClobberPhysRegOnly(*AS);
disallowClobberPhysRegOnly(PhysReg);
}
Chris Lattner
committed
/// ClobberPhysRegOnly - This is called when the specified physreg changes
/// value. We use this to invalidate any info about stuff we thing lives in it.
void AvailableSpills::ClobberPhysRegOnly(unsigned PhysReg) {
std::multimap<unsigned, int>::iterator I =
PhysRegsAvailable.lower_bound(PhysReg);
while (I != PhysRegsAvailable.end() && I->first == PhysReg) {
int SlotOrReMat = I->second;
PhysRegsAvailable.erase(I++);
assert((SpillSlotsOrReMatsAvailable[SlotOrReMat] >> 1) == PhysReg &&
Chris Lattner
committed
"Bidirectional map mismatch!");
SpillSlotsOrReMatsAvailable.erase(SlotOrReMat);
DOUT << "PhysReg " << TRI->getName(PhysReg)
<< " clobbered, invalidating ";
if (SlotOrReMat > VirtRegMap::MAX_STACK_SLOT)
DOUT << "RM#" << SlotOrReMat-VirtRegMap::MAX_STACK_SLOT-1 << "\n";
DOUT << "SS#" << SlotOrReMat << "\n";
}
}
Chris Lattner
committed
/// ClobberPhysReg - This is called when the specified physreg changes
/// value. We use this to invalidate any info about stuff we thing lives in
/// it and any of its aliases.
void AvailableSpills::ClobberPhysReg(unsigned PhysReg) {
for (const unsigned *AS = TRI->getAliasSet(PhysReg); *AS; ++AS)
Chris Lattner
committed
ClobberPhysRegOnly(*AS);
ClobberPhysRegOnly(PhysReg);
}
/// ModifyStackSlotOrReMat - This method is called when the value in a stack
/// slot changes. This removes information about which register the previous
/// value for this slot lives in (as the previous value is dead now).
void AvailableSpills::ModifyStackSlotOrReMat(int SlotOrReMat) {
std::map<int, unsigned>::iterator It =
SpillSlotsOrReMatsAvailable.find(SlotOrReMat);
if (It == SpillSlotsOrReMatsAvailable.end()) return;
SpillSlotsOrReMatsAvailable.erase(It);
// This register may hold the value of multiple stack slots, only remove this
// stack slot from the set of values the register contains.
std::multimap<unsigned, int>::iterator I = PhysRegsAvailable.lower_bound(Reg);
for (; ; ++I) {
assert(I != PhysRegsAvailable.end() && I->first == Reg &&
"Map inverse broken!");
if (I->second == SlotOrReMat) break;
}
PhysRegsAvailable.erase(I);
}
Evan Cheng
committed
/// InvalidateKills - MI is going to be deleted. If any of its operands are
/// marked kill, then invalidate the information.
static void InvalidateKills(MachineInstr &MI, BitVector &RegKills,
Evan Cheng
committed
std::vector<MachineOperand*> &KillOps,
Evan Cheng
committed
for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI.getOperand(i);
if (!MO.isReg() || !MO.isUse() || !MO.isKill())
Evan Cheng
committed
continue;
unsigned Reg = MO.getReg();
if (TargetRegisterInfo::isVirtualRegister(Reg))
continue;
if (KillRegs)
KillRegs->push_back(Reg);
Evan Cheng
committed
if (KillOps[Reg] == &MO) {
RegKills.reset(Reg);
KillOps[Reg] = NULL;
}
}
}
Evan Cheng
committed
/// InvalidateKill - A MI that defines the specified register is being deleted,
/// invalidate the register kill information.
static void InvalidateKill(unsigned Reg, BitVector &RegKills,
std::vector<MachineOperand*> &KillOps) {
if (RegKills[Reg]) {
Evan Cheng
committed
KillOps[Reg] = NULL;
RegKills.reset(Reg);
}
}
/// InvalidateRegDef - If the def operand of the specified def MI is now dead
/// (since it's spill instruction is removed), mark it isDead. Also checks if
/// the def MI has other definition operands that are not dead. Returns it by
/// reference.
static bool InvalidateRegDef(MachineBasicBlock::iterator I,
MachineInstr &NewDef, unsigned Reg,
bool &HasLiveDef) {
// Due to remat, it's possible this reg isn't being reused. That is,
// the def of this reg (by prev MI) is now dead.
MachineInstr *DefMI = I;
MachineOperand *DefOp = NULL;
for (unsigned i = 0, e = DefMI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = DefMI->getOperand(i);
if (MO.isReg() && MO.isDef()) {
if (MO.getReg() == Reg)
DefOp = &MO;
else if (!MO.isDead())
HasLiveDef = true;
}
}
if (!DefOp)
return false;
bool FoundUse = false, Done = false;
MachineBasicBlock::iterator E = &NewDef;
++I; ++E;
for (; !Done && I != E; ++I) {
MachineInstr *NMI = I;
for (unsigned j = 0, ee = NMI->getNumOperands(); j != ee; ++j) {
MachineOperand &MO = NMI->getOperand(j);
if (!MO.isReg() || MO.getReg() != Reg)
continue;
if (MO.isUse())
FoundUse = true;
Done = true; // Stop after scanning all the operands of this MI.
}
}
if (!FoundUse) {
// Def is dead!
DefOp->setIsDead();
return true;
}
return false;
}
Evan Cheng
committed
/// UpdateKills - Track and update kill info. If a MI reads a register that is
/// marked kill, then it must be due to register reuse. Transfer the kill info
/// over.
static void UpdateKills(MachineInstr &MI, BitVector &RegKills,
Evan Cheng
committed
std::vector<MachineOperand*> &KillOps,
const TargetRegisterInfo* TRI) {
const TargetInstrDesc &TID = MI.getDesc();
Evan Cheng
committed
for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI.getOperand(i);
if (!MO.isReg() || !MO.isUse())
Evan Cheng
committed
continue;
unsigned Reg = MO.getReg();
if (Reg == 0)
continue;
if (RegKills[Reg] && KillOps[Reg]->getParent() != &MI) {
Evan Cheng
committed
// That can't be right. Register is killed but not re-defined and it's
// being reused. Let's fix that.
Evan Cheng
committed
KillOps[Reg] = NULL;
RegKills.reset(Reg);
if (i < TID.getNumOperands() &&
TID.getOperandConstraint(i, TOI::TIED_TO) == -1)
Evan Cheng
committed
// Unless it's a two-address operand, this is the new kill.
MO.setIsKill();
}
if (MO.isKill()) {
RegKills.set(Reg);
KillOps[Reg] = &MO;
}
}
for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
const MachineOperand &MO = MI.getOperand(i);
if (!MO.isReg() || !MO.isDef())
Evan Cheng
committed
continue;
unsigned Reg = MO.getReg();
RegKills.reset(Reg);
KillOps[Reg] = NULL;
Evan Cheng
committed
// It also defines (or partially define) aliases.
for (const unsigned *AS = TRI->getAliasSet(Reg); *AS; ++AS) {
RegKills.reset(*AS);
KillOps[*AS] = NULL;
}
Evan Cheng
committed
}
}
Evan Cheng
committed
/// ReMaterialize - Re-materialize definition for Reg targetting DestReg.
///
static void ReMaterialize(MachineBasicBlock &MBB,
MachineBasicBlock::iterator &MII,
unsigned DestReg, unsigned Reg,
const TargetInstrInfo *TII,
Evan Cheng
committed
const TargetRegisterInfo *TRI,
VirtRegMap &VRM) {
TII->reMaterialize(MBB, MII, DestReg, VRM.getReMaterializedMI(Reg));
Evan Cheng
committed
MachineInstr *NewMI = prior(MII);
for (unsigned i = 0, e = NewMI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = NewMI->getOperand(i);
if (!MO.isReg() || MO.getReg() == 0)
Evan Cheng
committed
continue;
unsigned VirtReg = MO.getReg();
if (TargetRegisterInfo::isPhysicalRegister(VirtReg))
continue;
assert(MO.isUse());
unsigned SubIdx = MO.getSubReg();
unsigned Phys = VRM.getPhys(VirtReg);
assert(Phys);
unsigned RReg = SubIdx ? TRI->getSubReg(Phys, SubIdx) : Phys;
MO.setReg(RReg);
}
++NumReMats;
}
Evan Cheng
committed
// ReusedOp - For each reused operand, we keep track of a bit of information, in
// case we need to rollback upon processing a new operand. See comments below.
namespace {
struct ReusedOp {
// The MachineInstr operand that reused an available value.
unsigned Operand;
// StackSlotOrReMat - The spill slot or remat id of the value being reused.
unsigned StackSlotOrReMat;
// PhysRegReused - The physical register the value was available in.
unsigned PhysRegReused;
// AssignedPhysReg - The physreg that was assigned for use by the reload.
unsigned AssignedPhysReg;
// VirtReg - The virtual register itself.
unsigned VirtReg;
ReusedOp(unsigned o, unsigned ss, unsigned prr, unsigned apr,
unsigned vreg)
: Operand(o), StackSlotOrReMat(ss), PhysRegReused(prr),
AssignedPhysReg(apr), VirtReg(vreg) {}
};
Chris Lattner
committed
/// ReuseInfo - This maintains a collection of ReuseOp's for each operand that
/// is reused instead of reloaded.
class VISIBILITY_HIDDEN ReuseInfo {
Chris Lattner
committed
MachineInstr &MI;
std::vector<ReusedOp> Reuses;
BitVector PhysRegsClobbered;
Chris Lattner
committed
public:
ReuseInfo(MachineInstr &mi, const TargetRegisterInfo *tri) : MI(mi) {
PhysRegsClobbered.resize(tri->getNumRegs());
}
Chris Lattner
committed
bool hasReuses() const {
return !Reuses.empty();
}
/// addReuse - If we choose to reuse a virtual register that is already
/// available instead of reloading it, remember that we did so.
void addReuse(unsigned OpNo, unsigned StackSlotOrReMat,
Chris Lattner
committed
unsigned PhysRegReused, unsigned AssignedPhysReg,
unsigned VirtReg) {
// If the reload is to the assigned register anyway, no undo will be
// required.
if (PhysRegReused == AssignedPhysReg) return;
// Otherwise, remember this.
Reuses.push_back(ReusedOp(OpNo, StackSlotOrReMat, PhysRegReused,
Chris Lattner
committed
AssignedPhysReg, VirtReg));
}
void markClobbered(unsigned PhysReg) {
PhysRegsClobbered.set(PhysReg);
}
bool isClobbered(unsigned PhysReg) const {
return PhysRegsClobbered.test(PhysReg);
}
Chris Lattner
committed
/// GetRegForReload - We are about to emit a reload into PhysReg. If there
/// is some other operand that is using the specified register, either pick
/// a new register to use, or evict the previous reload and use this reg.
unsigned GetRegForReload(unsigned PhysReg, MachineInstr *MI,
AvailableSpills &Spills,
std::vector<MachineInstr*> &MaybeDeadStores,
Evan Cheng
committed
SmallSet<unsigned, 8> &Rejected,
BitVector &RegKills,
std::vector<MachineOperand*> &KillOps,
VirtRegMap &VRM) {
Owen Anderson
committed
const TargetInstrInfo* TII = MI->getParent()->getParent()->getTarget()
.getInstrInfo();
Chris Lattner
committed
if (Reuses.empty()) return PhysReg; // This is most often empty.
for (unsigned ro = 0, e = Reuses.size(); ro != e; ++ro) {
ReusedOp &Op = Reuses[ro];
// If we find some other reuse that was supposed to use this register
// exactly for its reload, we can change this reload to use ITS reload
// register. That is, unless its reload register has already been
// considered and subsequently rejected because it has also been reused
// by another operand.
if (Op.PhysRegReused == PhysReg &&
Rejected.count(Op.AssignedPhysReg) == 0) {
Chris Lattner
committed
// Yup, use the reload register that we didn't use before.
unsigned NewReg = Op.AssignedPhysReg;
Rejected.insert(PhysReg);
Evan Cheng
committed
return GetRegForReload(NewReg, MI, Spills, MaybeDeadStores, Rejected,
RegKills, KillOps, VRM);
Chris Lattner
committed
} else {
// Otherwise, we might also have a problem if a previously reused
// value aliases the new register. If so, codegen the previous reload
// and use this one.
unsigned PRRU = Op.PhysRegReused;
const TargetRegisterInfo *TRI = Spills.getRegInfo();
if (TRI->areAliases(PRRU, PhysReg)) {
Chris Lattner
committed
// Okay, we found out that an alias of a reused register
// was used. This isn't good because it means we have
// to undo a previous reuse.
MachineBasicBlock *MBB = MI->getParent();
const TargetRegisterClass *AliasRC =
MBB->getParent()->getRegInfo().getRegClass(Op.VirtReg);
// Copy Op out of the vector and remove it, we're going to insert an
// explicit load for it.
ReusedOp NewOp = Op;
Reuses.erase(Reuses.begin()+ro);
// Ok, we're going to try to reload the assigned physreg into the
// slot that we were supposed to in the first place. However, that
// register could hold a reuse. Check to see if it conflicts or
// would prefer us to use a different register.
unsigned NewPhysReg = GetRegForReload(NewOp.AssignedPhysReg,
Evan Cheng
committed
MI, Spills, MaybeDeadStores,
Rejected, RegKills, KillOps, VRM);
Evan Cheng
committed
MachineBasicBlock::iterator MII = MI;
if (NewOp.StackSlotOrReMat > VirtRegMap::MAX_STACK_SLOT) {
ReMaterialize(*MBB, MII, NewPhysReg, NewOp.VirtReg, TII, TRI,VRM);
} else {
Evan Cheng
committed
TII->loadRegFromStackSlot(*MBB, MII, NewPhysReg,
NewOp.StackSlotOrReMat, AliasRC);
MachineInstr *LoadMI = prior(MII);
VRM.addSpillSlotUse(NewOp.StackSlotOrReMat, LoadMI);
// Any stores to this stack slot are not dead anymore.
MaybeDeadStores[NewOp.StackSlotOrReMat] = NULL;
++NumLoads;
}
Spills.ClobberPhysReg(NewPhysReg);
Spills.ClobberPhysReg(NewOp.PhysRegReused);
Evan Cheng
committed
unsigned SubIdx = MI->getOperand(NewOp.Operand).getSubReg();
unsigned RReg = SubIdx ? TRI->getSubReg(NewPhysReg, SubIdx) : NewPhysReg;
MI->getOperand(NewOp.Operand).setReg(RReg);
Chris Lattner
committed
Spills.addAvailable(NewOp.StackSlotOrReMat, MI, NewPhysReg);
Evan Cheng
committed
--MII;
Evan Cheng
committed
UpdateKills(*MII, RegKills, KillOps, TRI);
Evan Cheng
committed
DOUT << '\t' << *MII;
Chris Lattner
committed
DOUT << "Reuse undone!\n";
Chris Lattner
committed
--NumReused;
// Finally, PhysReg is now available, go ahead and use it.
Chris Lattner
committed
return PhysReg;
}
}
}
return PhysReg;
}
/// GetRegForReload - Helper for the above GetRegForReload(). Add a
/// 'Rejected' set to remember which registers have been considered and
/// rejected for the reload. This avoids infinite looping in case like
/// this:
/// t1 := op t2, t3
/// t2 <- assigned r0 for use by the reload but ended up reuse r1
/// t3 <- assigned r1 for use by the reload but ended up reuse r0
/// t1 <- desires r1
/// sees r1 is taken by t2, tries t2's reload register r0
/// sees r0 is taken by t3, tries t3's reload register r1
/// sees r1 is taken by t2, tries t2's reload register r0 ...
unsigned GetRegForReload(unsigned PhysReg, MachineInstr *MI,
AvailableSpills &Spills,
std::vector<MachineInstr*> &MaybeDeadStores,
Evan Cheng
committed
BitVector &RegKills,
std::vector<MachineOperand*> &KillOps,
VirtRegMap &VRM) {
SmallSet<unsigned, 8> Rejected;
Evan Cheng
committed
return GetRegForReload(PhysReg, MI, Spills, MaybeDeadStores, Rejected,
RegKills, KillOps, VRM);
}
Chris Lattner
committed
};
}
/// PrepForUnfoldOpti - Turn a store folding instruction into a load folding
/// instruction. e.g.
/// xorl %edi, %eax
/// movl %eax, -32(%ebp)
/// movl -36(%ebp), %eax
/// ==>
/// xorl %edi, %eax
/// orl -36(%ebp), %eax
/// mov %eax, -32(%ebp)
/// This enables unfolding optimization for a subsequent instruction which will
/// also eliminate the newly introduced store instruction.
bool LocalSpiller::PrepForUnfoldOpti(MachineBasicBlock &MBB,
Evan Cheng
committed
MachineBasicBlock::iterator &MII,
Evan Cheng
committed
AvailableSpills &Spills,
BitVector &RegKills,
std::vector<MachineOperand*> &KillOps,
VirtRegMap &VRM) {
MachineFunction &MF = *MBB.getParent();
MachineInstr &MI = *MII;
unsigned UnfoldedOpc = 0;
unsigned UnfoldPR = 0;
unsigned UnfoldVR = 0;
int FoldedSS = VirtRegMap::NO_STACK_SLOT;
VirtRegMap::MI2VirtMapTy::const_iterator I, End;
Evan Cheng
committed
for (tie(I, End) = VRM.getFoldedVirts(&MI); I != End; ) {
// Only transform a MI that folds a single register.
if (UnfoldedOpc)
return false;
UnfoldVR = I->second.first;
VirtRegMap::ModRef MR = I->second.second;
Evan Cheng
committed
// MI2VirtMap be can updated which invalidate the iterator.
// Increment the iterator first.
++I;
if (VRM.isAssignedReg(UnfoldVR))
continue;
// If this reference is not a use, any previous store is now dead.
// Otherwise, the store to this stack slot is not dead anymore.
FoldedSS = VRM.getStackSlot(UnfoldVR);
MachineInstr* DeadStore = MaybeDeadStores[FoldedSS];
if (DeadStore && (MR & VirtRegMap::isModRef)) {
unsigned PhysReg = Spills.getSpillSlotOrReMatPhysReg(FoldedSS);
if (!PhysReg || !DeadStore->readsRegister(PhysReg))
UnfoldedOpc = TII->getOpcodeAfterMemoryUnfold(MI.getOpcode(),
false, true);
}
}
if (!UnfoldedOpc)
return false;
for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI.getOperand(i);
if (!MO.isReg() || MO.getReg() == 0 || !MO.isUse())
if (TargetRegisterInfo::isPhysicalRegister(VirtReg) || MO.getSubReg())
continue;
if (VRM.isAssignedReg(VirtReg)) {
unsigned PhysReg = VRM.getPhys(VirtReg);
if (PhysReg && TRI->regsOverlap(PhysReg, UnfoldPR))
return false;
} else if (VRM.isReMaterialized(VirtReg))
continue;
int SS = VRM.getStackSlot(VirtReg);
unsigned PhysReg = Spills.getSpillSlotOrReMatPhysReg(SS);
if (PhysReg) {
if (TRI->regsOverlap(PhysReg, UnfoldPR))
if (VRM.hasPhys(VirtReg)) {
PhysReg = VRM.getPhys(VirtReg);
if (!TRI->regsOverlap(PhysReg, UnfoldPR))
continue;
}
// Ok, we'll need to reload the value into a register which makes
// it impossible to perform the store unfolding optimization later.
// Let's see if it is possible to fold the load if the store is
// unfolded. This allows us to perform the store unfolding
// optimization.
SmallVector<MachineInstr*, 4> NewMIs;
if (TII->unfoldMemoryOperand(MF, &MI, UnfoldVR, false, false, NewMIs)) {
assert(NewMIs.size() == 1);
MachineInstr *NewMI = NewMIs.back();
NewMIs.clear();
int Idx = NewMI->findRegisterUseOperandIdx(VirtReg, false);
SmallVector<unsigned, 2> Ops;
Ops.push_back(Idx);
MachineInstr *FoldedMI = TII->foldMemoryOperand(MF, NewMI, Ops, SS);
VRM.addSpillSlotUse(SS, FoldedMI);
Evan Cheng
committed
if (!VRM.hasPhys(UnfoldVR))
VRM.assignVirt2Phys(UnfoldVR, UnfoldPR);
VRM.virtFolded(VirtReg, FoldedMI, VirtRegMap::isRef);
MII = MBB.insert(MII, FoldedMI);
MF.DeleteMachineInstr(NewMI);
MF.DeleteMachineInstr(NewMI);
Evan Cheng
committed
/// CommuteToFoldReload -
/// Look for