VirtRegRewriter.cpp

//===-- llvm/CodeGen/Rewriter.cpp -  Rewriter -----------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#define DEBUG_TYPE "virtregrewriter"
#include "VirtRegRewriter.h"
#include "llvm/Function.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetInstrInfo.h"
#include "llvm/Target/TargetLowering.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/Statistic.h"
#include <algorithm>
using namespace llvm;

STATISTIC(NumDSE     , "Number of dead stores elided");
STATISTIC(NumDSS     , "Number of dead spill slots removed");
STATISTIC(NumCommutes, "Number of instructions commuted");
STATISTIC(NumDRM     , "Number of re-materializable defs elided");
STATISTIC(NumStores  , "Number of stores added");
STATISTIC(NumPSpills , "Number of physical register spills");
STATISTIC(NumOmitted , "Number of reloads omited");
STATISTIC(NumAvoided , "Number of reloads deemed unnecessary");
STATISTIC(NumCopified, "Number of available reloads turned into copies");
STATISTIC(NumReMats  , "Number of re-materialization");
STATISTIC(NumLoads   , "Number of loads added");
STATISTIC(NumReused  , "Number of values reused");
STATISTIC(NumDCE     , "Number of copies elided");
STATISTIC(NumSUnfold , "Number of stores unfolded");
STATISTIC(NumModRefUnfold, "Number of modref unfolded");

namespace {
  enum RewriterName { local, trivial };
}

static cl::opt<RewriterName>
RewriterOpt("rewriter",
            cl::desc("Rewriter to use (default=local)"),
            cl::Prefix,
            cl::values(clEnumVal(local,   "local rewriter"),
                       clEnumVal(trivial, "trivial rewriter"),
                       clEnumValEnd),
            cl::init(local));

static cl::opt<bool>
ScheduleSpills("schedule-spills",
               cl::desc("Schedule spill code"),
               cl::init(false));

VirtRegRewriter::~VirtRegRewriter() {}

/// substitutePhysReg - Replace virtual register in MachineOperand with a
/// physical register. Do the right thing with the sub-register index.
/// Note that operands may be added, so the MO reference is no longer valid.
static void substitutePhysReg(MachineOperand &MO, unsigned Reg,
                              const TargetRegisterInfo &TRI) {
  if (unsigned SubIdx = MO.getSubReg()) {
    // Insert the physical subreg and reset the subreg field.
    MO.setReg(TRI.getSubReg(Reg, SubIdx));
    MO.setSubReg(0);

    // Any def, dead, and kill flags apply to the full virtual register, so they
    // also apply to the full physical register. Add imp-def/dead and imp-kill
    // as needed.
    MachineInstr &MI = *MO.getParent();
    if (MO.isDef())
      if (MO.isDead())
        MI.addRegisterDead(Reg, &TRI, /*AddIfNotFound=*/ true);
      else
        MI.addRegisterDefined(Reg, &TRI);
    else if (!MO.isUndef() &&
             (MO.isKill() ||
              MI.isRegTiedToDefOperand(&MO-&MI.getOperand(0))))
      MI.addRegisterKilled(Reg, &TRI, /*AddIfNotFound=*/ true);
  } else {
    MO.setReg(Reg);
  }
}

namespace {

/// This class is intended for use with the new spilling framework only. It
/// rewrites vreg def/uses to use the assigned preg, but does not insert any
/// spill code.
struct TrivialRewriter : public VirtRegRewriter {

  bool runOnMachineFunction(MachineFunction &MF, VirtRegMap &VRM,
                            LiveIntervals* LIs) {
    DEBUG(dbgs() << "********** REWRITE MACHINE CODE **********\n");
    DEBUG(dbgs() << "********** Function: "
          << MF.getFunction()->getName() << '\n');
    DEBUG(dbgs() << "**** Machine Instrs"
          << "(NOTE! Does not include spills and reloads!) ****\n");
    DEBUG(MF.dump());

    MachineRegisterInfo *mri = &MF.getRegInfo();
    const TargetRegisterInfo *tri = MF.getTarget().getRegisterInfo();

    bool changed = false;

    for (LiveIntervals::iterator liItr = LIs->begin(), liEnd = LIs->end();
         liItr != liEnd; ++liItr) {

      const LiveInterval *li = liItr->second;
      unsigned reg = li->reg;

      if (TargetRegisterInfo::isPhysicalRegister(reg)) {
        if (!li->empty())
          mri->setPhysRegUsed(reg);
      }
      else {
        if (!VRM.hasPhys(reg))
          continue;
        unsigned pReg = VRM.getPhys(reg);
        mri->setPhysRegUsed(pReg);
        // Copy the register use-list before traversing it.
        SmallVector<std::pair<MachineInstr*, unsigned>, 32> reglist;
        for (MachineRegisterInfo::reg_iterator I = mri->reg_begin(reg),
               E = mri->reg_end(); I != E; ++I)
          reglist.push_back(std::make_pair(&*I, I.getOperandNo()));
        for (unsigned N=0; N != reglist.size(); ++N)
          substitutePhysReg(reglist[N].first->getOperand(reglist[N].second),
                            pReg, *tri);
        changed |= !reglist.empty();
      }
    }

    DEBUG(dbgs() << "**** Post Machine Instrs ****\n");
    DEBUG(MF.dump());

    return changed;
  }

};

}

// ************************************************************************ //

namespace {

/// AvailableSpills - As the local rewriter is scanning and rewriting an MBB
/// from top down, keep track of which spill slots or remat are available in
/// each register.
///
/// Note that not all physregs are created equal here.  In particular, some
/// physregs are reloads that we are allowed to clobber or ignore at any time.
/// Other physregs are values that the register allocated program is using
/// that we cannot CHANGE, but we can read if we like.  We keep track of this
/// on a per-stack-slot / remat id basis as the low bit in the value of the
/// SpillSlotsAvailable entries.  The predicate 'canClobberPhysReg()' checks
/// this bit and addAvailable sets it if.
class AvailableSpills {
  const TargetRegisterInfo *TRI;
  const TargetInstrInfo *TII;

  // SpillSlotsOrReMatsAvailable - This map keeps track of all of the spilled
  // or remat'ed virtual register values that are still available, due to
  // being loaded or stored to, but not invalidated yet.
  std::map<int, unsigned> SpillSlotsOrReMatsAvailable;

  // PhysRegsAvailable - This is the inverse of SpillSlotsOrReMatsAvailable,
  // indicating which stack slot values are currently held by a physreg.  This
  // is used to invalidate entries in SpillSlotsOrReMatsAvailable when a
  // physreg is modified.
  std::multimap<unsigned, int> PhysRegsAvailable;

  void disallowClobberPhysRegOnly(unsigned PhysReg);

  void ClobberPhysRegOnly(unsigned PhysReg);
public:
  AvailableSpills(const TargetRegisterInfo *tri, const TargetInstrInfo *tii)
    : TRI(tri), TII(tii) {
  }

  /// clear - Reset the state.
  void clear() {
    SpillSlotsOrReMatsAvailable.clear();
    PhysRegsAvailable.clear();
  }

  const TargetRegisterInfo *getRegInfo() const { return TRI; }

  /// getSpillSlotOrReMatPhysReg - If the specified stack slot or remat is
  /// available in a physical register, return that PhysReg, otherwise
  /// return 0.
  unsigned getSpillSlotOrReMatPhysReg(int Slot) const {
    std::map<int, unsigned>::const_iterator I =
      SpillSlotsOrReMatsAvailable.find(Slot);
    if (I != SpillSlotsOrReMatsAvailable.end()) {
      return I->second >> 1;  // Remove the CanClobber bit.
    }
    return 0;
  }

  /// addAvailable - Mark that the specified stack slot / remat is available
  /// in the specified physreg.  If CanClobber is true, the physreg can be
  /// modified at any time without changing the semantics of the program.
  void addAvailable(int SlotOrReMat, unsigned Reg, bool CanClobber = true) {
    // If this stack slot is thought to be available in some other physreg,
    // remove its record.
    ModifyStackSlotOrReMat(SlotOrReMat);

    PhysRegsAvailable.insert(std::make_pair(Reg, SlotOrReMat));
    SpillSlotsOrReMatsAvailable[SlotOrReMat]= (Reg << 1) |
                                              (unsigned)CanClobber;

    if (SlotOrReMat > VirtRegMap::MAX_STACK_SLOT)
      DEBUG(dbgs() << "Remembering RM#"
                   << SlotOrReMat-VirtRegMap::MAX_STACK_SLOT-1);
    else
      DEBUG(dbgs() << "Remembering SS#" << SlotOrReMat);
    DEBUG(dbgs() << " in physreg " << TRI->getName(Reg) << "\n");
  }

  /// canClobberPhysRegForSS - Return true if the spiller is allowed to change
  /// the value of the specified stackslot register if it desires. The
  /// specified stack slot must be available in a physreg for this query to
  /// make sense.
  bool canClobberPhysRegForSS(int SlotOrReMat) const {
    assert(SpillSlotsOrReMatsAvailable.count(SlotOrReMat) &&
           "Value not available!");
    return SpillSlotsOrReMatsAvailable.find(SlotOrReMat)->second & 1;
  }

  /// canClobberPhysReg - Return true if the spiller is allowed to clobber the
  /// physical register where values for some stack slot(s) might be
  /// available.
  bool canClobberPhysReg(unsigned PhysReg) const {
    std::multimap<unsigned, int>::const_iterator I =
      PhysRegsAvailable.lower_bound(PhysReg);
    while (I != PhysRegsAvailable.end() && I->first == PhysReg) {
      int SlotOrReMat = I->second;
      I++;
      if (!canClobberPhysRegForSS(SlotOrReMat))
        return false;
    }
    return true;
  }

  /// disallowClobberPhysReg - Unset the CanClobber bit of the specified
  /// stackslot register. The register is still available but is no longer
  /// allowed to be modifed.
  void disallowClobberPhysReg(unsigned PhysReg);

  /// ClobberPhysReg - This is called when the specified physreg changes
  /// value.  We use this to invalidate any info about stuff that lives in
  /// it and any of its aliases.
  void ClobberPhysReg(unsigned PhysReg);

  /// ModifyStackSlotOrReMat - This method is called when the value in a stack
  /// slot changes.  This removes information about which register the
  /// previous value for this slot lives in (as the previous value is dead
  /// now).
  void ModifyStackSlotOrReMat(int SlotOrReMat);

  /// AddAvailableRegsToLiveIn - Availability information is being kept coming
  /// into the specified MBB. Add available physical registers as potential
  /// live-in's. If they are reused in the MBB, they will be added to the
  /// live-in set to make register scavenger and post-allocation scheduler.
  void AddAvailableRegsToLiveIn(MachineBasicBlock &MBB, BitVector &RegKills,
                                std::vector<MachineOperand*> &KillOps);
};

}

// ************************************************************************ //

// Given a location where a reload of a spilled register or a remat of
// a constant is to be inserted, attempt to find a safe location to
// insert the load at an earlier point in the basic-block, to hide
// latency of the load and to avoid address-generation interlock
// issues.
static MachineBasicBlock::iterator
ComputeReloadLoc(MachineBasicBlock::iterator const InsertLoc,
                 MachineBasicBlock::iterator const Begin,
                 unsigned PhysReg,
                 const TargetRegisterInfo *TRI,
                 bool DoReMat,
                 int SSorRMId,
                 const TargetInstrInfo *TII,
                 const MachineFunction &MF)
{
  if (!ScheduleSpills)
    return InsertLoc;

  // Spill backscheduling is of primary interest to addresses, so
  // don't do anything if the register isn't in the register class
  // used for pointers.

  const TargetLowering *TL = MF.getTarget().getTargetLowering();

  if (!TL->isTypeLegal(TL->getPointerTy()))
    // Believe it or not, this is true on PIC16.
    return InsertLoc;

  const TargetRegisterClass *ptrRegClass =
    TL->getRegClassFor(TL->getPointerTy());
  if (!ptrRegClass->contains(PhysReg))
    return InsertLoc;

  // Scan upwards through the preceding instructions. If an instruction doesn't
  // reference the stack slot or the register we're loading, we can
  // backschedule the reload up past it.
  MachineBasicBlock::iterator NewInsertLoc = InsertLoc;
  while (NewInsertLoc != Begin) {
    MachineBasicBlock::iterator Prev = prior(NewInsertLoc);
    for (unsigned i = 0; i < Prev->getNumOperands(); ++i) {
      MachineOperand &Op = Prev->getOperand(i);
      if (!DoReMat && Op.isFI() && Op.getIndex() == SSorRMId)
        goto stop;
    }
    if (Prev->findRegisterUseOperandIdx(PhysReg) != -1 ||
        Prev->findRegisterDefOperand(PhysReg))
      goto stop;
    for (const unsigned *Alias = TRI->getAliasSet(PhysReg); *Alias; ++Alias)
      if (Prev->findRegisterUseOperandIdx(*Alias) != -1 ||
          Prev->findRegisterDefOperand(*Alias))
        goto stop;
    NewInsertLoc = Prev;
  }
stop:;

  // If we made it to the beginning of the block, turn around and move back
  // down just past any existing reloads. They're likely to be reloads/remats
  // for instructions earlier than what our current reload/remat is for, so
  // they should be scheduled earlier.
  if (NewInsertLoc == Begin) {
    int FrameIdx;
    while (InsertLoc != NewInsertLoc &&
           (TII->isLoadFromStackSlot(NewInsertLoc, FrameIdx) ||
            TII->isTriviallyReMaterializable(NewInsertLoc)))
      ++NewInsertLoc;
  }

  return NewInsertLoc;
}

namespace {

// ReusedOp - For each reused operand, we keep track of a bit of information,
// in case we need to rollback upon processing a new operand.  See comments
// below.
struct ReusedOp {
  // The MachineInstr operand that reused an available value.
  unsigned Operand;

  // StackSlotOrReMat - The spill slot or remat id of the value being reused.
  unsigned StackSlotOrReMat;

  // PhysRegReused - The physical register the value was available in.
  unsigned PhysRegReused;

  // AssignedPhysReg - The physreg that was assigned for use by the reload.
  unsigned AssignedPhysReg;

  // VirtReg - The virtual register itself.
  unsigned VirtReg;

  ReusedOp(unsigned o, unsigned ss, unsigned prr, unsigned apr,
           unsigned vreg)
    : Operand(o), StackSlotOrReMat(ss), PhysRegReused(prr),
      AssignedPhysReg(apr), VirtReg(vreg) {}
};

/// ReuseInfo - This maintains a collection of ReuseOp's for each operand that
/// is reused instead of reloaded.
class ReuseInfo {
  MachineInstr &MI;
  std::vector<ReusedOp> Reuses;
  BitVector PhysRegsClobbered;
public:
  ReuseInfo(MachineInstr &mi, const TargetRegisterInfo *tri) : MI(mi) {
    PhysRegsClobbered.resize(tri->getNumRegs());
  }

  bool hasReuses() const {
    return !Reuses.empty();
  }

  /// addReuse - If we choose to reuse a virtual register that is already
  /// available instead of reloading it, remember that we did so.
  void addReuse(unsigned OpNo, unsigned StackSlotOrReMat,
                unsigned PhysRegReused, unsigned AssignedPhysReg,
                unsigned VirtReg) {
    // If the reload is to the assigned register anyway, no undo will be
    // required.
    if (PhysRegReused == AssignedPhysReg) return;

    // Otherwise, remember this.
    Reuses.push_back(ReusedOp(OpNo, StackSlotOrReMat, PhysRegReused,
                              AssignedPhysReg, VirtReg));
  }

  void markClobbered(unsigned PhysReg) {
    PhysRegsClobbered.set(PhysReg);
  }

  bool isClobbered(unsigned PhysReg) const {
    return PhysRegsClobbered.test(PhysReg);
  }

  /// GetRegForReload - We are about to emit a reload into PhysReg.  If there
  /// is some other operand that is using the specified register, either pick
  /// a new register to use, or evict the previous reload and use this reg.
  unsigned GetRegForReload(const TargetRegisterClass *RC, unsigned PhysReg,
                           MachineFunction &MF, MachineInstr *MI,
                           AvailableSpills &Spills,
                           std::vector<MachineInstr*> &MaybeDeadStores,
                           SmallSet<unsigned, 8> &Rejected,
                           BitVector &RegKills,
                           std::vector<MachineOperand*> &KillOps,
                           VirtRegMap &VRM);

  /// GetRegForReload - Helper for the above GetRegForReload(). Add a
  /// 'Rejected' set to remember which registers have been considered and
  /// rejected for the reload. This avoids infinite looping in case like
  /// this:
  /// t1 := op t2, t3
  /// t2 <- assigned r0 for use by the reload but ended up reuse r1
  /// t3 <- assigned r1 for use by the reload but ended up reuse r0
  /// t1 <- desires r1
  ///       sees r1 is taken by t2, tries t2's reload register r0
  ///       sees r0 is taken by t3, tries t3's reload register r1
  ///       sees r1 is taken by t2, tries t2's reload register r0 ...
  unsigned GetRegForReload(unsigned VirtReg, unsigned PhysReg, MachineInstr *MI,
                           AvailableSpills &Spills,
                           std::vector<MachineInstr*> &MaybeDeadStores,
                           BitVector &RegKills,
                           std::vector<MachineOperand*> &KillOps,
                           VirtRegMap &VRM) {
    SmallSet<unsigned, 8> Rejected;
    MachineFunction &MF = *MI->getParent()->getParent();
    const TargetRegisterClass* RC = MF.getRegInfo().getRegClass(VirtReg);
    return GetRegForReload(RC, PhysReg, MF, MI, Spills, MaybeDeadStores,
                           Rejected, RegKills, KillOps, VRM);
  }
};

}

// ****************** //
// Utility Functions  //
// ****************** //

/// findSinglePredSuccessor - Return via reference a vector of machine basic
/// blocks each of which is a successor of the specified BB and has no other
/// predecessor.
static void findSinglePredSuccessor(MachineBasicBlock *MBB,
                                   SmallVectorImpl<MachineBasicBlock *> &Succs) {
  for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(),
         SE = MBB->succ_end(); SI != SE; ++SI) {
    MachineBasicBlock *SuccMBB = *SI;
    if (SuccMBB->pred_size() == 1)
      Succs.push_back(SuccMBB);
  }
}

/// InvalidateKill - Invalidate register kill information for a specific
/// register. This also unsets the kills marker on the last kill operand.
static void InvalidateKill(unsigned Reg,
                           const TargetRegisterInfo* TRI,
                           BitVector &RegKills,
                           std::vector<MachineOperand*> &KillOps) {
  if (RegKills[Reg]) {
    KillOps[Reg]->setIsKill(false);
    // KillOps[Reg] might be a def of a super-register.
    unsigned KReg = KillOps[Reg]->getReg();
    KillOps[KReg] = NULL;
    RegKills.reset(KReg);
    for (const unsigned *SR = TRI->getSubRegisters(KReg); *SR; ++SR) {
      if (RegKills[*SR]) {
        KillOps[*SR]->setIsKill(false);
        KillOps[*SR] = NULL;
        RegKills.reset(*SR);
      }
    }
  }
}

/// InvalidateKills - MI is going to be deleted. If any of its operands are
/// marked kill, then invalidate the information.
static void InvalidateKills(MachineInstr &MI,
                            const TargetRegisterInfo* TRI,
                            BitVector &RegKills,
                            std::vector<MachineOperand*> &KillOps,
                            SmallVector<unsigned, 2> *KillRegs = NULL) {
  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
    MachineOperand &MO = MI.getOperand(i);
    if (!MO.isReg() || !MO.isUse() || !MO.isKill() || MO.isUndef())
      continue;
    unsigned Reg = MO.getReg();
    if (TargetRegisterInfo::isVirtualRegister(Reg))
      continue;
    if (KillRegs)
      KillRegs->push_back(Reg);
    assert(Reg < KillOps.size());
    if (KillOps[Reg] == &MO) {
      KillOps[Reg] = NULL;
      RegKills.reset(Reg);
      for (const unsigned *SR = TRI->getSubRegisters(Reg); *SR; ++SR) {
        if (RegKills[*SR]) {
          KillOps[*SR] = NULL;
          RegKills.reset(*SR);
        }
      }
    }
  }
}

/// InvalidateRegDef - If the def operand of the specified def MI is now dead
/// (since its spill instruction is removed), mark it isDead. Also checks if
/// the def MI has other definition operands that are not dead. Returns it by
/// reference.
static bool InvalidateRegDef(MachineBasicBlock::iterator I,
                             MachineInstr &NewDef, unsigned Reg,
                             bool &HasLiveDef,
                             const TargetRegisterInfo *TRI) {
  // Due to remat, it's possible this reg isn't being reused. That is,
  // the def of this reg (by prev MI) is now dead.
  MachineInstr *DefMI = I;
  MachineOperand *DefOp = NULL;
  for (unsigned i = 0, e = DefMI->getNumOperands(); i != e; ++i) {
    MachineOperand &MO = DefMI->getOperand(i);
    if (!MO.isReg() || !MO.isDef() || !MO.isKill() || MO.isUndef())
      continue;
    if (MO.getReg() == Reg)
      DefOp = &MO;
    else if (!MO.isDead())
      HasLiveDef = true;
  }
  if (!DefOp)
    return false;

  bool FoundUse = false, Done = false;
  MachineBasicBlock::iterator E = &NewDef;
  ++I; ++E;
  for (; !Done && I != E; ++I) {
    MachineInstr *NMI = I;
    for (unsigned j = 0, ee = NMI->getNumOperands(); j != ee; ++j) {
      MachineOperand &MO = NMI->getOperand(j);
      if (!MO.isReg() || MO.getReg() == 0 ||
          (MO.getReg() != Reg && !TRI->isSubRegister(Reg, MO.getReg())))
        continue;
      if (MO.isUse())
        FoundUse = true;
      Done = true; // Stop after scanning all the operands of this MI.
    }
  }
  if (!FoundUse) {
    // Def is dead!
    DefOp->setIsDead();
    return true;
  }
  return false;
}

/// UpdateKills - Track and update kill info. If a MI reads a register that is
/// marked kill, then it must be due to register reuse. Transfer the kill info
/// over.
static void UpdateKills(MachineInstr &MI, const TargetRegisterInfo* TRI,
                        BitVector &RegKills,
                        std::vector<MachineOperand*> &KillOps) {
  // These do not affect kill info at all.
  if (MI.isDebugValue())
    return;
  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
    MachineOperand &MO = MI.getOperand(i);
    if (!MO.isReg() || !MO.isUse() || MO.isUndef())
      continue;
    unsigned Reg = MO.getReg();
    if (Reg == 0)
      continue;

    if (RegKills[Reg] && KillOps[Reg]->getParent() != &MI) {
      // That can't be right. Register is killed but not re-defined and it's
      // being reused. Let's fix that.
      KillOps[Reg]->setIsKill(false);
      // KillOps[Reg] might be a def of a super-register.
      unsigned KReg = KillOps[Reg]->getReg();
      KillOps[KReg] = NULL;
      RegKills.reset(KReg);

      // Must be a def of a super-register. Its other sub-regsters are no
      // longer killed as well.
      for (const unsigned *SR = TRI->getSubRegisters(KReg); *SR; ++SR) {
        KillOps[*SR] = NULL;
        RegKills.reset(*SR);
      }
    } else {
      // Check for subreg kills as well.
      // d4 =
      // store d4, fi#0
      // ...
      //    = s8<kill>
      // ...
      //    = d4  <avoiding reload>
      for (const unsigned *SR = TRI->getSubRegisters(Reg); *SR; ++SR) {
        unsigned SReg = *SR;
        if (RegKills[SReg] && KillOps[SReg]->getParent() != &MI) {
          KillOps[SReg]->setIsKill(false);
          unsigned KReg = KillOps[SReg]->getReg();
          KillOps[KReg] = NULL;
          RegKills.reset(KReg);

          for (const unsigned *SSR = TRI->getSubRegisters(KReg); *SSR; ++SSR) {
            KillOps[*SSR] = NULL;
            RegKills.reset(*SSR);
          }
        }
      }
    }

    if (MO.isKill()) {
      RegKills.set(Reg);
      KillOps[Reg] = &MO;
      for (const unsigned *SR = TRI->getSubRegisters(Reg); *SR; ++SR) {
        RegKills.set(*SR);
        KillOps[*SR] = &MO;
      }
    }
  }

  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
    const MachineOperand &MO = MI.getOperand(i);
    if (!MO.isReg() || !MO.getReg() || !MO.isDef())
      continue;
    unsigned Reg = MO.getReg();
    RegKills.reset(Reg);
    KillOps[Reg] = NULL;
    // It also defines (or partially define) aliases.
    for (const unsigned *SR = TRI->getSubRegisters(Reg); *SR; ++SR) {
      RegKills.reset(*SR);
      KillOps[*SR] = NULL;
    }
    for (const unsigned *SR = TRI->getSuperRegisters(Reg); *SR; ++SR) {
      RegKills.reset(*SR);
      KillOps[*SR] = NULL;
    }
  }
}

/// ReMaterialize - Re-materialize definition for Reg targetting DestReg.
///
static void ReMaterialize(MachineBasicBlock &MBB,
                          MachineBasicBlock::iterator &MII,
                          unsigned DestReg, unsigned Reg,
                          const TargetInstrInfo *TII,
                          const TargetRegisterInfo *TRI,
                          VirtRegMap &VRM) {
  MachineInstr *ReMatDefMI = VRM.getReMaterializedMI(Reg);
#ifndef NDEBUG
  const TargetInstrDesc &TID = ReMatDefMI->getDesc();
  assert(TID.getNumDefs() == 1 &&
         "Don't know how to remat instructions that define > 1 values!");
#endif
  TII->reMaterialize(MBB, MII, DestReg,
                     ReMatDefMI->getOperand(0).getSubReg(), ReMatDefMI, TRI);
  MachineInstr *NewMI = prior(MII);
  for (unsigned i = 0, e = NewMI->getNumOperands(); i != e; ++i) {
    MachineOperand &MO = NewMI->getOperand(i);
    if (!MO.isReg() || MO.getReg() == 0)
      continue;
    unsigned VirtReg = MO.getReg();
    if (TargetRegisterInfo::isPhysicalRegister(VirtReg))
      continue;
    assert(MO.isUse());
    unsigned Phys = VRM.getPhys(VirtReg);
    assert(Phys && "Virtual register is not assigned a register?");
    substitutePhysReg(MO, Phys, *TRI);
  }
  ++NumReMats;
}

/// findSuperReg - Find the SubReg's super-register of given register class
/// where its SubIdx sub-register is SubReg.
static unsigned findSuperReg(const TargetRegisterClass *RC, unsigned SubReg,
                             unsigned SubIdx, const TargetRegisterInfo *TRI) {
  for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
       I != E; ++I) {
    unsigned Reg = *I;
    if (TRI->getSubReg(Reg, SubIdx) == SubReg)
      return Reg;
  }
  return 0;
}

// ******************************** //
// Available Spills Implementation  //
// ******************************** //

/// disallowClobberPhysRegOnly - Unset the CanClobber bit of the specified
/// stackslot register. The register is still available but is no longer
/// allowed to be modifed.
void AvailableSpills::disallowClobberPhysRegOnly(unsigned PhysReg) {
  std::multimap<unsigned, int>::iterator I =
    PhysRegsAvailable.lower_bound(PhysReg);
  while (I != PhysRegsAvailable.end() && I->first == PhysReg) {
    int SlotOrReMat = I->second;
    I++;
    assert((SpillSlotsOrReMatsAvailable[SlotOrReMat] >> 1) == PhysReg &&
           "Bidirectional map mismatch!");
    SpillSlotsOrReMatsAvailable[SlotOrReMat] &= ~1;
    DEBUG(dbgs() << "PhysReg " << TRI->getName(PhysReg)
         << " copied, it is available for use but can no longer be modified\n");
  }
}

/// disallowClobberPhysReg - Unset the CanClobber bit of the specified
/// stackslot register and its aliases. The register and its aliases may
/// still available but is no longer allowed to be modifed.
void AvailableSpills::disallowClobberPhysReg(unsigned PhysReg) {
  for (const unsigned *AS = TRI->getAliasSet(PhysReg); *AS; ++AS)
    disallowClobberPhysRegOnly(*AS);
  disallowClobberPhysRegOnly(PhysReg);
}

/// ClobberPhysRegOnly - This is called when the specified physreg changes
/// value.  We use this to invalidate any info about stuff we thing lives in it.
void AvailableSpills::ClobberPhysRegOnly(unsigned PhysReg) {
  std::multimap<unsigned, int>::iterator I =
    PhysRegsAvailable.lower_bound(PhysReg);
  while (I != PhysRegsAvailable.end() && I->first == PhysReg) {
    int SlotOrReMat = I->second;
    PhysRegsAvailable.erase(I++);
    assert((SpillSlotsOrReMatsAvailable[SlotOrReMat] >> 1) == PhysReg &&
           "Bidirectional map mismatch!");
    SpillSlotsOrReMatsAvailable.erase(SlotOrReMat);
    DEBUG(dbgs() << "PhysReg " << TRI->getName(PhysReg)
          << " clobbered, invalidating ");
    if (SlotOrReMat > VirtRegMap::MAX_STACK_SLOT)
      DEBUG(dbgs() << "RM#" << SlotOrReMat-VirtRegMap::MAX_STACK_SLOT-1 <<"\n");
    else
      DEBUG(dbgs() << "SS#" << SlotOrReMat << "\n");
  }
}

/// ClobberPhysReg - This is called when the specified physreg changes
/// value.  We use this to invalidate any info about stuff we thing lives in
/// it and any of its aliases.
void AvailableSpills::ClobberPhysReg(unsigned PhysReg) {
  for (const unsigned *AS = TRI->getAliasSet(PhysReg); *AS; ++AS)
    ClobberPhysRegOnly(*AS);
  ClobberPhysRegOnly(PhysReg);
}

/// AddAvailableRegsToLiveIn - Availability information is being kept coming
/// into the specified MBB. Add available physical registers as potential
/// live-in's. If they are reused in the MBB, they will be added to the
/// live-in set to make register scavenger and post-allocation scheduler.
void AvailableSpills::AddAvailableRegsToLiveIn(MachineBasicBlock &MBB,
                                        BitVector &RegKills,
                                        std::vector<MachineOperand*> &KillOps) {
  std::set<unsigned> NotAvailable;
  for (std::multimap<unsigned, int>::iterator
         I = PhysRegsAvailable.begin(), E = PhysRegsAvailable.end();
       I != E; ++I) {
    unsigned Reg = I->first;
    const TargetRegisterClass* RC = TRI->getPhysicalRegisterRegClass(Reg);
    // FIXME: A temporary workaround. We can't reuse available value if it's
    // not safe to move the def of the virtual register's class. e.g.
    // X86::RFP* register classes. Do not add it as a live-in.
    if (!TII->isSafeToMoveRegClassDefs(RC))
      // This is no longer available.
      NotAvailable.insert(Reg);
    else {
      MBB.addLiveIn(Reg);
      InvalidateKill(Reg, TRI, RegKills, KillOps);
    }

    // Skip over the same register.
    std::multimap<unsigned, int>::iterator NI = llvm::next(I);
    while (NI != E && NI->first == Reg) {
      ++I;
      ++NI;
    }
  }

  for (std::set<unsigned>::iterator I = NotAvailable.begin(),
         E = NotAvailable.end(); I != E; ++I) {
    ClobberPhysReg(*I);
    for (const unsigned *SubRegs = TRI->getSubRegisters(*I);
       *SubRegs; ++SubRegs)
      ClobberPhysReg(*SubRegs);
  }
}

/// ModifyStackSlotOrReMat - This method is called when the value in a stack
/// slot changes.  This removes information about which register the previous
/// value for this slot lives in (as the previous value is dead now).
void AvailableSpills::ModifyStackSlotOrReMat(int SlotOrReMat) {
  std::map<int, unsigned>::iterator It =
    SpillSlotsOrReMatsAvailable.find(SlotOrReMat);
  if (It == SpillSlotsOrReMatsAvailable.end()) return;
  unsigned Reg = It->second >> 1;
  SpillSlotsOrReMatsAvailable.erase(It);

  // This register may hold the value of multiple stack slots, only remove this
  // stack slot from the set of values the register contains.
  std::multimap<unsigned, int>::iterator I = PhysRegsAvailable.lower_bound(Reg);
  for (; ; ++I) {
    assert(I != PhysRegsAvailable.end() && I->first == Reg &&
           "Map inverse broken!");
    if (I->second == SlotOrReMat) break;
  }
  PhysRegsAvailable.erase(I);
}

// ************************** //
// Reuse Info Implementation  //
// ************************** //

/// GetRegForReload - We are about to emit a reload into PhysReg.  If there
/// is some other operand that is using the specified register, either pick
/// a new register to use, or evict the previous reload and use this reg.
unsigned ReuseInfo::GetRegForReload(const TargetRegisterClass *RC,
                         unsigned PhysReg,
                         MachineFunction &MF,
                         MachineInstr *MI, AvailableSpills &Spills,
                         std::vector<MachineInstr*> &MaybeDeadStores,
                         SmallSet<unsigned, 8> &Rejected,
                         BitVector &RegKills,
                         std::vector<MachineOperand*> &KillOps,
                         VirtRegMap &VRM) {
  const TargetInstrInfo* TII = MF.getTarget().getInstrInfo();
  const TargetRegisterInfo *TRI = Spills.getRegInfo();

  if (Reuses.empty()) return PhysReg;  // This is most often empty.

  for (unsigned ro = 0, e = Reuses.size(); ro != e; ++ro) {
    ReusedOp &Op = Reuses[ro];
    // If we find some other reuse that was supposed to use this register
    // exactly for its reload, we can change this reload to use ITS reload
    // register. That is, unless its reload register has already been
    // considered and subsequently rejected because it has also been reused
    // by another operand.
    if (Op.PhysRegReused == PhysReg &&
        Rejected.count(Op.AssignedPhysReg) == 0 &&
        RC->contains(Op.AssignedPhysReg)) {
      // Yup, use the reload register that we didn't use before.
      unsigned NewReg = Op.AssignedPhysReg;
      Rejected.insert(PhysReg);
      return GetRegForReload(RC, NewReg, MF, MI, Spills, MaybeDeadStores, Rejected,
                             RegKills, KillOps, VRM);
    } else {
      // Otherwise, we might also have a problem if a previously reused
      // value aliases the new register. If so, codegen the previous reload
      // and use this one.
      unsigned PRRU = Op.PhysRegReused;
      if (TRI->regsOverlap(PRRU, PhysReg)) {
        // Okay, we found out that an alias of a reused register
        // was used.  This isn't good because it means we have
        // to undo a previous reuse.
        MachineBasicBlock *MBB = MI->getParent();
        const TargetRegisterClass *AliasRC =
          MBB->getParent()->getRegInfo().getRegClass(Op.VirtReg);

        // Copy Op out of the vector and remove it, we're going to insert an
        // explicit load for it.
        ReusedOp NewOp = Op;
        Reuses.erase(Reuses.begin()+ro);

        // MI may be using only a sub-register of PhysRegUsed.
        unsigned RealPhysRegUsed = MI->getOperand(NewOp.Operand).getReg();
        unsigned SubIdx = 0;
        assert(TargetRegisterInfo::isPhysicalRegister(RealPhysRegUsed) &&
               "A reuse cannot be a virtual register");
        if (PRRU != RealPhysRegUsed) {
          // What was the sub-register index?
          SubIdx = TRI->getSubRegIndex(PRRU, RealPhysRegUsed);
          assert(SubIdx &&
                 "Operand physreg is not a sub-register of PhysRegUsed");
        }

        // Ok, we're going to try to reload the assigned physreg into the
        // slot that we were supposed to in the first place.  However, that
        // register could hold a reuse.  Check to see if it conflicts or
        // would prefer us to use a different register.
        unsigned NewPhysReg = GetRegForReload(RC, NewOp.AssignedPhysReg,
                                              MF, MI, Spills, MaybeDeadStores,
                                              Rejected, RegKills, KillOps, VRM);

        bool DoReMat = NewOp.StackSlotOrReMat > VirtRegMap::MAX_STACK_SLOT;
        int SSorRMId = DoReMat
          ? VRM.getReMatId(NewOp.VirtReg) : NewOp.StackSlotOrReMat;

        // Back-schedule reloads and remats.
        MachineBasicBlock::iterator InsertLoc =
          ComputeReloadLoc(MI, MBB->begin(), PhysReg, TRI,
                           DoReMat, SSorRMId, TII, MF);

        if (DoReMat) {
          ReMaterialize(*MBB, InsertLoc, NewPhysReg, NewOp.VirtReg, TII,
                        TRI, VRM);
        } else {
          TII->loadRegFromStackSlot(*MBB, InsertLoc, NewPhysReg,
                                    NewOp.StackSlotOrReMat, AliasRC);
          MachineInstr *LoadMI = prior(InsertLoc);
          VRM.addSpillSlotUse(NewOp.StackSlotOrReMat, LoadMI);
          // Any stores to this stack slot are not dead anymore.
          MaybeDeadStores[NewOp.StackSlotOrReMat] = NULL;
          ++NumLoads;
        }
        Spills.ClobberPhysReg(NewPhysReg);
        Spills.ClobberPhysReg(NewOp.PhysRegReused);

        unsigned RReg = SubIdx ? TRI->getSubReg(NewPhysReg, SubIdx) :NewPhysReg;
        MI->getOperand(NewOp.Operand).setReg(RReg);
        MI->getOperand(NewOp.Operand).setSubReg(0);

        Spills.addAvailable(NewOp.StackSlotOrReMat, NewPhysReg);
        UpdateKills(*prior(InsertLoc), TRI, RegKills, KillOps);
        DEBUG(dbgs() << '\t' << *prior(InsertLoc));

        DEBUG(dbgs() << "Reuse undone!\n");
        --NumReused;

        // Finally, PhysReg is now available, go ahead and use it.
        return PhysReg;
      }
    }
  }
  return PhysReg;
}

// ************************************************************************ //

/// FoldsStackSlotModRef - Return true if the specified MI folds the specified
/// stack slot mod/ref. It also checks if it's possible to unfold the
/// instruction by having it define a specified physical register instead.
static bool FoldsStackSlotModRef(MachineInstr &MI, int SS, unsigned PhysReg,
                                 const TargetInstrInfo *TII,
                                 const TargetRegisterInfo *TRI,
                                 VirtRegMap &VRM) {
  if (VRM.hasEmergencySpills(&MI) || VRM.isSpillPt(&MI))
    return false;

  bool Found = false;
  VirtRegMap::MI2VirtMapTy::const_iterator I, End;
  for (tie(I, End) = VRM.getFoldedVirts(&MI); I != End; ++I) {
    unsigned VirtReg = I->second.first;
    VirtRegMap::ModRef MR = I->second.second;
    if (MR & VirtRegMap::isModRef)
      if (VRM.getStackSlot(VirtReg) == SS) {
        Found= TII->getOpcodeAfterMemoryUnfold(MI.getOpcode(), true, true) != 0;
        break;
      }
  }
  if (!Found)
    return false;

  // Does the instruction uses a register that overlaps the scratch register?
  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
    MachineOperand &MO = MI.getOperand(i);
    if (!MO.isReg() || MO.getReg() == 0)
      continue;
    unsigned Reg = MO.getReg();
    if (TargetRegisterInfo::isVirtualRegister(Reg)) {
      if (!VRM.hasPhys(Reg))
        continue;
      Reg = VRM.getPhys(Reg);
    }
    if (TRI->regsOverlap(PhysReg, Reg))
      return false;
  }
  return true;
}

/// FindFreeRegister - Find a free register of a given register class by looking
/// at (at most) the last two machine instructions.
static unsigned FindFreeRegister(MachineBasicBlock::iterator MII,
                                 MachineBasicBlock &MBB,
                                 const TargetRegisterClass *RC,
                                 const TargetRegisterInfo *TRI,
                                 BitVector &AllocatableRegs) {
  BitVector Defs(TRI->getNumRegs());
  BitVector Uses(TRI->getNumRegs());
  SmallVector<unsigned, 4> LocalUses;
  SmallVector<unsigned, 4> Kills;

  // Take a look at 2 instructions at most.
  for (unsigned Count = 0; Count < 2; ++Count) {
    if (MII == MBB.begin())
      break;
    MachineInstr *PrevMI = prior(MII);
    for (unsigned i = 0, e = PrevMI->getNumOperands(); i != e; ++i) {
      MachineOperand &MO = PrevMI->getOperand(i);
      if (!MO.isReg() || MO.getReg() == 0)
        continue;