InstSelectSimple.cpp

//===-- InstSelectSimple.cpp - A simple instruction selector for x86 ------===//
//
// This file defines a simple peephole instruction selector for the x86 target
//
//===----------------------------------------------------------------------===//

#include "X86.h"
#include "X86InstrInfo.h"
#include "X86InstrBuilder.h"
#include "llvm/Function.h"
#include "llvm/Instructions.h"
#include "llvm/DerivedTypes.h"
#include "llvm/Constants.h"
#include "llvm/Pass.h"
#include "llvm/Intrinsics.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/SSARegMap.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/MRegisterInfo.h"
#include "llvm/Support/InstVisitor.h"

/// BMI - A special BuildMI variant that takes an iterator to insert the
/// instruction at as well as a basic block.  This is the version for when you
/// have a destination register in mind.
inline static MachineInstrBuilder BMI(MachineBasicBlock *MBB,
                                      MachineBasicBlock::iterator &I,
                                      int Opcode, unsigned NumOperands,
                                      unsigned DestReg) {
  assert(I >= MBB->begin() && I <= MBB->end() && "Bad iterator!");
  MachineInstr *MI = new MachineInstr(Opcode, NumOperands+1, true, true);
  I = MBB->insert(I, MI)+1;
  return MachineInstrBuilder(MI).addReg(DestReg, MOTy::Def);
}

/// BMI - A special BuildMI variant that takes an iterator to insert the
/// instruction at as well as a basic block.
inline static MachineInstrBuilder BMI(MachineBasicBlock *MBB,
                                      MachineBasicBlock::iterator &I,
                                      int Opcode, unsigned NumOperands) {
  assert(I >= MBB->begin() && I <= MBB->end() && "Bad iterator!");
  MachineInstr *MI = new MachineInstr(Opcode, NumOperands, true, true);
  I = MBB->insert(I, MI)+1;
  return MachineInstrBuilder(MI);
}


namespace {
  struct ISel : public FunctionPass, InstVisitor<ISel> {
    TargetMachine &TM;
    MachineFunction *F;                 // The function we are compiling into
    MachineBasicBlock *BB;              // The current MBB we are compiling
    int VarArgsFrameIndex;              // FrameIndex for start of varargs area

    std::map<Value*, unsigned> RegMap;  // Mapping between Val's and SSA Regs

    // MBBMap - Mapping between LLVM BB -> Machine BB
    std::map<const BasicBlock*, MachineBasicBlock*> MBBMap;

    ISel(TargetMachine &tm) : TM(tm), F(0), BB(0) {}

    /// runOnFunction - Top level implementation of instruction selection for
    /// the entire function.
    ///
    bool runOnFunction(Function &Fn) {
      F = &MachineFunction::construct(&Fn, TM);

      // Create all of the machine basic blocks for the function...
      for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I)
        F->getBasicBlockList().push_back(MBBMap[I] = new MachineBasicBlock(I));

      BB = &F->front();

      // Copy incoming arguments off of the stack...
      LoadArgumentsToVirtualRegs(Fn);

      // Instruction select everything except PHI nodes
      visit(Fn);

      // Select the PHI nodes
      SelectPHINodes();

      RegMap.clear();
      MBBMap.clear();
      F = 0;
      // We always build a machine code representation for the function
      return true;
    }

    virtual const char *getPassName() const {
      return "X86 Simple Instruction Selection";
    }

    /// visitBasicBlock - This method is called when we are visiting a new basic
    /// block.  This simply creates a new MachineBasicBlock to emit code into
    /// and adds it to the current MachineFunction.  Subsequent visit* for
    /// instructions will be invoked for all instructions in the basic block.
    ///
    void visitBasicBlock(BasicBlock &LLVM_BB) {
      BB = MBBMap[&LLVM_BB];
    }

    /// LoadArgumentsToVirtualRegs - Load all of the arguments to this function
    /// from the stack into virtual registers.
    ///
    void LoadArgumentsToVirtualRegs(Function &F);

    /// SelectPHINodes - Insert machine code to generate phis.  This is tricky
    /// because we have to generate our sources into the source basic blocks,
    /// not the current one.
    ///
    void SelectPHINodes();

    // Visitation methods for various instructions.  These methods simply emit
    // fixed X86 code for each instruction.
    //

    // Control flow operators
    void visitReturnInst(ReturnInst &RI);
    void visitBranchInst(BranchInst &BI);

    struct ValueRecord {
      Value *Val;
      unsigned Reg;
      const Type *Ty;
      ValueRecord(unsigned R, const Type *T) : Val(0), Reg(R), Ty(T) {}
      ValueRecord(Value *V) : Val(V), Reg(0), Ty(V->getType()) {}
    };
    void doCall(const ValueRecord &Ret, MachineInstr *CallMI,
		const std::vector<ValueRecord> &Args);
    void visitCallInst(CallInst &I);
    void visitIntrinsicCall(LLVMIntrinsic::ID ID, CallInst &I);

    // Arithmetic operators
    void visitSimpleBinary(BinaryOperator &B, unsigned OpcodeClass);
    void visitAdd(BinaryOperator &B) { visitSimpleBinary(B, 0); }
    void visitSub(BinaryOperator &B) { visitSimpleBinary(B, 1); }
    void doMultiply(MachineBasicBlock *MBB, MachineBasicBlock::iterator &MBBI,
                    unsigned DestReg, const Type *DestTy,
		    unsigned Op0Reg, unsigned Op1Reg);
    void visitMul(BinaryOperator &B);

    void visitDiv(BinaryOperator &B) { visitDivRem(B); }
    void visitRem(BinaryOperator &B) { visitDivRem(B); }
    void visitDivRem(BinaryOperator &B);

    // Bitwise operators
    void visitAnd(BinaryOperator &B) { visitSimpleBinary(B, 2); }
    void visitOr (BinaryOperator &B) { visitSimpleBinary(B, 3); }
    void visitXor(BinaryOperator &B) { visitSimpleBinary(B, 4); }

    // Comparison operators...
    void visitSetCondInst(SetCondInst &I);
    bool EmitComparisonGetSignedness(unsigned OpNum, Value *Op0, Value *Op1,
                                     MachineBasicBlock *MBB,
                                     MachineBasicBlock::iterator &MBBI);

    // Memory Instructions
    MachineInstr *doFPLoad(MachineBasicBlock *MBB,
			   MachineBasicBlock::iterator &MBBI,
			   const Type *Ty, unsigned DestReg);
    void visitLoadInst(LoadInst &I);
    void doFPStore(const Type *Ty, unsigned DestAddrReg, unsigned SrcReg);
    void visitStoreInst(StoreInst &I);
    void visitGetElementPtrInst(GetElementPtrInst &I);
    void visitAllocaInst(AllocaInst &I);
    void visitMallocInst(MallocInst &I);
    void visitFreeInst(FreeInst &I);
    
    // Other operators
    void visitShiftInst(ShiftInst &I);
    void visitPHINode(PHINode &I) {}      // PHI nodes handled by second pass
    void visitCastInst(CastInst &I);
    void visitVarArgInst(VarArgInst &I);

    void visitInstruction(Instruction &I) {
      std::cerr << "Cannot instruction select: " << I;
      abort();
    }

    /// promote32 - Make a value 32-bits wide, and put it somewhere.
    ///
    void promote32(unsigned targetReg, const ValueRecord &VR);

    /// EmitByteSwap - Byteswap SrcReg into DestReg.
    ///
    void EmitByteSwap(unsigned DestReg, unsigned SrcReg, unsigned Class);
    
    /// emitGEPOperation - Common code shared between visitGetElementPtrInst and
    /// constant expression GEP support.
    ///
    void emitGEPOperation(MachineBasicBlock *BB, MachineBasicBlock::iterator&IP,
                          Value *Src, User::op_iterator IdxBegin,
                          User::op_iterator IdxEnd, unsigned TargetReg);

    /// emitCastOperation - Common code shared between visitCastInst and
    /// constant expression cast support.
    void emitCastOperation(MachineBasicBlock *BB,MachineBasicBlock::iterator&IP,
                           Value *Src, const Type *DestTy, unsigned TargetReg);

    /// emitSimpleBinaryOperation - Common code shared between visitSimpleBinary
    /// and constant expression support.
    void emitSimpleBinaryOperation(MachineBasicBlock *BB,
                                   MachineBasicBlock::iterator &IP,
                                   Value *Op0, Value *Op1,
                                   unsigned OperatorClass, unsigned TargetReg);

    /// emitSetCCOperation - Common code shared between visitSetCondInst and
    /// constant expression support.
    void emitSetCCOperation(MachineBasicBlock *BB,
                            MachineBasicBlock::iterator &IP,
                            Value *Op0, Value *Op1, unsigned Opcode,
                            unsigned TargetReg);
 

    /// copyConstantToRegister - Output the instructions required to put the
    /// specified constant into the specified register.
    ///
    void copyConstantToRegister(MachineBasicBlock *MBB,
                                MachineBasicBlock::iterator &MBBI,
                                Constant *C, unsigned Reg);

    /// makeAnotherReg - This method returns the next register number we haven't
    /// yet used.
    ///
    /// Long values are handled somewhat specially.  They are always allocated
    /// as pairs of 32 bit integer values.  The register number returned is the
    /// lower 32 bits of the long value, and the regNum+1 is the upper 32 bits
    /// of the long value.
    ///
    unsigned makeAnotherReg(const Type *Ty) {
      assert(dynamic_cast<const X86RegisterInfo*>(TM.getRegisterInfo()) &&
             "Current target doesn't have X86 reg info??");
      const X86RegisterInfo *MRI =
        static_cast<const X86RegisterInfo*>(TM.getRegisterInfo());
      if (Ty == Type::LongTy || Ty == Type::ULongTy) {
	const TargetRegisterClass *RC = MRI->getRegClassForType(Type::IntTy);
	// Create the lower part
	F->getSSARegMap()->createVirtualRegister(RC);
	// Create the upper part.
	return F->getSSARegMap()->createVirtualRegister(RC)-1;
      }

      // Add the mapping of regnumber => reg class to MachineFunction
      const TargetRegisterClass *RC = MRI->getRegClassForType(Ty);
      return F->getSSARegMap()->createVirtualRegister(RC);
    }

    /// getReg - This method turns an LLVM value into a register number.  This
    /// is guaranteed to produce the same register number for a particular value
    /// every time it is queried.
    ///
    unsigned getReg(Value &V) { return getReg(&V); }  // Allow references
    unsigned getReg(Value *V) {
      // Just append to the end of the current bb.
      MachineBasicBlock::iterator It = BB->end();
      return getReg(V, BB, It);
    }
    unsigned getReg(Value *V, MachineBasicBlock *MBB,
                    MachineBasicBlock::iterator &IPt) {
      unsigned &Reg = RegMap[V];
      if (Reg == 0) {
        Reg = makeAnotherReg(V->getType());
        RegMap[V] = Reg;
      }

      // If this operand is a constant, emit the code to copy the constant into
      // the register here...
      //
      if (Constant *C = dyn_cast<Constant>(V)) {
        copyConstantToRegister(MBB, IPt, C, Reg);
        RegMap.erase(V);  // Assign a new name to this constant if ref'd again
      } else if (GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
        // Move the address of the global into the register
        BMI(MBB, IPt, X86::MOVir32, 1, Reg).addGlobalAddress(GV);
        RegMap.erase(V);  // Assign a new name to this address if ref'd again
      }

      return Reg;
    }
  };
}

/// TypeClass - Used by the X86 backend to group LLVM types by their basic X86
/// Representation.
///
enum TypeClass {
  cByte, cShort, cInt, cFP, cLong
};

/// getClass - Turn a primitive type into a "class" number which is based on the
/// size of the type, and whether or not it is floating point.
///
static inline TypeClass getClass(const Type *Ty) {
  switch (Ty->getPrimitiveID()) {
  case Type::SByteTyID:
  case Type::UByteTyID:   return cByte;      // Byte operands are class #0
  case Type::ShortTyID:
  case Type::UShortTyID:  return cShort;     // Short operands are class #1
  case Type::IntTyID:
  case Type::UIntTyID:
  case Type::PointerTyID: return cInt;       // Int's and pointers are class #2

  case Type::FloatTyID:
  case Type::DoubleTyID:  return cFP;        // Floating Point is #3

  case Type::LongTyID:
  case Type::ULongTyID:   return cLong;      // Longs are class #4
  default:
    assert(0 && "Invalid type to getClass!");
    return cByte;  // not reached
  }
}

// getClassB - Just like getClass, but treat boolean values as bytes.
static inline TypeClass getClassB(const Type *Ty) {
  if (Ty == Type::BoolTy) return cByte;
  return getClass(Ty);
}


/// copyConstantToRegister - Output the instructions required to put the
/// specified constant into the specified register.
///
void ISel::copyConstantToRegister(MachineBasicBlock *MBB,
                                  MachineBasicBlock::iterator &IP,
                                  Constant *C, unsigned R) {
  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
    unsigned Class = 0;
    switch (CE->getOpcode()) {
    case Instruction::GetElementPtr:
      emitGEPOperation(MBB, IP, CE->getOperand(0),
                       CE->op_begin()+1, CE->op_end(), R);
      return;
    case Instruction::Cast:
      emitCastOperation(MBB, IP, CE->getOperand(0), CE->getType(), R);
      return;

    case Instruction::Xor: ++Class; // FALL THROUGH
    case Instruction::Or:  ++Class; // FALL THROUGH
    case Instruction::And: ++Class; // FALL THROUGH
    case Instruction::Sub: ++Class; // FALL THROUGH
    case Instruction::Add:
      emitSimpleBinaryOperation(MBB, IP, CE->getOperand(0), CE->getOperand(1),
                                Class, R);
      return;

    case Instruction::SetNE:
    case Instruction::SetEQ:
    case Instruction::SetLT:
    case Instruction::SetGT:
    case Instruction::SetLE:
    case Instruction::SetGE:
      emitSetCCOperation(MBB, IP, CE->getOperand(0), CE->getOperand(1),
                         CE->getOpcode(), R);
      return;

    default:
      std::cerr << "Offending expr: " << C << "\n";
      assert(0 && "Constant expressions not yet handled!\n");
    }
  }

  if (C->getType()->isIntegral()) {
    unsigned Class = getClassB(C->getType());

    if (Class == cLong) {
      // Copy the value into the register pair.
      uint64_t Val = cast<ConstantInt>(C)->getRawValue();
      BMI(MBB, IP, X86::MOVir32, 1, R).addZImm(Val & 0xFFFFFFFF);
      BMI(MBB, IP, X86::MOVir32, 1, R+1).addZImm(Val >> 32);
      return;
    }

    assert(Class <= cInt && "Type not handled yet!");

    static const unsigned IntegralOpcodeTab[] = {
      X86::MOVir8, X86::MOVir16, X86::MOVir32
    };

    if (C->getType() == Type::BoolTy) {
      BMI(MBB, IP, X86::MOVir8, 1, R).addZImm(C == ConstantBool::True);
    } else {
      ConstantInt *CI = cast<ConstantInt>(C);
      BMI(MBB, IP, IntegralOpcodeTab[Class], 1, R).addZImm(CI->getRawValue());
    }
  } else if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
    double Value = CFP->getValue();
    if (Value == +0.0)
      BMI(MBB, IP, X86::FLD0, 0, R);
    else if (Value == +1.0)
      BMI(MBB, IP, X86::FLD1, 0, R);
    else {
      // Otherwise we need to spill the constant to memory...
      MachineConstantPool *CP = F->getConstantPool();
      unsigned CPI = CP->getConstantPoolIndex(CFP);
      addConstantPoolReference(doFPLoad(MBB, IP, CFP->getType(), R), CPI);
    }

  } else if (isa<ConstantPointerNull>(C)) {
    // Copy zero (null pointer) to the register.
    BMI(MBB, IP, X86::MOVir32, 1, R).addZImm(0);
  } else if (ConstantPointerRef *CPR = dyn_cast<ConstantPointerRef>(C)) {
    unsigned SrcReg = getReg(CPR->getValue(), MBB, IP);
    BMI(MBB, IP, X86::MOVrr32, 1, R).addReg(SrcReg);
  } else {
    std::cerr << "Offending constant: " << C << "\n";
    assert(0 && "Type not handled yet!");
  }
}

/// LoadArgumentsToVirtualRegs - Load all of the arguments to this function from
/// the stack into virtual registers.
///
void ISel::LoadArgumentsToVirtualRegs(Function &Fn) {
  // Emit instructions to load the arguments...  On entry to a function on the
  // X86, the stack frame looks like this:
  //
  // [ESP] -- return address
  // [ESP + 4] -- first argument (leftmost lexically)
  // [ESP + 8] -- second argument, if first argument is four bytes in size
  //    ... 
  //
  unsigned ArgOffset = 0;   // Frame mechanisms handle retaddr slot
  MachineFrameInfo *MFI = F->getFrameInfo();

  for (Function::aiterator I = Fn.abegin(), E = Fn.aend(); I != E; ++I) {
    unsigned Reg = getReg(*I);
    
    int FI;          // Frame object index
    switch (getClassB(I->getType())) {
    case cByte:
      FI = MFI->CreateFixedObject(1, ArgOffset);
      addFrameReference(BuildMI(BB, X86::MOVmr8, 4, Reg), FI);
      break;
    case cShort:
      FI = MFI->CreateFixedObject(2, ArgOffset);
      addFrameReference(BuildMI(BB, X86::MOVmr16, 4, Reg), FI);
      break;
    case cInt:
      FI = MFI->CreateFixedObject(4, ArgOffset);
      addFrameReference(BuildMI(BB, X86::MOVmr32, 4, Reg), FI);
      break;
    case cLong:
      FI = MFI->CreateFixedObject(8, ArgOffset);
      addFrameReference(BuildMI(BB, X86::MOVmr32, 4, Reg), FI);
      addFrameReference(BuildMI(BB, X86::MOVmr32, 4, Reg+1), FI, 4);
      ArgOffset += 4;   // longs require 4 additional bytes
      break;
    case cFP:
      unsigned Opcode;
      if (I->getType() == Type::FloatTy) {
	Opcode = X86::FLDr32;
	FI = MFI->CreateFixedObject(4, ArgOffset);
      } else {
	Opcode = X86::FLDr64;
	FI = MFI->CreateFixedObject(8, ArgOffset);
	ArgOffset += 4;   // doubles require 4 additional bytes
      }
      addFrameReference(BuildMI(BB, Opcode, 4, Reg), FI);
      break;
    default:
      assert(0 && "Unhandled argument type!");
    }
    ArgOffset += 4;  // Each argument takes at least 4 bytes on the stack...
  }

  // If the function takes variable number of arguments, add a frame offset for
  // the start of the first vararg value... this is used to expand
  // llvm.va_start.
  if (Fn.getFunctionType()->isVarArg())
    VarArgsFrameIndex = MFI->CreateFixedObject(1, ArgOffset);
}


/// SelectPHINodes - Insert machine code to generate phis.  This is tricky
/// because we have to generate our sources into the source basic blocks, not
/// the current one.
///
void ISel::SelectPHINodes() {
  const TargetInstrInfo &TII = TM.getInstrInfo();
  const Function &LF = *F->getFunction();  // The LLVM function...
  for (Function::const_iterator I = LF.begin(), E = LF.end(); I != E; ++I) {
    const BasicBlock *BB = I;
    MachineBasicBlock *MBB = MBBMap[I];

    // Loop over all of the PHI nodes in the LLVM basic block...
    unsigned NumPHIs = 0;
    for (BasicBlock::const_iterator I = BB->begin();
         PHINode *PN = (PHINode*)dyn_cast<PHINode>(I); ++I) {

      // Create a new machine instr PHI node, and insert it.
      unsigned PHIReg = getReg(*PN);
      MachineInstr *PhiMI = BuildMI(X86::PHI, PN->getNumOperands(), PHIReg);
      MBB->insert(MBB->begin()+NumPHIs++, PhiMI);

      MachineInstr *LongPhiMI = 0;
      if (PN->getType() == Type::LongTy || PN->getType() == Type::ULongTy) {
	LongPhiMI = BuildMI(X86::PHI, PN->getNumOperands(), PHIReg+1);
	MBB->insert(MBB->begin()+NumPHIs++, LongPhiMI);
      }

      // PHIValues - Map of blocks to incoming virtual registers.  We use this
      // so that we only initialize one incoming value for a particular block,
      // even if the block has multiple entries in the PHI node.
      //
      std::map<MachineBasicBlock*, unsigned> PHIValues;

      for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
        MachineBasicBlock *PredMBB = MBBMap[PN->getIncomingBlock(i)];
        unsigned ValReg;
        std::map<MachineBasicBlock*, unsigned>::iterator EntryIt =
          PHIValues.lower_bound(PredMBB);

        if (EntryIt != PHIValues.end() && EntryIt->first == PredMBB) {
          // We already inserted an initialization of the register for this
          // predecessor.  Recycle it.
          ValReg = EntryIt->second;

        } else {        
          // Get the incoming value into a virtual register.  If it is not
          // already available in a virtual register, insert the computation
          // code into PredMBB
          //
          MachineBasicBlock::iterator PI = PredMBB->end();
          while (PI != PredMBB->begin() &&
                 TII.isTerminatorInstr((*(PI-1))->getOpcode()))
            --PI;
          ValReg = getReg(PN->getIncomingValue(i), PredMBB, PI);

          // Remember that we inserted a value for this PHI for this predecessor
          PHIValues.insert(EntryIt, std::make_pair(PredMBB, ValReg));
        }

	PhiMI->addRegOperand(ValReg);
        PhiMI->addMachineBasicBlockOperand(PredMBB);
	if (LongPhiMI) {
	  LongPhiMI->addRegOperand(ValReg+1);
	  LongPhiMI->addMachineBasicBlockOperand(PredMBB);
	}
      }
    }
  }
}

// canFoldSetCCIntoBranch - Return the setcc instruction if we can fold it into
// the conditional branch instruction which is the only user of the cc
// instruction.  This is the case if the conditional branch is the only user of
// the setcc, and if the setcc is in the same basic block as the conditional
// branch.  We also don't handle long arguments below, so we reject them here as
// well.
//
static SetCondInst *canFoldSetCCIntoBranch(Value *V) {
  if (SetCondInst *SCI = dyn_cast<SetCondInst>(V))
    if (SCI->hasOneUse() && isa<BranchInst>(SCI->use_back()) &&
        SCI->getParent() == cast<BranchInst>(SCI->use_back())->getParent()) {
      const Type *Ty = SCI->getOperand(0)->getType();
      if (Ty != Type::LongTy && Ty != Type::ULongTy)
        return SCI;
    }
  return 0;
}

// Return a fixed numbering for setcc instructions which does not depend on the
// order of the opcodes.
//
static unsigned getSetCCNumber(unsigned Opcode) {
  switch(Opcode) {
  default: assert(0 && "Unknown setcc instruction!");
  case Instruction::SetEQ: return 0;
  case Instruction::SetNE: return 1;
  case Instruction::SetLT: return 2;
  case Instruction::SetGE: return 3;
  case Instruction::SetGT: return 4;
  case Instruction::SetLE: return 5;
  }
}

// LLVM  -> X86 signed  X86 unsigned
// -----    ----------  ------------
// seteq -> sete        sete
// setne -> setne       setne
// setlt -> setl        setb
// setge -> setge       setae
// setgt -> setg        seta
// setle -> setle       setbe
static const unsigned SetCCOpcodeTab[2][6] = {
  {X86::SETEr, X86::SETNEr, X86::SETBr, X86::SETAEr, X86::SETAr, X86::SETBEr},
  {X86::SETEr, X86::SETNEr, X86::SETLr, X86::SETGEr, X86::SETGr, X86::SETLEr},
};

bool ISel::EmitComparisonGetSignedness(unsigned OpNum, Value *Op0, Value *Op1,
                                       MachineBasicBlock *MBB,
                                       MachineBasicBlock::iterator &IP) {
  // The arguments are already supposed to be of the same type.
  const Type *CompTy = Op0->getType();
  bool isSigned = CompTy->isSigned();
  unsigned Class = getClassB(CompTy);
  unsigned Op0r = getReg(Op0, MBB, IP);

  // Special case handling of: cmp R, i
  if (Class == cByte || Class == cShort || Class == cInt)
    if (ConstantInt *CI = dyn_cast<ConstantInt>(Op1)) {
      uint64_t Op1v = cast<ConstantInt>(CI)->getRawValue();

      // Mask off any upper bits of the constant, if there are any...
      Op1v &= (1ULL << (8 << Class)) - 1;

      switch (Class) {
      case cByte:  BMI(MBB,IP, X86::CMPri8, 2).addReg(Op0r).addZImm(Op1v);break;
      case cShort: BMI(MBB,IP, X86::CMPri16,2).addReg(Op0r).addZImm(Op1v);break;
      case cInt:   BMI(MBB,IP, X86::CMPri32,2).addReg(Op0r).addZImm(Op1v);break;
      default:
        assert(0 && "Invalid class!");
      }
      return isSigned;
    }

  unsigned Op1r = getReg(Op1, MBB, IP);
  switch (Class) {
  default: assert(0 && "Unknown type class!");
    // Emit: cmp <var1>, <var2> (do the comparison).  We can
    // compare 8-bit with 8-bit, 16-bit with 16-bit, 32-bit with
    // 32-bit.
  case cByte:
    BMI(MBB, IP, X86::CMPrr8, 2).addReg(Op0r).addReg(Op1r);
    break;
  case cShort:
    BMI(MBB, IP, X86::CMPrr16, 2).addReg(Op0r).addReg(Op1r);
    break;
  case cInt:
    BMI(MBB, IP, X86::CMPrr32, 2).addReg(Op0r).addReg(Op1r);
    break;
  case cFP:
    BMI(MBB, IP, X86::FpUCOM, 2).addReg(Op0r).addReg(Op1r);
    BMI(MBB, IP, X86::FNSTSWr8, 0);
    BMI(MBB, IP, X86::SAHF, 1);
    isSigned = false;   // Compare with unsigned operators
    break;

  case cLong:
    if (OpNum < 2) {    // seteq, setne
      unsigned LoTmp = makeAnotherReg(Type::IntTy);
      unsigned HiTmp = makeAnotherReg(Type::IntTy);
      unsigned FinalTmp = makeAnotherReg(Type::IntTy);
      BMI(MBB, IP, X86::XORrr32, 2, LoTmp).addReg(Op0r).addReg(Op1r);
      BMI(MBB, IP, X86::XORrr32, 2, HiTmp).addReg(Op0r+1).addReg(Op1r+1);
      BMI(MBB, IP, X86::ORrr32,  2, FinalTmp).addReg(LoTmp).addReg(HiTmp);
      break;  // Allow the sete or setne to be generated from flags set by OR
    } else {
      // Emit a sequence of code which compares the high and low parts once
      // each, then uses a conditional move to handle the overflow case.  For
      // example, a setlt for long would generate code like this:
      //
      // AL = lo(op1) < lo(op2)   // Signedness depends on operands
      // BL = hi(op1) < hi(op2)   // Always unsigned comparison
      // dest = hi(op1) == hi(op2) ? AL : BL;
      //

      // FIXME: This would be much better if we had hierarchical register
      // classes!  Until then, hardcode registers so that we can deal with their
      // aliases (because we don't have conditional byte moves).
      //
      BMI(MBB, IP, X86::CMPrr32, 2).addReg(Op0r).addReg(Op1r);
      BMI(MBB, IP, SetCCOpcodeTab[0][OpNum], 0, X86::AL);
      BMI(MBB, IP, X86::CMPrr32, 2).addReg(Op0r+1).addReg(Op1r+1);
      BMI(MBB, IP, SetCCOpcodeTab[isSigned][OpNum], 0, X86::BL);
      BMI(MBB, IP, X86::IMPLICIT_DEF, 0, X86::BH);
      BMI(MBB, IP, X86::IMPLICIT_DEF, 0, X86::AH);
      BMI(MBB, IP, X86::CMOVErr16, 2, X86::BX).addReg(X86::BX).addReg(X86::AX);
      // NOTE: visitSetCondInst knows that the value is dumped into the BL
      // register at this point for long values...
      return isSigned;
    }
  }
  return isSigned;
}


/// SetCC instructions - Here we just emit boilerplate code to set a byte-sized
/// register, then move it to wherever the result should be. 
///
void ISel::visitSetCondInst(SetCondInst &I) {
  if (canFoldSetCCIntoBranch(&I)) return;  // Fold this into a branch...

  unsigned DestReg = getReg(I);
  MachineBasicBlock::iterator MII = BB->end();
  emitSetCCOperation(BB, MII, I.getOperand(0), I.getOperand(1), I.getOpcode(),
                     DestReg);
}

/// emitSetCCOperation - Common code shared between visitSetCondInst and
/// constant expression support.
void ISel::emitSetCCOperation(MachineBasicBlock *MBB,
                              MachineBasicBlock::iterator &IP,
                              Value *Op0, Value *Op1, unsigned Opcode,
                              unsigned TargetReg) {
  unsigned OpNum = getSetCCNumber(Opcode);
  bool isSigned = EmitComparisonGetSignedness(OpNum, Op0, Op1, MBB, IP);

  if (getClassB(Op0->getType()) != cLong || OpNum < 2) {
    // Handle normal comparisons with a setcc instruction...
    BMI(MBB, IP, SetCCOpcodeTab[isSigned][OpNum], 0, TargetReg);
  } else {
    // Handle long comparisons by copying the value which is already in BL into
    // the register we want...
    BMI(MBB, IP, X86::MOVrr8, 1, TargetReg).addReg(X86::BL);
  }
}




/// promote32 - Emit instructions to turn a narrow operand into a 32-bit-wide
/// operand, in the specified target register.
void ISel::promote32(unsigned targetReg, const ValueRecord &VR) {
  bool isUnsigned = VR.Ty->isUnsigned();

  // Make sure we have the register number for this value...
  unsigned Reg = VR.Val ? getReg(VR.Val) : VR.Reg;

  switch (getClassB(VR.Ty)) {
  case cByte:
    // Extend value into target register (8->32)
    if (isUnsigned)
      BuildMI(BB, X86::MOVZXr32r8, 1, targetReg).addReg(Reg);
    else
      BuildMI(BB, X86::MOVSXr32r8, 1, targetReg).addReg(Reg);
    break;
  case cShort:
    // Extend value into target register (16->32)
    if (isUnsigned)
      BuildMI(BB, X86::MOVZXr32r16, 1, targetReg).addReg(Reg);
    else
      BuildMI(BB, X86::MOVSXr32r16, 1, targetReg).addReg(Reg);
    break;
  case cInt:
    // Move value into target register (32->32)
    BuildMI(BB, X86::MOVrr32, 1, targetReg).addReg(Reg);
    break;
  default:
    assert(0 && "Unpromotable operand class in promote32");
  }
}

/// 'ret' instruction - Here we are interested in meeting the x86 ABI.  As such,
/// we have the following possibilities:
///
///   ret void: No return value, simply emit a 'ret' instruction
///   ret sbyte, ubyte : Extend value into EAX and return
///   ret short, ushort: Extend value into EAX and return
///   ret int, uint    : Move value into EAX and return
///   ret pointer      : Move value into EAX and return
///   ret long, ulong  : Move value into EAX/EDX and return
///   ret float/double : Top of FP stack
///
void ISel::visitReturnInst(ReturnInst &I) {
  if (I.getNumOperands() == 0) {
    BuildMI(BB, X86::RET, 0); // Just emit a 'ret' instruction
    return;
  }

  Value *RetVal = I.getOperand(0);
  unsigned RetReg = getReg(RetVal);
  switch (getClassB(RetVal->getType())) {
  case cByte:   // integral return values: extend or move into EAX and return
  case cShort:
  case cInt:
    promote32(X86::EAX, ValueRecord(RetReg, RetVal->getType()));
    // Declare that EAX is live on exit
    BuildMI(BB, X86::IMPLICIT_USE, 2).addReg(X86::EAX).addReg(X86::ESP);
    break;
  case cFP:                   // Floats & Doubles: Return in ST(0)
    BuildMI(BB, X86::FpSETRESULT, 1).addReg(RetReg);
    // Declare that top-of-stack is live on exit
    BuildMI(BB, X86::IMPLICIT_USE, 2).addReg(X86::ST0).addReg(X86::ESP);
    break;
  case cLong:
    BuildMI(BB, X86::MOVrr32, 1, X86::EAX).addReg(RetReg);
    BuildMI(BB, X86::MOVrr32, 1, X86::EDX).addReg(RetReg+1);
    // Declare that EAX & EDX are live on exit
    BuildMI(BB, X86::IMPLICIT_USE, 3).addReg(X86::EAX).addReg(X86::EDX).addReg(X86::ESP);
    break;
  default:
    visitInstruction(I);
  }
  // Emit a 'ret' instruction
  BuildMI(BB, X86::RET, 0);
}

// getBlockAfter - Return the basic block which occurs lexically after the
// specified one.
static inline BasicBlock *getBlockAfter(BasicBlock *BB) {
  Function::iterator I = BB; ++I;  // Get iterator to next block
  return I != BB->getParent()->end() ? &*I : 0;
}

/// visitBranchInst - Handle conditional and unconditional branches here.  Note
/// that since code layout is frozen at this point, that if we are trying to
/// jump to a block that is the immediate successor of the current block, we can
/// just make a fall-through (but we don't currently).
///
void ISel::visitBranchInst(BranchInst &BI) {
  BasicBlock *NextBB = getBlockAfter(BI.getParent());  // BB after current one

  if (!BI.isConditional()) {  // Unconditional branch?
    if (BI.getSuccessor(0) != NextBB)
      BuildMI(BB, X86::JMP, 1).addPCDisp(BI.getSuccessor(0));
    return;
  }

  // See if we can fold the setcc into the branch itself...
  SetCondInst *SCI = canFoldSetCCIntoBranch(BI.getCondition());
  if (SCI == 0) {
    // Nope, cannot fold setcc into this branch.  Emit a branch on a condition
    // computed some other way...
    unsigned condReg = getReg(BI.getCondition());
    BuildMI(BB, X86::CMPri8, 2).addReg(condReg).addZImm(0);
    if (BI.getSuccessor(1) == NextBB) {
      if (BI.getSuccessor(0) != NextBB)
        BuildMI(BB, X86::JNE, 1).addPCDisp(BI.getSuccessor(0));
    } else {
      BuildMI(BB, X86::JE, 1).addPCDisp(BI.getSuccessor(1));
      
      if (BI.getSuccessor(0) != NextBB)
        BuildMI(BB, X86::JMP, 1).addPCDisp(BI.getSuccessor(0));
    }
    return;
  }

  unsigned OpNum = getSetCCNumber(SCI->getOpcode());
  MachineBasicBlock::iterator MII = BB->end();
  bool isSigned = EmitComparisonGetSignedness(OpNum, SCI->getOperand(0),
                                              SCI->getOperand(1), BB, MII);
  
  // LLVM  -> X86 signed  X86 unsigned
  // -----    ----------  ------------
  // seteq -> je          je
  // setne -> jne         jne
  // setlt -> jl          jb
  // setge -> jge         jae
  // setgt -> jg          ja
  // setle -> jle         jbe
  static const unsigned OpcodeTab[2][6] = {
    { X86::JE, X86::JNE, X86::JB, X86::JAE, X86::JA, X86::JBE },
    { X86::JE, X86::JNE, X86::JL, X86::JGE, X86::JG, X86::JLE },
  };
  
  if (BI.getSuccessor(0) != NextBB) {
    BuildMI(BB, OpcodeTab[isSigned][OpNum], 1).addPCDisp(BI.getSuccessor(0));
    if (BI.getSuccessor(1) != NextBB)
      BuildMI(BB, X86::JMP, 1).addPCDisp(BI.getSuccessor(1));
  } else {
    // Change to the inverse condition...
    if (BI.getSuccessor(1) != NextBB) {
      OpNum ^= 1;
      BuildMI(BB, OpcodeTab[isSigned][OpNum], 1).addPCDisp(BI.getSuccessor(1));
    }
  }
}


/// doCall - This emits an abstract call instruction, setting up the arguments
/// and the return value as appropriate.  For the actual function call itself,
/// it inserts the specified CallMI instruction into the stream.
///
void ISel::doCall(const ValueRecord &Ret, MachineInstr *CallMI,
		  const std::vector<ValueRecord> &Args) {

  // Count how many bytes are to be pushed on the stack...
  unsigned NumBytes = 0;

  if (!Args.empty()) {
    for (unsigned i = 0, e = Args.size(); i != e; ++i)
      switch (getClassB(Args[i].Ty)) {
      case cByte: case cShort: case cInt:
	NumBytes += 4; break;
      case cLong:
	NumBytes += 8; break;
      case cFP:
	NumBytes += Args[i].Ty == Type::FloatTy ? 4 : 8;
	break;
      default: assert(0 && "Unknown class!");
      }

    // Adjust the stack pointer for the new arguments...
    BuildMI(BB, X86::ADJCALLSTACKDOWN, 1).addZImm(NumBytes);

    // Arguments go on the stack in reverse order, as specified by the ABI.
    unsigned ArgOffset = 0;
    for (unsigned i = 0, e = Args.size(); i != e; ++i) {
      unsigned ArgReg = Args[i].Val ? getReg(Args[i].Val) : Args[i].Reg;
      switch (getClassB(Args[i].Ty)) {
      case cByte:
      case cShort: {
	// Promote arg to 32 bits wide into a temporary register...
	unsigned R = makeAnotherReg(Type::UIntTy);
	promote32(R, Args[i]);
	addRegOffset(BuildMI(BB, X86::MOVrm32, 5),
		     X86::ESP, ArgOffset).addReg(R);
	break;
      }
      case cInt:
	addRegOffset(BuildMI(BB, X86::MOVrm32, 5),
		     X86::ESP, ArgOffset).addReg(ArgReg);
	break;
      case cLong:
	addRegOffset(BuildMI(BB, X86::MOVrm32, 5),
		     X86::ESP, ArgOffset).addReg(ArgReg);
	addRegOffset(BuildMI(BB, X86::MOVrm32, 5),
		     X86::ESP, ArgOffset+4).addReg(ArgReg+1);
	ArgOffset += 4;        // 8 byte entry, not 4.
	break;
	
      case cFP:
	if (Args[i].Ty == Type::FloatTy) {
	  addRegOffset(BuildMI(BB, X86::FSTr32, 5),
		       X86::ESP, ArgOffset).addReg(ArgReg);
	} else {
	  assert(Args[i].Ty == Type::DoubleTy && "Unknown FP type!");
	  addRegOffset(BuildMI(BB, X86::FSTr64, 5),
		       X86::ESP, ArgOffset).addReg(ArgReg);
	  ArgOffset += 4;       // 8 byte entry, not 4.
	}
	break;

      default: assert(0 && "Unknown class!");
      }
      ArgOffset += 4;
    }
  } else {
    BuildMI(BB, X86::ADJCALLSTACKDOWN, 1).addZImm(0);
  }

  BB->push_back(CallMI);

  BuildMI(BB, X86::ADJCALLSTACKUP, 1).addZImm(NumBytes);

  // If there is a return value, scavenge the result from the location the call
  // leaves it in...
  //
  if (Ret.Ty != Type::VoidTy) {
    unsigned DestClass = getClassB(Ret.Ty);
    switch (DestClass) {
    case cByte:
    case cShort:
    case cInt: {
      // Integral results are in %eax, or the appropriate portion
      // thereof.
      static const unsigned regRegMove[] = {
	X86::MOVrr8, X86::MOVrr16, X86::MOVrr32
      };
      static const unsigned AReg[] = { X86::AL, X86::AX, X86::EAX };
      BuildMI(BB, regRegMove[DestClass], 1, Ret.Reg).addReg(AReg[DestClass]);
      break;
    }
    case cFP:     // Floating-point return values live in %ST(0)
      BuildMI(BB, X86::FpGETRESULT, 1, Ret.Reg);
      break;
    case cLong:   // Long values are left in EDX:EAX
      BuildMI(BB, X86::MOVrr32, 1, Ret.Reg).addReg(X86::EAX);
      BuildMI(BB, X86::MOVrr32, 1, Ret.Reg+1).addReg(X86::EDX);
      break;
    default: assert(0 && "Unknown class!");
    }
  }
}


/// visitCallInst - Push args on stack and do a procedure call instruction.
void ISel::visitCallInst(CallInst &CI) {
  MachineInstr *TheCall;
  if (Function *F = CI.getCalledFunction()) {
    // Is it an intrinsic function call?
    if (LLVMIntrinsic::ID ID = (LLVMIntrinsic::ID)F->getIntrinsicID()) {
      visitIntrinsicCall(ID, CI);   // Special intrinsics are not handled here
      return;
    }

    // Emit a CALL instruction with PC-relative displacement.
    TheCall = BuildMI(X86::CALLpcrel32, 1).addGlobalAddress(F, true);
  } else {  // Emit an indirect call...
    unsigned Reg = getReg(CI.getCalledValue());
    TheCall = BuildMI(X86::CALLr32, 1).addReg(Reg);
  }

  std::vector<ValueRecord> Args;
  for (unsigned i = 1, e = CI.getNumOperands(); i != e; ++i)
    Args.push_back(ValueRecord(CI.getOperand(i)));

  unsigned DestReg = CI.getType() != Type::VoidTy ? getReg(CI) : 0;
  doCall(ValueRecord(DestReg, CI.getType()), TheCall, Args);
}	 


void ISel::visitIntrinsicCall(LLVMIntrinsic::ID ID, CallInst &CI) {
  unsigned TmpReg1, TmpReg2;
  switch (ID) {