X86ISelLowering.cpp

  SDValue CWD2 =
    DAG.getNode(ISD::SRL, DL, MVT::i16,
                DAG.getNode(ISD::AND, DL, MVT::i16,
                            CWD, DAG.getConstant(0x400, MVT::i16)),
                DAG.getConstant(9, MVT::i8));

  SDValue RetVal =
    DAG.getNode(ISD::AND, DL, MVT::i16,
                DAG.getNode(ISD::ADD, DL, MVT::i16,
                            DAG.getNode(ISD::OR, DL, MVT::i16, CWD1, CWD2),
                            DAG.getConstant(1, MVT::i16)),
                DAG.getConstant(3, MVT::i16));


  return DAG.getNode((VT.getSizeInBits() < 16 ?
                      ISD::TRUNCATE : ISD::ZERO_EXTEND), DL, VT, RetVal);
}

SDValue X86TargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) const {
  EVT VT = Op.getValueType();
  EVT OpVT = VT;
  unsigned NumBits = VT.getSizeInBits();
  DebugLoc dl = Op.getDebugLoc();

  Op = Op.getOperand(0);
  if (VT == MVT::i8) {
    // Zero extend to i32 since there is not an i8 bsr.
    OpVT = MVT::i32;
    Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
  }

  // Issue a bsr (scan bits in reverse) which also sets EFLAGS.
  SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
  Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);

  // If src is zero (i.e. bsr sets ZF), returns NumBits.
  SDValue Ops[] = {
    Op,
    DAG.getConstant(NumBits+NumBits-1, OpVT),
    DAG.getConstant(X86::COND_E, MVT::i8),
    Op.getValue(1)
  };
  Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops, array_lengthof(Ops));

  // Finally xor with NumBits-1.
  Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT));

  if (VT == MVT::i8)
    Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
  return Op;
}

SDValue X86TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
  EVT VT = Op.getValueType();
  EVT OpVT = VT;
  unsigned NumBits = VT.getSizeInBits();
  DebugLoc dl = Op.getDebugLoc();

  Op = Op.getOperand(0);
  if (VT == MVT::i8) {
    OpVT = MVT::i32;
    Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
  }

  // Issue a bsf (scan bits forward) which also sets EFLAGS.
  SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
  Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op);

  // If src is zero (i.e. bsf sets ZF), returns NumBits.
  SDValue Ops[] = {
    Op,
    DAG.getConstant(NumBits, OpVT),
    DAG.getConstant(X86::COND_E, MVT::i8),
    Op.getValue(1)
  };
  Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops, array_lengthof(Ops));

  if (VT == MVT::i8)
    Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
  return Op;
}

SDValue X86TargetLowering::LowerMUL_V2I64(SDValue Op, SelectionDAG &DAG) const {
  EVT VT = Op.getValueType();
  assert(VT == MVT::v2i64 && "Only know how to lower V2I64 multiply");
  DebugLoc dl = Op.getDebugLoc();

  //  ulong2 Ahi = __builtin_ia32_psrlqi128( a, 32);
  //  ulong2 Bhi = __builtin_ia32_psrlqi128( b, 32);
  //  ulong2 AloBlo = __builtin_ia32_pmuludq128( a, b );
  //  ulong2 AloBhi = __builtin_ia32_pmuludq128( a, Bhi );
  //  ulong2 AhiBlo = __builtin_ia32_pmuludq128( Ahi, b );
  //
  //  AloBhi = __builtin_ia32_psllqi128( AloBhi, 32 );
  //  AhiBlo = __builtin_ia32_psllqi128( AhiBlo, 32 );
  //  return AloBlo + AloBhi + AhiBlo;

  SDValue A = Op.getOperand(0);
  SDValue B = Op.getOperand(1);

  SDValue Ahi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                       DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32),
                       A, DAG.getConstant(32, MVT::i32));
  SDValue Bhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                       DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32),
                       B, DAG.getConstant(32, MVT::i32));
  SDValue AloBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                       DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32),
                       A, B);
  SDValue AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                       DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32),
                       A, Bhi);
  SDValue AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                       DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32),
                       Ahi, B);
  AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                       DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32),
                       AloBhi, DAG.getConstant(32, MVT::i32));
  AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                       DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32),
                       AhiBlo, DAG.getConstant(32, MVT::i32));
  SDValue Res = DAG.getNode(ISD::ADD, dl, VT, AloBlo, AloBhi);
  Res = DAG.getNode(ISD::ADD, dl, VT, Res, AhiBlo);
  return Res;
}

SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const {

  EVT VT = Op.getValueType();
  DebugLoc dl = Op.getDebugLoc();
  SDValue R = Op.getOperand(0);
  SDValue Amt = Op.getOperand(1);

  LLVMContext *Context = DAG.getContext();

  // Must have SSE2.
  if (!Subtarget->hasSSE2()) return SDValue();

  // Optimize shl/srl/sra with constant shift amount.
  if (isSplatVector(Amt.getNode())) {
    SDValue SclrAmt = Amt->getOperand(0);
    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(SclrAmt)) {
      uint64_t ShiftAmt = C->getZExtValue();

      if (VT == MVT::v2i64 && Op.getOpcode() == ISD::SHL)
       return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                     DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32),
                     R, DAG.getConstant(ShiftAmt, MVT::i32));

      if (VT == MVT::v4i32 && Op.getOpcode() == ISD::SHL)
       return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                     DAG.getConstant(Intrinsic::x86_sse2_pslli_d, MVT::i32),
                     R, DAG.getConstant(ShiftAmt, MVT::i32));

      if (VT == MVT::v8i16 && Op.getOpcode() == ISD::SHL)
       return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                     DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32),
                     R, DAG.getConstant(ShiftAmt, MVT::i32));

      if (VT == MVT::v2i64 && Op.getOpcode() == ISD::SRL)
       return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                     DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32),
                     R, DAG.getConstant(ShiftAmt, MVT::i32));

      if (VT == MVT::v4i32 && Op.getOpcode() == ISD::SRL)
       return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                     DAG.getConstant(Intrinsic::x86_sse2_psrli_d, MVT::i32),
                     R, DAG.getConstant(ShiftAmt, MVT::i32));

      if (VT == MVT::v8i16 && Op.getOpcode() == ISD::SRL)
       return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                     DAG.getConstant(Intrinsic::x86_sse2_psrli_w, MVT::i32),
                     R, DAG.getConstant(ShiftAmt, MVT::i32));

      if (VT == MVT::v4i32 && Op.getOpcode() == ISD::SRA)
       return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                     DAG.getConstant(Intrinsic::x86_sse2_psrai_d, MVT::i32),
                     R, DAG.getConstant(ShiftAmt, MVT::i32));

      if (VT == MVT::v8i16 && Op.getOpcode() == ISD::SRA)
       return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                     DAG.getConstant(Intrinsic::x86_sse2_psrai_w, MVT::i32),
                     R, DAG.getConstant(ShiftAmt, MVT::i32));
    }
  }

  // Lower SHL with variable shift amount.
  // Cannot lower SHL without SSE2 or later.
  if (!Subtarget->hasSSE2()) return SDValue();

  if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) {
    Op = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                     DAG.getConstant(Intrinsic::x86_sse2_pslli_d, MVT::i32),
                     Op.getOperand(1), DAG.getConstant(23, MVT::i32));

    ConstantInt *CI = ConstantInt::get(*Context, APInt(32, 0x3f800000U));

    std::vector<Constant*> CV(4, CI);
    Constant *C = ConstantVector::get(CV);
    SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16);
    SDValue Addend = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
                                 MachinePointerInfo::getConstantPool(),
                                 false, false, 16);

    Op = DAG.getNode(ISD::ADD, dl, VT, Op, Addend);
    Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, Op);
    Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op);
    return DAG.getNode(ISD::MUL, dl, VT, Op, R);
  }
  if (VT == MVT::v16i8 && Op->getOpcode() == ISD::SHL) {
    // a = a << 5;
    Op = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                     DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32),
                     Op.getOperand(1), DAG.getConstant(5, MVT::i32));

    ConstantInt *CM1 = ConstantInt::get(*Context, APInt(8, 15));
    ConstantInt *CM2 = ConstantInt::get(*Context, APInt(8, 63));

    std::vector<Constant*> CVM1(16, CM1);
    std::vector<Constant*> CVM2(16, CM2);
    Constant *C = ConstantVector::get(CVM1);
    SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16);
    SDValue M = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
                            MachinePointerInfo::getConstantPool(),
                            false, false, 16);

    // r = pblendv(r, psllw(r & (char16)15, 4), a);
    M = DAG.getNode(ISD::AND, dl, VT, R, M);
    M = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                    DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), M,
                    DAG.getConstant(4, MVT::i32));
    R = DAG.getNode(X86ISD::PBLENDVB, dl, VT, R, M, Op);
    // a += a
    Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);

    C = ConstantVector::get(CVM2);
    CPIdx = DAG.getConstantPool(C, getPointerTy(), 16);
    M = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
                    MachinePointerInfo::getConstantPool(),
                    false, false, 16);

    // r = pblendv(r, psllw(r & (char16)63, 2), a);
    M = DAG.getNode(ISD::AND, dl, VT, R, M);
    M = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                    DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), M,
                    DAG.getConstant(2, MVT::i32));
    R = DAG.getNode(X86ISD::PBLENDVB, dl, VT, R, M, Op);
    // a += a
    Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);

    // return pblendv(r, r+r, a);
    R = DAG.getNode(X86ISD::PBLENDVB, dl, VT,
                    R, DAG.getNode(ISD::ADD, dl, VT, R, R), Op);
    return R;
  }
  return SDValue();
}

SDValue X86TargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) const {
  // Lower the "add/sub/mul with overflow" instruction into a regular ins plus
  // a "setcc" instruction that checks the overflow flag. The "brcond" lowering
  // looks for this combo and may remove the "setcc" instruction if the "setcc"
  // has only one use.
  SDNode *N = Op.getNode();
  SDValue LHS = N->getOperand(0);
  SDValue RHS = N->getOperand(1);
  unsigned BaseOp = 0;
  unsigned Cond = 0;
  DebugLoc DL = Op.getDebugLoc();
  switch (Op.getOpcode()) {
  default: llvm_unreachable("Unknown ovf instruction!");
  case ISD::SADDO:
    // A subtract of one will be selected as a INC. Note that INC doesn't
    // set CF, so we can't do this for UADDO.
    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS))
      if (C->isOne()) {
        BaseOp = X86ISD::INC;
        Cond = X86::COND_O;
        break;
      }
    BaseOp = X86ISD::ADD;
    Cond = X86::COND_O;
    break;
  case ISD::UADDO:
    BaseOp = X86ISD::ADD;
    Cond = X86::COND_B;
    break;
  case ISD::SSUBO:
    // A subtract of one will be selected as a DEC. Note that DEC doesn't
    // set CF, so we can't do this for USUBO.
    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS))
      if (C->isOne()) {
        BaseOp = X86ISD::DEC;
        Cond = X86::COND_O;
        break;
      }
    BaseOp = X86ISD::SUB;
    Cond = X86::COND_O;
    break;
  case ISD::USUBO:
    BaseOp = X86ISD::SUB;
    Cond = X86::COND_B;
    break;
  case ISD::SMULO:
    BaseOp = X86ISD::SMUL;
    Cond = X86::COND_O;
    break;
  case ISD::UMULO: { // i64, i8 = umulo lhs, rhs --> i64, i64, i32 umul lhs,rhs
    SDVTList VTs = DAG.getVTList(N->getValueType(0), N->getValueType(0),
                                 MVT::i32);
    SDValue Sum = DAG.getNode(X86ISD::UMUL, DL, VTs, LHS, RHS);

    SDValue SetCC =
      DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
                  DAG.getConstant(X86::COND_O, MVT::i32),
                  SDValue(Sum.getNode(), 2));

    return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
  }
  }

  // Also sets EFLAGS.
  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32);
  SDValue Sum = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);

  SDValue SetCC =
    DAG.getNode(X86ISD::SETCC, DL, N->getValueType(1),
                DAG.getConstant(Cond, MVT::i32),
                SDValue(Sum.getNode(), 1));

  return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
}

SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const{
  DebugLoc dl = Op.getDebugLoc();
  SDNode* Node = Op.getNode();
  EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
  EVT VT = Node->getValueType(0);

  if (Subtarget->hasSSE2() && VT.isVector()) {
    unsigned BitsDiff = VT.getScalarType().getSizeInBits() -
                        ExtraVT.getScalarType().getSizeInBits();
    SDValue ShAmt = DAG.getConstant(BitsDiff, MVT::i32);

    unsigned SHLIntrinsicsID = 0;
    unsigned SRAIntrinsicsID = 0;
    switch (VT.getSimpleVT().SimpleTy) {
      default:
        return SDValue();
      case MVT::v2i64: {
        SHLIntrinsicsID = Intrinsic::x86_sse2_pslli_q;
        SRAIntrinsicsID = 0;
        break;
      }
      case MVT::v4i32: {
        SHLIntrinsicsID = Intrinsic::x86_sse2_pslli_d;
        SRAIntrinsicsID = Intrinsic::x86_sse2_psrai_d;
        break;
      }
      case MVT::v8i16: {
        SHLIntrinsicsID = Intrinsic::x86_sse2_pslli_w;
        SRAIntrinsicsID = Intrinsic::x86_sse2_psrai_w;
        break;
      }
    }

    SDValue Tmp1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                         DAG.getConstant(SHLIntrinsicsID, MVT::i32),
                         Node->getOperand(0), ShAmt);

    // In case of 1 bit sext, no need to shr
    if (ExtraVT.getScalarType().getSizeInBits() == 1) return Tmp1;

    if (SRAIntrinsicsID) {
      Tmp1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
                         DAG.getConstant(SRAIntrinsicsID, MVT::i32),
                         Tmp1, ShAmt);
    }
    return Tmp1;
  }

  return SDValue();
}


SDValue X86TargetLowering::LowerMEMBARRIER(SDValue Op, SelectionDAG &DAG) const{
  DebugLoc dl = Op.getDebugLoc();

  // Go ahead and emit the fence on x86-64 even if we asked for no-sse2.
  // There isn't any reason to disable it if the target processor supports it.
  if (!Subtarget->hasSSE2() && !Subtarget->is64Bit()) {
    SDValue Chain = Op.getOperand(0);
    SDValue Zero = DAG.getConstant(0, MVT::i32);
    SDValue Ops[] = {
      DAG.getRegister(X86::ESP, MVT::i32), // Base
      DAG.getTargetConstant(1, MVT::i8),   // Scale
      DAG.getRegister(0, MVT::i32),        // Index
      DAG.getTargetConstant(0, MVT::i32),  // Disp
      DAG.getRegister(0, MVT::i32),        // Segment.
      Zero,
      Chain
    };
    SDNode *Res =
      DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops,
                          array_lengthof(Ops));
    return SDValue(Res, 0);
  }

  unsigned isDev = cast<ConstantSDNode>(Op.getOperand(5))->getZExtValue();
  if (!isDev)
    return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));

  unsigned Op1 = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
  unsigned Op2 = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
  unsigned Op3 = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
  unsigned Op4 = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();

  // def : Pat<(membarrier (i8 0), (i8 0), (i8 0), (i8 1), (i8 1)), (SFENCE)>;
  if (!Op1 && !Op2 && !Op3 && Op4)
    return DAG.getNode(X86ISD::SFENCE, dl, MVT::Other, Op.getOperand(0));

  // def : Pat<(membarrier (i8 1), (i8 0), (i8 0), (i8 0), (i8 1)), (LFENCE)>;
  if (Op1 && !Op2 && !Op3 && !Op4)
    return DAG.getNode(X86ISD::LFENCE, dl, MVT::Other, Op.getOperand(0));

  // def : Pat<(membarrier (i8 imm), (i8 imm), (i8 imm), (i8 imm), (i8 1)),
  //           (MFENCE)>;
  return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
}

SDValue X86TargetLowering::LowerATOMIC_FENCE(SDValue Op,
                                             SelectionDAG &DAG) const {
  DebugLoc dl = Op.getDebugLoc();
  AtomicOrdering FenceOrdering = static_cast<AtomicOrdering>(
    cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue());
  SynchronizationScope FenceScope = static_cast<SynchronizationScope>(
    cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());

  // The only fence that needs an instruction is a sequentially-consistent
  // cross-thread fence.
  if (FenceOrdering == SequentiallyConsistent && FenceScope == CrossThread) {
    // Use mfence if we have SSE2 or we're on x86-64 (even if we asked for
    // no-sse2). There isn't any reason to disable it if the target processor
    // supports it.
    if (Subtarget->hasSSE2() || Subtarget->is64Bit())
      return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));

    SDValue Chain = Op.getOperand(0);
    SDValue Zero = DAG.getConstant(0, MVT::i32);
    SDValue Ops[] = {
      DAG.getRegister(X86::ESP, MVT::i32), // Base
      DAG.getTargetConstant(1, MVT::i8),   // Scale
      DAG.getRegister(0, MVT::i32),        // Index
      DAG.getTargetConstant(0, MVT::i32),  // Disp
      DAG.getRegister(0, MVT::i32),        // Segment.
      Zero,
      Chain
    };
    SDNode *Res =
      DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops,
                         array_lengthof(Ops));
    return SDValue(Res, 0);
  }

  // MEMBARRIER is a compiler barrier; it codegens to a no-op.
  return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
}


SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) const {
  EVT T = Op.getValueType();
  DebugLoc DL = Op.getDebugLoc();
  unsigned Reg = 0;
  unsigned size = 0;
  switch(T.getSimpleVT().SimpleTy) {
  default:
    assert(false && "Invalid value type!");
  case MVT::i8:  Reg = X86::AL;  size = 1; break;
  case MVT::i16: Reg = X86::AX;  size = 2; break;
  case MVT::i32: Reg = X86::EAX; size = 4; break;
  case MVT::i64:
    assert(Subtarget->is64Bit() && "Node not type legal!");
    Reg = X86::RAX; size = 8;
    break;
  }
  SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
                                    Op.getOperand(2), SDValue());
  SDValue Ops[] = { cpIn.getValue(0),
                    Op.getOperand(1),
                    Op.getOperand(3),
                    DAG.getTargetConstant(size, MVT::i8),
                    cpIn.getValue(1) };
  SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
  MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
  SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
                                           Ops, 5, T, MMO);
  SDValue cpOut =
    DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
  return cpOut;
}

SDValue X86TargetLowering::LowerREADCYCLECOUNTER(SDValue Op,
                                                 SelectionDAG &DAG) const {
  assert(Subtarget->is64Bit() && "Result not type legalized?");
  SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
  SDValue TheChain = Op.getOperand(0);
  DebugLoc dl = Op.getDebugLoc();
  SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1);
  SDValue rax = DAG.getCopyFromReg(rd, dl, X86::RAX, MVT::i64, rd.getValue(1));
  SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), dl, X86::RDX, MVT::i64,
                                   rax.getValue(2));
  SDValue Tmp = DAG.getNode(ISD::SHL, dl, MVT::i64, rdx,
                            DAG.getConstant(32, MVT::i8));
  SDValue Ops[] = {
    DAG.getNode(ISD::OR, dl, MVT::i64, rax, Tmp),
    rdx.getValue(1)
  };
  return DAG.getMergeValues(Ops, 2, dl);
}

SDValue X86TargetLowering::LowerBITCAST(SDValue Op,
                                            SelectionDAG &DAG) const {
  EVT SrcVT = Op.getOperand(0).getValueType();
  EVT DstVT = Op.getValueType();
  assert(Subtarget->is64Bit() && !Subtarget->hasSSE2() &&
         Subtarget->hasMMX() && "Unexpected custom BITCAST");
  assert((DstVT == MVT::i64 ||
          (DstVT.isVector() && DstVT.getSizeInBits()==64)) &&
         "Unexpected custom BITCAST");
  // i64 <=> MMX conversions are Legal.
  if (SrcVT==MVT::i64 && DstVT.isVector())
    return Op;
  if (DstVT==MVT::i64 && SrcVT.isVector())
    return Op;
  // MMX <=> MMX conversions are Legal.
  if (SrcVT.isVector() && DstVT.isVector())
    return Op;
  // All other conversions need to be expanded.
  return SDValue();
}

SDValue X86TargetLowering::LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) const {
  SDNode *Node = Op.getNode();
  DebugLoc dl = Node->getDebugLoc();
  EVT T = Node->getValueType(0);
  SDValue negOp = DAG.getNode(ISD::SUB, dl, T,
                              DAG.getConstant(0, T), Node->getOperand(2));
  return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl,
                       cast<AtomicSDNode>(Node)->getMemoryVT(),
                       Node->getOperand(0),
                       Node->getOperand(1), negOp,
                       cast<AtomicSDNode>(Node)->getSrcValue(),
                       cast<AtomicSDNode>(Node)->getAlignment());
}

static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
  EVT VT = Op.getNode()->getValueType(0);

  // Let legalize expand this if it isn't a legal type yet.
  if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
    return SDValue();

  SDVTList VTs = DAG.getVTList(VT, MVT::i32);

  unsigned Opc;
  bool ExtraOp = false;
  switch (Op.getOpcode()) {
  default: assert(0 && "Invalid code");
  case ISD::ADDC: Opc = X86ISD::ADD; break;
  case ISD::ADDE: Opc = X86ISD::ADC; ExtraOp = true; break;
  case ISD::SUBC: Opc = X86ISD::SUB; break;
  case ISD::SUBE: Opc = X86ISD::SBB; ExtraOp = true; break;
  }

  if (!ExtraOp)
    return DAG.getNode(Opc, Op->getDebugLoc(), VTs, Op.getOperand(0),
                       Op.getOperand(1));
  return DAG.getNode(Opc, Op->getDebugLoc(), VTs, Op.getOperand(0),
                     Op.getOperand(1), Op.getOperand(2));
}

/// LowerOperation - Provide custom lowering hooks for some operations.
///
SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
  switch (Op.getOpcode()) {
  default: llvm_unreachable("Should not custom lower this!");
  case ISD::SIGN_EXTEND_INREG:  return LowerSIGN_EXTEND_INREG(Op,DAG);
  case ISD::MEMBARRIER:         return LowerMEMBARRIER(Op,DAG);
  case ISD::ATOMIC_FENCE:       return LowerATOMIC_FENCE(Op,DAG);
  case ISD::ATOMIC_CMP_SWAP:    return LowerCMP_SWAP(Op,DAG);
  case ISD::ATOMIC_LOAD_SUB:    return LowerLOAD_SUB(Op,DAG);
  case ISD::BUILD_VECTOR:       return LowerBUILD_VECTOR(Op, DAG);
  case ISD::CONCAT_VECTORS:     return LowerCONCAT_VECTORS(Op, DAG);
  case ISD::VECTOR_SHUFFLE:     return LowerVECTOR_SHUFFLE(Op, DAG);
  case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
  case ISD::INSERT_VECTOR_ELT:  return LowerINSERT_VECTOR_ELT(Op, DAG);
  case ISD::EXTRACT_SUBVECTOR:  return LowerEXTRACT_SUBVECTOR(Op, DAG);
  case ISD::INSERT_SUBVECTOR:   return LowerINSERT_SUBVECTOR(Op, DAG);
  case ISD::SCALAR_TO_VECTOR:   return LowerSCALAR_TO_VECTOR(Op, DAG);
  case ISD::ConstantPool:       return LowerConstantPool(Op, DAG);
  case ISD::GlobalAddress:      return LowerGlobalAddress(Op, DAG);
  case ISD::GlobalTLSAddress:   return LowerGlobalTLSAddress(Op, DAG);
  case ISD::ExternalSymbol:     return LowerExternalSymbol(Op, DAG);
  case ISD::BlockAddress:       return LowerBlockAddress(Op, DAG);
  case ISD::SHL_PARTS:
  case ISD::SRA_PARTS:
  case ISD::SRL_PARTS:          return LowerShiftParts(Op, DAG);
  case ISD::SINT_TO_FP:         return LowerSINT_TO_FP(Op, DAG);
  case ISD::UINT_TO_FP:         return LowerUINT_TO_FP(Op, DAG);
  case ISD::FP_TO_SINT:         return LowerFP_TO_SINT(Op, DAG);
  case ISD::FP_TO_UINT:         return LowerFP_TO_UINT(Op, DAG);
  case ISD::FABS:               return LowerFABS(Op, DAG);
  case ISD::FNEG:               return LowerFNEG(Op, DAG);
  case ISD::FCOPYSIGN:          return LowerFCOPYSIGN(Op, DAG);
  case ISD::FGETSIGN:           return LowerFGETSIGN(Op, DAG);
  case ISD::SETCC:              return LowerSETCC(Op, DAG);
  case ISD::VSETCC:             return LowerVSETCC(Op, DAG);
  case ISD::SELECT:             return LowerSELECT(Op, DAG);
  case ISD::BRCOND:             return LowerBRCOND(Op, DAG);
  case ISD::JumpTable:          return LowerJumpTable(Op, DAG);
  case ISD::VASTART:            return LowerVASTART(Op, DAG);
  case ISD::VAARG:              return LowerVAARG(Op, DAG);
  case ISD::VACOPY:             return LowerVACOPY(Op, DAG);
  case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
  case ISD::RETURNADDR:         return LowerRETURNADDR(Op, DAG);
  case ISD::FRAMEADDR:          return LowerFRAMEADDR(Op, DAG);
  case ISD::FRAME_TO_ARGS_OFFSET:
                                return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
  case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
  case ISD::EH_RETURN:          return LowerEH_RETURN(Op, DAG);
  case ISD::TRAMPOLINE:         return LowerTRAMPOLINE(Op, DAG);
  case ISD::FLT_ROUNDS_:        return LowerFLT_ROUNDS_(Op, DAG);
  case ISD::CTLZ:               return LowerCTLZ(Op, DAG);
  case ISD::CTTZ:               return LowerCTTZ(Op, DAG);
  case ISD::MUL:                return LowerMUL_V2I64(Op, DAG);
  case ISD::SRA:
  case ISD::SRL:
  case ISD::SHL:                return LowerShift(Op, DAG);
  case ISD::SADDO:
  case ISD::UADDO:
  case ISD::SSUBO:
  case ISD::USUBO:
  case ISD::SMULO:
  case ISD::UMULO:              return LowerXALUO(Op, DAG);
  case ISD::READCYCLECOUNTER:   return LowerREADCYCLECOUNTER(Op, DAG);
  case ISD::BITCAST:            return LowerBITCAST(Op, DAG);
  case ISD::ADDC:
  case ISD::ADDE:
  case ISD::SUBC:
  case ISD::SUBE:               return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
  }
}

void X86TargetLowering::
ReplaceATOMIC_BINARY_64(SDNode *Node, SmallVectorImpl<SDValue>&Results,
                        SelectionDAG &DAG, unsigned NewOp) const {
  EVT T = Node->getValueType(0);
  DebugLoc dl = Node->getDebugLoc();
  assert (T == MVT::i64 && "Only know how to expand i64 atomics");

  SDValue Chain = Node->getOperand(0);
  SDValue In1 = Node->getOperand(1);
  SDValue In2L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
                             Node->getOperand(2), DAG.getIntPtrConstant(0));
  SDValue In2H = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
                             Node->getOperand(2), DAG.getIntPtrConstant(1));
  SDValue Ops[] = { Chain, In1, In2L, In2H };
  SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
  SDValue Result =
    DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64,
                            cast<MemSDNode>(Node)->getMemOperand());
  SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)};
  Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2));
  Results.push_back(Result.getValue(2));
}

/// ReplaceNodeResults - Replace a node with an illegal result type
/// with a new node built out of custom code.
void X86TargetLowering::ReplaceNodeResults(SDNode *N,
                                           SmallVectorImpl<SDValue>&Results,
                                           SelectionDAG &DAG) const {
  DebugLoc dl = N->getDebugLoc();
  switch (N->getOpcode()) {
  default:
    assert(false && "Do not know how to custom type legalize this operation!");
    return;
  case ISD::SIGN_EXTEND_INREG:
  case ISD::ADDC:
  case ISD::ADDE:
  case ISD::SUBC:
  case ISD::SUBE:
    // We don't want to expand or promote these.
    return;
  case ISD::FP_TO_SINT: {
    std::pair<SDValue,SDValue> Vals =
        FP_TO_INTHelper(SDValue(N, 0), DAG, true);
    SDValue FIST = Vals.first, StackSlot = Vals.second;
    if (FIST.getNode() != 0) {
      EVT VT = N->getValueType(0);
      // Return a load from the stack slot.
      Results.push_back(DAG.getLoad(VT, dl, FIST, StackSlot,
                                    MachinePointerInfo(), false, false, 0));
    }
    return;
  }
  case ISD::READCYCLECOUNTER: {
    SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
    SDValue TheChain = N->getOperand(0);
    SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1);
    SDValue eax = DAG.getCopyFromReg(rd, dl, X86::EAX, MVT::i32,
                                     rd.getValue(1));
    SDValue edx = DAG.getCopyFromReg(eax.getValue(1), dl, X86::EDX, MVT::i32,
                                     eax.getValue(2));
    // Use a buildpair to merge the two 32-bit values into a 64-bit one.
    SDValue Ops[] = { eax, edx };
    Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Ops, 2));
    Results.push_back(edx.getValue(1));
    return;
  }
  case ISD::ATOMIC_CMP_SWAP: {
    EVT T = N->getValueType(0);
    assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap");
    SDValue cpInL, cpInH;
    cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2),
                        DAG.getConstant(0, MVT::i32));
    cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2),
                        DAG.getConstant(1, MVT::i32));
    cpInL = DAG.getCopyToReg(N->getOperand(0), dl, X86::EAX, cpInL, SDValue());
    cpInH = DAG.getCopyToReg(cpInL.getValue(0), dl, X86::EDX, cpInH,
                             cpInL.getValue(1));
    SDValue swapInL, swapInH;
    swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3),
                          DAG.getConstant(0, MVT::i32));
    swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3),
                          DAG.getConstant(1, MVT::i32));
    swapInL = DAG.getCopyToReg(cpInH.getValue(0), dl, X86::EBX, swapInL,
                               cpInH.getValue(1));
    swapInH = DAG.getCopyToReg(swapInL.getValue(0), dl, X86::ECX, swapInH,
                               swapInL.getValue(1));
    SDValue Ops[] = { swapInH.getValue(0),
                      N->getOperand(1),
                      swapInH.getValue(1) };
    SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
    MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
    SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG8_DAG, dl, Tys,
                                             Ops, 3, T, MMO);
    SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl, X86::EAX,
                                        MVT::i32, Result.getValue(1));
    SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl, X86::EDX,
                                        MVT::i32, cpOutL.getValue(2));
    SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
    Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2));
    Results.push_back(cpOutH.getValue(1));
    return;
  }
  case ISD::ATOMIC_LOAD_ADD:
    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMADD64_DAG);
    return;
  case ISD::ATOMIC_LOAD_AND:
    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMAND64_DAG);
    return;
  case ISD::ATOMIC_LOAD_NAND:
    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMNAND64_DAG);
    return;
  case ISD::ATOMIC_LOAD_OR:
    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMOR64_DAG);
    return;
  case ISD::ATOMIC_LOAD_SUB:
    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSUB64_DAG);
    return;
  case ISD::ATOMIC_LOAD_XOR:
    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMXOR64_DAG);
    return;
  case ISD::ATOMIC_SWAP:
    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSWAP64_DAG);
    return;
  }
}

const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
  switch (Opcode) {
  default: return NULL;
  case X86ISD::BSF:                return "X86ISD::BSF";
  case X86ISD::BSR:                return "X86ISD::BSR";
  case X86ISD::SHLD:               return "X86ISD::SHLD";
  case X86ISD::SHRD:               return "X86ISD::SHRD";
  case X86ISD::FAND:               return "X86ISD::FAND";
  case X86ISD::FOR:                return "X86ISD::FOR";
  case X86ISD::FXOR:               return "X86ISD::FXOR";
  case X86ISD::FSRL:               return "X86ISD::FSRL";
  case X86ISD::FILD:               return "X86ISD::FILD";
  case X86ISD::FILD_FLAG:          return "X86ISD::FILD_FLAG";
  case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
  case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
  case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
  case X86ISD::FLD:                return "X86ISD::FLD";
  case X86ISD::FST:                return "X86ISD::FST";
  case X86ISD::CALL:               return "X86ISD::CALL";
  case X86ISD::RDTSC_DAG:          return "X86ISD::RDTSC_DAG";
  case X86ISD::BT:                 return "X86ISD::BT";
  case X86ISD::CMP:                return "X86ISD::CMP";
  case X86ISD::COMI:               return "X86ISD::COMI";
  case X86ISD::UCOMI:              return "X86ISD::UCOMI";
  case X86ISD::SETCC:              return "X86ISD::SETCC";
  case X86ISD::SETCC_CARRY:        return "X86ISD::SETCC_CARRY";
  case X86ISD::FSETCCsd:           return "X86ISD::FSETCCsd";
  case X86ISD::FSETCCss:           return "X86ISD::FSETCCss";
  case X86ISD::CMOV:               return "X86ISD::CMOV";
  case X86ISD::BRCOND:             return "X86ISD::BRCOND";
  case X86ISD::RET_FLAG:           return "X86ISD::RET_FLAG";
  case X86ISD::REP_STOS:           return "X86ISD::REP_STOS";
  case X86ISD::REP_MOVS:           return "X86ISD::REP_MOVS";
  case X86ISD::GlobalBaseReg:      return "X86ISD::GlobalBaseReg";
  case X86ISD::Wrapper:            return "X86ISD::Wrapper";
  case X86ISD::WrapperRIP:         return "X86ISD::WrapperRIP";
  case X86ISD::PEXTRB:             return "X86ISD::PEXTRB";
  case X86ISD::PEXTRW:             return "X86ISD::PEXTRW";
  case X86ISD::INSERTPS:           return "X86ISD::INSERTPS";
  case X86ISD::PINSRB:             return "X86ISD::PINSRB";
  case X86ISD::PINSRW:             return "X86ISD::PINSRW";
  case X86ISD::PSHUFB:             return "X86ISD::PSHUFB";
  case X86ISD::ANDNP:              return "X86ISD::ANDNP";
  case X86ISD::PSIGNB:             return "X86ISD::PSIGNB";
  case X86ISD::PSIGNW:             return "X86ISD::PSIGNW";
  case X86ISD::PSIGND:             return "X86ISD::PSIGND";
  case X86ISD::PBLENDVB:           return "X86ISD::PBLENDVB";
  case X86ISD::FMAX:               return "X86ISD::FMAX";
  case X86ISD::FMIN:               return "X86ISD::FMIN";
  case X86ISD::FRSQRT:             return "X86ISD::FRSQRT";
  case X86ISD::FRCP:               return "X86ISD::FRCP";
  case X86ISD::TLSADDR:            return "X86ISD::TLSADDR";
  case X86ISD::TLSCALL:            return "X86ISD::TLSCALL";
  case X86ISD::EH_RETURN:          return "X86ISD::EH_RETURN";
  case X86ISD::TC_RETURN:          return "X86ISD::TC_RETURN";
  case X86ISD::FNSTCW16m:          return "X86ISD::FNSTCW16m";
  case X86ISD::LCMPXCHG_DAG:       return "X86ISD::LCMPXCHG_DAG";
  case X86ISD::LCMPXCHG8_DAG:      return "X86ISD::LCMPXCHG8_DAG";
  case X86ISD::ATOMADD64_DAG:      return "X86ISD::ATOMADD64_DAG";
  case X86ISD::ATOMSUB64_DAG:      return "X86ISD::ATOMSUB64_DAG";
  case X86ISD::ATOMOR64_DAG:       return "X86ISD::ATOMOR64_DAG";
  case X86ISD::ATOMXOR64_DAG:      return "X86ISD::ATOMXOR64_DAG";
  case X86ISD::ATOMAND64_DAG:      return "X86ISD::ATOMAND64_DAG";
  case X86ISD::ATOMNAND64_DAG:     return "X86ISD::ATOMNAND64_DAG";
  case X86ISD::VZEXT_MOVL:         return "X86ISD::VZEXT_MOVL";
  case X86ISD::VZEXT_LOAD:         return "X86ISD::VZEXT_LOAD";
  case X86ISD::VSHL:               return "X86ISD::VSHL";
  case X86ISD::VSRL:               return "X86ISD::VSRL";
  case X86ISD::CMPPD:              return "X86ISD::CMPPD";
  case X86ISD::CMPPS:              return "X86ISD::CMPPS";
  case X86ISD::PCMPEQB:            return "X86ISD::PCMPEQB";
  case X86ISD::PCMPEQW:            return "X86ISD::PCMPEQW";
  case X86ISD::PCMPEQD:            return "X86ISD::PCMPEQD";
  case X86ISD::PCMPEQQ:            return "X86ISD::PCMPEQQ";
  case X86ISD::PCMPGTB:            return "X86ISD::PCMPGTB";
  case X86ISD::PCMPGTW:            return "X86ISD::PCMPGTW";
  case X86ISD::PCMPGTD:            return "X86ISD::PCMPGTD";
  case X86ISD::PCMPGTQ:            return "X86ISD::PCMPGTQ";
  case X86ISD::ADD:                return "X86ISD::ADD";
  case X86ISD::SUB:                return "X86ISD::SUB";
  case X86ISD::ADC:                return "X86ISD::ADC";
  case X86ISD::SBB:                return "X86ISD::SBB";
  case X86ISD::SMUL:               return "X86ISD::SMUL";
  case X86ISD::UMUL:               return "X86ISD::UMUL";
  case X86ISD::INC:                return "X86ISD::INC";
  case X86ISD::DEC:                return "X86ISD::DEC";
  case X86ISD::OR:                 return "X86ISD::OR";
  case X86ISD::XOR:                return "X86ISD::XOR";
  case X86ISD::AND:                return "X86ISD::AND";
  case X86ISD::MUL_IMM:            return "X86ISD::MUL_IMM";
  case X86ISD::PTEST:              return "X86ISD::PTEST";
  case X86ISD::TESTP:              return "X86ISD::TESTP";
  case X86ISD::PALIGN:             return "X86ISD::PALIGN";
  case X86ISD::PSHUFD:             return "X86ISD::PSHUFD";
  case X86ISD::PSHUFHW:            return "X86ISD::PSHUFHW";
  case X86ISD::PSHUFHW_LD:         return "X86ISD::PSHUFHW_LD";
  case X86ISD::PSHUFLW:            return "X86ISD::PSHUFLW";
  case X86ISD::PSHUFLW_LD:         return "X86ISD::PSHUFLW_LD";
  case X86ISD::SHUFPS:             return "X86ISD::SHUFPS";
  case X86ISD::SHUFPD:             return "X86ISD::SHUFPD";
  case X86ISD::MOVLHPS:            return "X86ISD::MOVLHPS";
  case X86ISD::MOVLHPD:            return "X86ISD::MOVLHPD";
  case X86ISD::MOVHLPS:            return "X86ISD::MOVHLPS";
  case X86ISD::MOVHLPD:            return "X86ISD::MOVHLPD";
  case X86ISD::MOVLPS:             return "X86ISD::MOVLPS";
  case X86ISD::MOVLPD:             return "X86ISD::MOVLPD";
  case X86ISD::MOVDDUP:            return "X86ISD::MOVDDUP";
  case X86ISD::MOVSHDUP:           return "X86ISD::MOVSHDUP";
  case X86ISD::MOVSLDUP:           return "X86ISD::MOVSLDUP";
  case X86ISD::MOVSHDUP_LD:        return "X86ISD::MOVSHDUP_LD";
  case X86ISD::MOVSLDUP_LD:        return "X86ISD::MOVSLDUP_LD";
  case X86ISD::MOVSD:              return "X86ISD::MOVSD";
  case X86ISD::MOVSS:              return "X86ISD::MOVSS";
  case X86ISD::UNPCKLPS:           return "X86ISD::UNPCKLPS";
  case X86ISD::UNPCKLPD:           return "X86ISD::UNPCKLPD";
  case X86ISD::VUNPCKLPDY:         return "X86ISD::VUNPCKLPDY";
  case X86ISD::UNPCKHPS:           return "X86ISD::UNPCKHPS";
  case X86ISD::UNPCKHPD:           return "X86ISD::UNPCKHPD";
  case X86ISD::PUNPCKLBW:          return "X86ISD::PUNPCKLBW";
  case X86ISD::PUNPCKLWD:          return "X86ISD::PUNPCKLWD";
  case X86ISD::PUNPCKLDQ:          return "X86ISD::PUNPCKLDQ";
  case X86ISD::PUNPCKLQDQ:         return "X86ISD::PUNPCKLQDQ";
  case X86ISD::PUNPCKHBW:          return "X86ISD::PUNPCKHBW";
  case X86ISD::PUNPCKHWD:          return "X86ISD::PUNPCKHWD";
  case X86ISD::PUNPCKHDQ:          return "X86ISD::PUNPCKHDQ";
  case X86ISD::PUNPCKHQDQ:         return "X86ISD::PUNPCKHQDQ";
  case X86ISD::VPERMILPS:          return "X86ISD::VPERMILPS";
  case X86ISD::VPERMILPSY:         return "X86ISD::VPERMILPSY";
  case X86ISD::VPERMILPD:          return "X86ISD::VPERMILPD";
  case X86ISD::VPERMILPDY:         return "X86ISD::VPERMILPDY";
  case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS";
  case X86ISD::VAARG_64:           return "X86ISD::VAARG_64";
  case X86ISD::WIN_ALLOCA:         return "X86ISD::WIN_ALLOCA";
  case X86ISD::MEMBARRIER:         return "X86ISD::MEMBARRIER";
  }
}

// isLegalAddressingMode - Return true if the addressing mode represented
// by AM is legal for this target, for a load/store of the specified type.
bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM,
                                              Type *Ty) const {
  // X86 supports extremely general addressing modes.
  CodeModel::Model M = getTargetMachine().getCodeModel();
  Reloc::Model R = getTargetMachine().getRelocationModel();

  // X86 allows a sign-extended 32-bit immediate field as a displacement.
  if (!X86::isOffsetSuitableForCodeModel(AM.BaseOffs, M, AM.BaseGV != NULL))
    return false;

  if (AM.BaseGV) {
    unsigned GVFlags =
      Subtarget->ClassifyGlobalReference(AM.BaseGV, getTargetMachine());

    // If a reference to this global requires an extra load, we can't fold it.
    if (isGlobalStubReference(GVFlags))
      return false;

    // If BaseGV requires a register for the PIC base, we cannot also have a
    // BaseReg specified.
    if (AM.HasBaseReg && isGlobalRelativeToPICBase(GVFlags))
      return false;

    // If lower 4G is not available, then we must use rip-relative addressing.
    if ((M != CodeModel::Small || R != Reloc::Static) &&
        Subtarget->is64Bit() && (AM.BaseOffs || AM.Scale > 1))
      return false;
  }

  switch (AM.Scale) {
  case 0:
  case 1:
  case 2:
  case 4:
  case 8:
    // These scales always work.
    break;
  case 3:
  case 5:
  case 9:
    // These scales are formed with basereg+scalereg.  Only accept if there is
    // no basereg yet.
    if (AM.HasBaseReg)
      return false;
    break;
  default:  // Other stuff never works.
    return false;
  }

  return true;
}


bool X86TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
  if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
    return false;
  unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
  unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
  if (NumBits1 <= NumBits2)
    return false;
  return true;
}

bool X86TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
  if (!VT1.isInteger() || !VT2.isInteger())
    return false;
  unsigned NumBits1 = VT1.getSizeInBits();
  unsigned NumBits2 = VT2.getSizeInBits();
  if (NumBits1 <= NumBits2)
    return false;
  return true;
}

bool X86TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
  // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
  return Ty1->isIntegerTy(32) && Ty2->isIntegerTy(64) && Subtarget->is64Bit();
}

bool X86TargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
  // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
  return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget->is64Bit();
}