Newer
Older
SDOperand());
} else if (!X86ScalarSSE) {
// FP return with fp-stack value.
if (DAG.getMachineFunction().liveout_empty())
DAG.getMachineFunction().addLiveOut(X86::ST0);
SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
SDOperand Ops[] = { Op.getOperand(0), Op.getOperand(1) };
Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops, 2);
} else {
// FP return with ScalarSSE (return on fp-stack).
if (DAG.getMachineFunction().liveout_empty())
DAG.getMachineFunction().addLiveOut(X86::ST0);
SDOperand MemLoc;
SDOperand Chain = Op.getOperand(0);
SDOperand Value = Op.getOperand(1);
if (ISD::isNON_EXTLoad(Value.Val) &&
(Chain == Value.getValue(1) || Chain == Value.getOperand(0))) {
Chain = Value.getOperand(0);
MemLoc = Value.getOperand(1);
} else {
// Spill the value to memory and reload it into top of stack.
unsigned Size = MVT::getSizeInBits(ArgVT)/8;
MachineFunction &MF = DAG.getMachineFunction();
int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
MemLoc = DAG.getFrameIndex(SSFI, getPointerTy());
Chain = DAG.getStore(Op.getOperand(0), Value, MemLoc, NULL, 0);
SDVTList Tys = DAG.getVTList(MVT::f64, MVT::Other);
SDOperand Ops[] = { Chain, MemLoc, DAG.getValueType(ArgVT) };
Copy = DAG.getNode(X86ISD::FLD, Tys, Ops, 3);
Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Ops[0] = Copy.getValue(1);
Ops[1] = Copy;
Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops, 2);
}
break;
}
case 5: {
unsigned Reg1 = Subtarget->is64Bit() ? X86::RAX : X86::EAX;
unsigned Reg2 = Subtarget->is64Bit() ? X86::RDX : X86::EDX;
if (DAG.getMachineFunction().liveout_empty()) {
DAG.getMachineFunction().addLiveOut(Reg1);
DAG.getMachineFunction().addLiveOut(Reg2);
}
Copy = DAG.getCopyToReg(Op.getOperand(0), Reg2, Op.getOperand(3),
SDOperand());
Copy = DAG.getCopyToReg(Copy, Reg1, Op.getOperand(1), Copy.getValue(1));
break;
}
return DAG.getNode(X86ISD::RET_FLAG, MVT::Other,
Copy, DAG.getConstant(getBytesToPopOnReturn(), MVT::i16),
Copy.getValue(1));
}
SDOperand
X86TargetLowering::LowerFORMAL_ARGUMENTS(SDOperand Op, SelectionDAG &DAG) {
MachineFunction &MF = DAG.getMachineFunction();
const Function* Fn = MF.getFunction();
if (Fn->hasExternalLinkage() &&
MF.getInfo<X86FunctionInfo>()->setForceFramePointer(true);
unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue();
if (Subtarget->is64Bit())
return LowerX86_64CCCArguments(Op, DAG);
switch(CC) {
default:
assert(0 && "Unsupported calling convention");
case CallingConv::Fast:
if (EnableFastCC) {
return LowerFastCCArguments(Op, DAG);
}
// Falls through
return LowerCCCArguments(Op, DAG);
MF.getInfo<X86FunctionInfo>()->setDecorationStyle(StdCall);
return LowerCCCArguments(Op, DAG, true);
MF.getInfo<X86FunctionInfo>()->setDecorationStyle(FastCall);
return LowerFastCCArguments(Op, DAG, true);
}
SDOperand X86TargetLowering::LowerMEMSET(SDOperand Op, SelectionDAG &DAG) {
SDOperand InFlag(0, 0);
SDOperand Chain = Op.getOperand(0);
unsigned Align =
(unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
if (Align == 0) Align = 1;
ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3));
// If not DWORD aligned, call memset if size is less than the threshold.
// It knows how to align to the right boundary first.
if ((Align & 3) != 0 ||
(I && I->getValue() < Subtarget->getMinRepStrSizeThreshold())) {
MVT::ValueType IntPtr = getPointerTy();
const Type *IntPtrTy = getTargetData()->getIntPtrType();
TargetLowering::ArgListTy Args;
TargetLowering::ArgListEntry Entry;
Entry.Node = Op.getOperand(1);
Entry.Ty = IntPtrTy;
Entry.isSigned = false;
Entry.isInReg = false;
Entry.isSRet = false;
// Extend the unsigned i8 argument to be an int value for the call.
Entry.Node = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Op.getOperand(2));
Entry.Ty = IntPtrTy;
Entry.isSigned = false;
Entry.isInReg = false;
Entry.isSRet = false;
Args.push_back(Entry);
Entry.Node = Op.getOperand(3);
Args.push_back(Entry);
std::pair<SDOperand,SDOperand> CallResult =
LowerCallTo(Chain, Type::VoidTy, false, false, CallingConv::C, false,
DAG.getExternalSymbol("memset", IntPtr), Args, DAG);
return CallResult.second;
}
MVT::ValueType AVT;
SDOperand Count;
ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Op.getOperand(2));
unsigned BytesLeft = 0;
bool TwoRepStos = false;
if (ValC) {
unsigned ValReg;
// If the value is a constant, then we can potentially use larger sets.
switch (Align & 3) {
case 2: // WORD aligned
AVT = MVT::i16;
ValReg = X86::AX;
AVT = MVT::i32;
Val = (Val << 8) | Val;
Val = (Val << 16) | Val;
if (Subtarget->is64Bit() && ((Align & 0xF) == 0)) { // QWORD aligned
AVT = MVT::i64;
ValReg = X86::RAX;
Val = (Val << 32) | Val;
}
break;
default: // Byte aligned
AVT = MVT::i8;
ValReg = X86::AL;
}
if (AVT > MVT::i8) {
if (I) {
unsigned UBytes = MVT::getSizeInBits(AVT) / 8;
Count = DAG.getConstant(I->getValue() / UBytes, getPointerTy());
BytesLeft = I->getValue() % UBytes;
} else {
assert(AVT >= MVT::i32 &&
"Do not use rep;stos if not at least DWORD aligned");
Count = DAG.getNode(ISD::SRL, Op.getOperand(3).getValueType(),
Op.getOperand(3), DAG.getConstant(2, MVT::i8));
TwoRepStos = true;
}
}
Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT),
InFlag);
InFlag = Chain.getValue(1);
} else {
AVT = MVT::i8;
Count = Op.getOperand(3);
Chain = DAG.getCopyToReg(Chain, X86::AL, Op.getOperand(2), InFlag);
InFlag = Chain.getValue(1);
Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX,
Count, InFlag);
InFlag = Chain.getValue(1);
Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI,
Op.getOperand(1), InFlag);
InFlag = Chain.getValue(1);
SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
std::vector<SDOperand> Ops;
Ops.push_back(Chain);
Ops.push_back(DAG.getValueType(AVT));
Ops.push_back(InFlag);
Evan Cheng
committed
Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size());
if (TwoRepStos) {
InFlag = Chain.getValue(1);
Count = Op.getOperand(3);
MVT::ValueType CVT = Count.getValueType();
SDOperand Left = DAG.getNode(ISD::AND, CVT, Count,
DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT));
Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX,
Left, InFlag);
InFlag = Chain.getValue(1);
Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Ops.clear();
Ops.push_back(Chain);
Ops.push_back(DAG.getValueType(MVT::i8));
Ops.push_back(InFlag);
Evan Cheng
committed
Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size());
} else if (BytesLeft) {
SDOperand Value;
unsigned Val = ValC->getValue() & 255;
unsigned Offset = I->getValue() - BytesLeft;
SDOperand DstAddr = Op.getOperand(1);
MVT::ValueType AddrVT = DstAddr.getValueType();
if (BytesLeft >= 4) {
Val = (Val << 8) | Val;
Val = (Val << 16) | Val;
Value = DAG.getConstant(Val, MVT::i32);
Chain = DAG.getStore(Chain, Value,
DAG.getNode(ISD::ADD, AddrVT, DstAddr,
DAG.getConstant(Offset, AddrVT)),
NULL, 0);
if (BytesLeft >= 2) {
Value = DAG.getConstant((Val << 8) | Val, MVT::i16);
Chain = DAG.getStore(Chain, Value,
DAG.getNode(ISD::ADD, AddrVT, DstAddr,
DAG.getConstant(Offset, AddrVT)),
NULL, 0);
BytesLeft -= 2;
Offset += 2;
}
if (BytesLeft == 1) {
Value = DAG.getConstant(Val, MVT::i8);
Chain = DAG.getStore(Chain, Value,
DAG.getNode(ISD::ADD, AddrVT, DstAddr,
DAG.getConstant(Offset, AddrVT)),
NULL, 0);
}
}
return Chain;
}
SDOperand X86TargetLowering::LowerMEMCPY(SDOperand Op, SelectionDAG &DAG) {
SDOperand Chain = Op.getOperand(0);
unsigned Align =
(unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
if (Align == 0) Align = 1;
ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3));
// If not DWORD aligned, call memcpy if size is less than the threshold.
// It knows how to align to the right boundary first.
if ((Align & 3) != 0 ||
(I && I->getValue() < Subtarget->getMinRepStrSizeThreshold())) {
MVT::ValueType IntPtr = getPointerTy();
TargetLowering::ArgListTy Args;
TargetLowering::ArgListEntry Entry;
Entry.Ty = getTargetData()->getIntPtrType();
Entry.isSigned = false;
Entry.isInReg = false;
Entry.isSRet = false;
Entry.Node = Op.getOperand(1); Args.push_back(Entry);
Entry.Node = Op.getOperand(2); Args.push_back(Entry);
Entry.Node = Op.getOperand(3); Args.push_back(Entry);
std::pair<SDOperand,SDOperand> CallResult =
LowerCallTo(Chain, Type::VoidTy, false, false, CallingConv::C, false,
DAG.getExternalSymbol("memcpy", IntPtr), Args, DAG);
return CallResult.second;
}
MVT::ValueType AVT;
SDOperand Count;
unsigned BytesLeft = 0;
bool TwoRepMovs = false;
switch (Align & 3) {
case 2: // WORD aligned
AVT = MVT::i16;
break;
AVT = MVT::i32;
if (Subtarget->is64Bit() && ((Align & 0xF) == 0)) // QWORD aligned
AVT = MVT::i64;
break;
default: // Byte aligned
AVT = MVT::i8;
Count = Op.getOperand(3);
break;
}
if (AVT > MVT::i8) {
if (I) {
unsigned UBytes = MVT::getSizeInBits(AVT) / 8;
Count = DAG.getConstant(I->getValue() / UBytes, getPointerTy());
BytesLeft = I->getValue() % UBytes;
} else {
assert(AVT >= MVT::i32 &&
"Do not use rep;movs if not at least DWORD aligned");
Count = DAG.getNode(ISD::SRL, Op.getOperand(3).getValueType(),
Op.getOperand(3), DAG.getConstant(2, MVT::i8));
TwoRepMovs = true;
}
}
SDOperand InFlag(0, 0);
Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX,
Count, InFlag);
InFlag = Chain.getValue(1);
Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI,
Op.getOperand(1), InFlag);
InFlag = Chain.getValue(1);
Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RSI : X86::ESI,
Op.getOperand(2), InFlag);
InFlag = Chain.getValue(1);
SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
std::vector<SDOperand> Ops;
Ops.push_back(Chain);
Ops.push_back(DAG.getValueType(AVT));
Ops.push_back(InFlag);
Evan Cheng
committed
Chain = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size());
if (TwoRepMovs) {
InFlag = Chain.getValue(1);
Count = Op.getOperand(3);
MVT::ValueType CVT = Count.getValueType();
SDOperand Left = DAG.getNode(ISD::AND, CVT, Count,
DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT));
Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX,
Left, InFlag);
InFlag = Chain.getValue(1);
Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Ops.clear();
Ops.push_back(Chain);
Ops.push_back(DAG.getValueType(MVT::i8));
Ops.push_back(InFlag);
Evan Cheng
committed
Chain = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size());
} else if (BytesLeft) {
// Issue loads and stores for the last 1 - 7 bytes.
unsigned Offset = I->getValue() - BytesLeft;
SDOperand DstAddr = Op.getOperand(1);
MVT::ValueType DstVT = DstAddr.getValueType();
SDOperand SrcAddr = Op.getOperand(2);
MVT::ValueType SrcVT = SrcAddr.getValueType();
SDOperand Value;
if (BytesLeft >= 4) {
Value = DAG.getLoad(MVT::i32, Chain,
DAG.getNode(ISD::ADD, SrcVT, SrcAddr,
DAG.getConstant(Offset, SrcVT)),
Chain = DAG.getStore(Chain, Value,
DAG.getNode(ISD::ADD, DstVT, DstAddr,
DAG.getConstant(Offset, DstVT)),
NULL, 0);
if (BytesLeft >= 2) {
Value = DAG.getLoad(MVT::i16, Chain,
DAG.getNode(ISD::ADD, SrcVT, SrcAddr,
DAG.getConstant(Offset, SrcVT)),
Chain = Value.getValue(1);
Chain = DAG.getStore(Chain, Value,
DAG.getNode(ISD::ADD, DstVT, DstAddr,
DAG.getConstant(Offset, DstVT)),
NULL, 0);
BytesLeft -= 2;
Offset += 2;
}
if (BytesLeft == 1) {
Value = DAG.getLoad(MVT::i8, Chain,
DAG.getNode(ISD::ADD, SrcVT, SrcAddr,
DAG.getConstant(Offset, SrcVT)),
Chain = Value.getValue(1);
Chain = DAG.getStore(Chain, Value,
DAG.getNode(ISD::ADD, DstVT, DstAddr,
DAG.getConstant(Offset, DstVT)),
NULL, 0);
return Chain;
}
SDOperand
X86TargetLowering::LowerREADCYCLCECOUNTER(SDOperand Op, SelectionDAG &DAG) {
SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
std::vector<SDOperand> Ops;
Ops.push_back(Op.getOperand(0));
Evan Cheng
committed
SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &Ops[0], Ops.size());
Ops.clear();
if (Subtarget->is64Bit()) {
SDOperand Copy1 = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1));
SDOperand Copy2 = DAG.getCopyFromReg(Copy1.getValue(1), X86::RDX,
MVT::i64, Copy1.getValue(2));
SDOperand Tmp = DAG.getNode(ISD::SHL, MVT::i64, Copy2,
DAG.getConstant(32, MVT::i8));
Ops.push_back(DAG.getNode(ISD::OR, MVT::i64, Copy1, Tmp));
Ops.push_back(Copy2.getValue(1));
Tys = DAG.getVTList(MVT::i64, MVT::Other);
} else {
SDOperand Copy1 = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1));
SDOperand Copy2 = DAG.getCopyFromReg(Copy1.getValue(1), X86::EDX,
MVT::i32, Copy1.getValue(2));
Ops.push_back(Copy1);
Ops.push_back(Copy2);
Ops.push_back(Copy2.getValue(1));
Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
Evan Cheng
committed
return DAG.getNode(ISD::MERGE_VALUES, Tys, &Ops[0], Ops.size());
}
SDOperand X86TargetLowering::LowerVASTART(SDOperand Op, SelectionDAG &DAG) {
SrcValueSDNode *SV = cast<SrcValueSDNode>(Op.getOperand(2));
if (!Subtarget->is64Bit()) {
// vastart just stores the address of the VarArgsFrameIndex slot into the
// memory location argument.
SDOperand FR = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy());
return DAG.getStore(Op.getOperand(0), FR,Op.getOperand(1), SV->getValue(),
SV->getOffset());
}
// __va_list_tag:
// gp_offset (0 - 6 * 8)
// fp_offset (48 - 48 + 8 * 16)
// overflow_arg_area (point to parameters coming in memory).
// reg_save_area
std::vector<SDOperand> MemOps;
SDOperand FIN = Op.getOperand(1);
// Store gp_offset
SDOperand Store = DAG.getStore(Op.getOperand(0),
DAG.getConstant(VarArgsGPOffset, MVT::i32),
FIN, SV->getValue(), SV->getOffset());
MemOps.push_back(Store);
// Store fp_offset
FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN,
DAG.getConstant(4, getPointerTy()));
Store = DAG.getStore(Op.getOperand(0),
DAG.getConstant(VarArgsFPOffset, MVT::i32),
FIN, SV->getValue(), SV->getOffset());
MemOps.push_back(Store);
// Store ptr to overflow_arg_area
FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN,
DAG.getConstant(4, getPointerTy()));
SDOperand OVFIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy());
Store = DAG.getStore(Op.getOperand(0), OVFIN, FIN, SV->getValue(),
SV->getOffset());
MemOps.push_back(Store);
// Store ptr to reg_save_area.
FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN,
DAG.getConstant(8, getPointerTy()));
SDOperand RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy());
Store = DAG.getStore(Op.getOperand(0), RSFIN, FIN, SV->getValue(),
SV->getOffset());
MemOps.push_back(Store);
return DAG.getNode(ISD::TokenFactor, MVT::Other, &MemOps[0], MemOps.size());
}
SDOperand
X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDOperand Op, SelectionDAG &DAG) {
unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getValue();
switch (IntNo) {
default: return SDOperand(); // Don't custom lower most intrinsics.
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
case Intrinsic::x86_sse_comieq_ss:
case Intrinsic::x86_sse_comilt_ss:
case Intrinsic::x86_sse_comile_ss:
case Intrinsic::x86_sse_comigt_ss:
case Intrinsic::x86_sse_comige_ss:
case Intrinsic::x86_sse_comineq_ss:
case Intrinsic::x86_sse_ucomieq_ss:
case Intrinsic::x86_sse_ucomilt_ss:
case Intrinsic::x86_sse_ucomile_ss:
case Intrinsic::x86_sse_ucomigt_ss:
case Intrinsic::x86_sse_ucomige_ss:
case Intrinsic::x86_sse_ucomineq_ss:
case Intrinsic::x86_sse2_comieq_sd:
case Intrinsic::x86_sse2_comilt_sd:
case Intrinsic::x86_sse2_comile_sd:
case Intrinsic::x86_sse2_comigt_sd:
case Intrinsic::x86_sse2_comige_sd:
case Intrinsic::x86_sse2_comineq_sd:
case Intrinsic::x86_sse2_ucomieq_sd:
case Intrinsic::x86_sse2_ucomilt_sd:
case Intrinsic::x86_sse2_ucomile_sd:
case Intrinsic::x86_sse2_ucomigt_sd:
case Intrinsic::x86_sse2_ucomige_sd:
case Intrinsic::x86_sse2_ucomineq_sd: {
unsigned Opc = 0;
ISD::CondCode CC = ISD::SETCC_INVALID;
switch (IntNo) {
default: break;
case Intrinsic::x86_sse_comieq_ss:
case Intrinsic::x86_sse2_comieq_sd:
Opc = X86ISD::COMI;
CC = ISD::SETEQ;
break;
case Intrinsic::x86_sse_comilt_ss:
case Intrinsic::x86_sse2_comilt_sd:
Opc = X86ISD::COMI;
CC = ISD::SETLT;
break;
case Intrinsic::x86_sse_comile_ss:
Opc = X86ISD::COMI;
CC = ISD::SETLE;
break;
case Intrinsic::x86_sse_comigt_ss:
Opc = X86ISD::COMI;
CC = ISD::SETGT;
break;
case Intrinsic::x86_sse_comige_ss:
Opc = X86ISD::COMI;
CC = ISD::SETGE;
break;
case Intrinsic::x86_sse_comineq_ss:
Opc = X86ISD::COMI;
CC = ISD::SETNE;
break;
case Intrinsic::x86_sse_ucomieq_ss:
Opc = X86ISD::UCOMI;
CC = ISD::SETEQ;
break;
case Intrinsic::x86_sse_ucomilt_ss:
Opc = X86ISD::UCOMI;
CC = ISD::SETLT;
break;
case Intrinsic::x86_sse_ucomile_ss:
Opc = X86ISD::UCOMI;
CC = ISD::SETLE;
break;
case Intrinsic::x86_sse_ucomigt_ss:
Opc = X86ISD::UCOMI;
CC = ISD::SETGT;
break;
case Intrinsic::x86_sse_ucomige_ss:
Opc = X86ISD::UCOMI;
CC = ISD::SETGE;
break;
case Intrinsic::x86_sse_ucomineq_ss:
case Intrinsic::x86_sse2_ucomineq_sd:
Opc = X86ISD::UCOMI;
CC = ISD::SETNE;
break;
unsigned X86CC;
SDOperand LHS = Op.getOperand(1);
SDOperand RHS = Op.getOperand(2);
translateX86CC(CC, true, X86CC, LHS, RHS, DAG);
const MVT::ValueType *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag);
SDOperand Ops1[] = { DAG.getEntryNode(), LHS, RHS };
SDOperand Cond = DAG.getNode(Opc, VTs, 2, Ops1, 3);
VTs = DAG.getNodeValueTypes(MVT::i8, MVT::Flag);
SDOperand Ops2[] = { DAG.getConstant(X86CC, MVT::i8), Cond };
SDOperand SetCC = DAG.getNode(X86ISD::SETCC, VTs, 2, Ops2, 2);
return DAG.getNode(ISD::ANY_EXTEND, MVT::i32, SetCC);
}
SDOperand X86TargetLowering::LowerRETURNADDR(SDOperand Op, SelectionDAG &DAG) {
// Depths > 0 not supported yet!
if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0)
return SDOperand();
// Just load the return address
SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG);
return DAG.getLoad(getPointerTy(), DAG.getEntryNode(), RetAddrFI, NULL, 0);
}
SDOperand X86TargetLowering::LowerFRAMEADDR(SDOperand Op, SelectionDAG &DAG) {
// Depths > 0 not supported yet!
if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0)
return SDOperand();
SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG);
return DAG.getNode(ISD::SUB, getPointerTy(), RetAddrFI,
DAG.getConstant(4, getPointerTy()));
}
/// LowerOperation - Provide custom lowering hooks for some operations.
///
SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) {
switch (Op.getOpcode()) {
default: assert(0 && "Should not custom lower this!");
case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG);
case ISD::SHL_PARTS:
case ISD::SRA_PARTS:
case ISD::SRL_PARTS: return LowerShift(Op, DAG);
case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG);
case ISD::FABS: return LowerFABS(Op, DAG);
case ISD::FNEG: return LowerFNEG(Op, DAG);
case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
case ISD::SETCC: return LowerSETCC(Op, DAG, DAG.getEntryNode());
case ISD::SELECT: return LowerSELECT(Op, DAG);
case ISD::BRCOND: return LowerBRCOND(Op, DAG);
case ISD::JumpTable: return LowerJumpTable(Op, DAG);
case ISD::CALL: return LowerCALL(Op, DAG);
case ISD::RET: return LowerRET(Op, DAG);
case ISD::FORMAL_ARGUMENTS: return LowerFORMAL_ARGUMENTS(Op, DAG);
case ISD::MEMSET: return LowerMEMSET(Op, DAG);
case ISD::MEMCPY: return LowerMEMCPY(Op, DAG);
case ISD::READCYCLECOUNTER: return LowerREADCYCLCECOUNTER(Op, DAG);
case ISD::VASTART: return LowerVASTART(Op, DAG);
case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
}
const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
switch (Opcode) {
default: return NULL;
case X86ISD::SHLD: return "X86ISD::SHLD";
case X86ISD::SHRD: return "X86ISD::SHRD";
case X86ISD::FOR: return "X86ISD::FOR";
case X86ISD::FXOR: return "X86ISD::FXOR";
case X86ISD::FSRL: return "X86ISD::FSRL";
case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG";
case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
case X86ISD::FLD: return "X86ISD::FLD";
case X86ISD::FST: return "X86ISD::FST";
case X86ISD::FP_GET_RESULT: return "X86ISD::FP_GET_RESULT";
case X86ISD::FP_SET_RESULT: return "X86ISD::FP_SET_RESULT";
case X86ISD::CALL: return "X86ISD::CALL";
case X86ISD::TAILCALL: return "X86ISD::TAILCALL";
case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG";
case X86ISD::CMP: return "X86ISD::CMP";
case X86ISD::COMI: return "X86ISD::COMI";
case X86ISD::UCOMI: return "X86ISD::UCOMI";
case X86ISD::SETCC: return "X86ISD::SETCC";
case X86ISD::CMOV: return "X86ISD::CMOV";
case X86ISD::BRCOND: return "X86ISD::BRCOND";
case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG";
case X86ISD::REP_STOS: return "X86ISD::REP_STOS";
case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS";
case X86ISD::LOAD_PACK: return "X86ISD::LOAD_PACK";
case X86ISD::LOAD_UA: return "X86ISD::LOAD_UA";
case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg";
case X86ISD::Wrapper: return "X86ISD::Wrapper";
case X86ISD::S2VEC: return "X86ISD::S2VEC";
case X86ISD::PEXTRW: return "X86ISD::PEXTRW";
case X86ISD::PINSRW: return "X86ISD::PINSRW";
case X86ISD::FMAX: return "X86ISD::FMAX";
case X86ISD::FMIN: return "X86ISD::FMIN";
/// isLegalAddressImmediate - Return true if the integer value or
/// GlobalValue can be used as the offset of the target addressing mode.
bool X86TargetLowering::isLegalAddressImmediate(int64_t V) const {
// X86 allows a sign-extended 32-bit immediate field.
return (V > -(1LL << 32) && V < (1LL << 32)-1);
}
bool X86TargetLowering::isLegalAddressImmediate(GlobalValue *GV) const {
// In 64-bit mode, GV is 64-bit so it won't fit in the 32-bit displacement
// field unless we are in small code model.
if (Subtarget->is64Bit() &&
getTargetMachine().getCodeModel() != CodeModel::Small)
Anton Korobeynikov
committed
return (!Subtarget->GVRequiresExtraLoad(GV, getTargetMachine(), false));
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
}
/// isShuffleMaskLegal - Targets can use this to indicate that they only
/// support *some* VECTOR_SHUFFLE operations, those with specific masks.
/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
/// are assumed to be legal.
bool
X86TargetLowering::isShuffleMaskLegal(SDOperand Mask, MVT::ValueType VT) const {
// Only do shuffles on 128-bit vector types for now.
if (MVT::getSizeInBits(VT) == 64) return false;
return (Mask.Val->getNumOperands() <= 4 ||
isSplatMask(Mask.Val) ||
isPSHUFHW_PSHUFLWMask(Mask.Val) ||
X86::isUNPCKLMask(Mask.Val) ||
X86::isUNPCKL_v_undef_Mask(Mask.Val) ||
X86::isUNPCKHMask(Mask.Val));
}
bool X86TargetLowering::isVectorClearMaskLegal(std::vector<SDOperand> &BVOps,
MVT::ValueType EVT,
SelectionDAG &DAG) const {
unsigned NumElts = BVOps.size();
// Only do shuffles on 128-bit vector types for now.
if (MVT::getSizeInBits(EVT) * NumElts == 64) return false;
if (NumElts == 2) return true;
if (NumElts == 4) {
return (isMOVLMask(BVOps) || isCommutedMOVL(BVOps, true) ||
isSHUFPMask(BVOps) || isCommutedSHUFP(BVOps));
}
return false;
}
//===----------------------------------------------------------------------===//
// X86 Scheduler Hooks
//===----------------------------------------------------------------------===//
MachineBasicBlock *
X86TargetLowering::InsertAtEndOfBasicBlock(MachineInstr *MI,
MachineBasicBlock *BB) {
const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
switch (MI->getOpcode()) {
default: assert(false && "Unexpected instr type to insert");
case X86::CMOV_FR32:
case X86::CMOV_FR64:
case X86::CMOV_V4F32:
case X86::CMOV_V2F64:
case X86::CMOV_V2I64: {
// To "insert" a SELECT_CC instruction, we actually have to insert the
// diamond control-flow pattern. The incoming instruction knows the
// destination vreg to set, the condition code register to branch on, the
// true/false values to select between, and a branch opcode to use.
const BasicBlock *LLVM_BB = BB->getBasicBlock();
ilist<MachineBasicBlock>::iterator It = BB;
++It;
// thisMBB:
// ...
// TrueVal = ...
// cmpTY ccX, r1, r2
// bCC copy1MBB
// fallthrough --> copy0MBB
MachineBasicBlock *thisMBB = BB;
MachineBasicBlock *copy0MBB = new MachineBasicBlock(LLVM_BB);
MachineBasicBlock *sinkMBB = new MachineBasicBlock(LLVM_BB);
unsigned Opc =
X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm());
BuildMI(BB, TII->get(Opc)).addMBB(sinkMBB);
MachineFunction *F = BB->getParent();
F->getBasicBlockList().insert(It, copy0MBB);
F->getBasicBlockList().insert(It, sinkMBB);
// Update machine-CFG edges by first adding all successors of the current
// block to the new block which will contain the Phi node for the select.
for(MachineBasicBlock::succ_iterator i = BB->succ_begin(),
e = BB->succ_end(); i != e; ++i)
sinkMBB->addSuccessor(*i);
// Next, remove all successors of the current block, and add the true
// and fallthrough blocks as its successors.
while(!BB->succ_empty())
BB->removeSuccessor(BB->succ_begin());
BB->addSuccessor(copy0MBB);
BB->addSuccessor(sinkMBB);
// copy0MBB:
// %FalseValue = ...
// # fallthrough to sinkMBB
BB = copy0MBB;
// Update machine-CFG edges
BB->addSuccessor(sinkMBB);
// sinkMBB:
// %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
// ...
BB = sinkMBB;
BuildMI(BB, TII->get(X86::PHI), MI->getOperand(0).getReg())
.addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB)
.addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
delete MI; // The pseudo instruction is gone now.
return BB;
}
case X86::FP_TO_INT16_IN_MEM:
case X86::FP_TO_INT32_IN_MEM:
case X86::FP_TO_INT64_IN_MEM: {
// Change the floating point control register to use "round towards zero"
// mode when truncating to an integer value.
MachineFunction *F = BB->getParent();
int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2);
addFrameReference(BuildMI(BB, TII->get(X86::FNSTCW16m)), CWFrameIdx);
// Load the old value of the high byte of the control word...
unsigned OldCW =
F->getSSARegMap()->createVirtualRegister(X86::GR16RegisterClass);
addFrameReference(BuildMI(BB, TII->get(X86::MOV16rm), OldCW), CWFrameIdx);
// Set the high part to be round to zero...
addFrameReference(BuildMI(BB, TII->get(X86::MOV16mi)), CWFrameIdx)
.addImm(0xC7F);
// Reload the modified control word now...
addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx);
// Restore the memory image of control word to original value
addFrameReference(BuildMI(BB, TII->get(X86::MOV16mr)), CWFrameIdx)
.addReg(OldCW);
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
// Get the X86 opcode to use.
unsigned Opc;
switch (MI->getOpcode()) {
default: assert(0 && "illegal opcode!");
case X86::FP_TO_INT16_IN_MEM: Opc = X86::FpIST16m; break;
case X86::FP_TO_INT32_IN_MEM: Opc = X86::FpIST32m; break;
case X86::FP_TO_INT64_IN_MEM: Opc = X86::FpIST64m; break;
}
X86AddressMode AM;
MachineOperand &Op = MI->getOperand(0);
if (Op.isRegister()) {
AM.BaseType = X86AddressMode::RegBase;
AM.Base.Reg = Op.getReg();
} else {
AM.BaseType = X86AddressMode::FrameIndexBase;
AM.Base.FrameIndex = Op.getFrameIndex();
}
Op = MI->getOperand(1);
if (Op.isImmediate())
AM.Scale = Op.getImm();
Op = MI->getOperand(2);
if (Op.isImmediate())
AM.IndexReg = Op.getImm();
Op = MI->getOperand(3);
if (Op.isGlobalAddress()) {
AM.GV = Op.getGlobal();
} else {
AM.Disp = Op.getImm();
addFullAddress(BuildMI(BB, TII->get(Opc)), AM)
.addReg(MI->getOperand(4).getReg());
// Reload the original control word now.
addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx);
delete MI; // The pseudo instruction is gone now.
return BB;
}
}
}
//===----------------------------------------------------------------------===//
// X86 Optimization Hooks
//===----------------------------------------------------------------------===//
void X86TargetLowering::computeMaskedBitsForTargetNode(const SDOperand Op,
uint64_t Mask,
uint64_t &KnownZero,
uint64_t &KnownOne,
unsigned Depth) const {
assert((Opc >= ISD::BUILTIN_OP_END ||
Opc == ISD::INTRINSIC_WO_CHAIN ||
Opc == ISD::INTRINSIC_W_CHAIN ||
Opc == ISD::INTRINSIC_VOID) &&
"Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!");
case X86ISD::SETCC:
KnownZero |= (MVT::getIntVTBitMask(Op.getValueType()) ^ 1ULL);
break;
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
/// getShuffleScalarElt - Returns the scalar element that will make up the ith
/// element of the result of the vector shuffle.
static SDOperand getShuffleScalarElt(SDNode *N, unsigned i, SelectionDAG &DAG) {
MVT::ValueType VT = N->getValueType(0);
SDOperand PermMask = N->getOperand(2);
unsigned NumElems = PermMask.getNumOperands();
SDOperand V = (i < NumElems) ? N->getOperand(0) : N->getOperand(1);
i %= NumElems;
if (V.getOpcode() == ISD::SCALAR_TO_VECTOR) {
return (i == 0)
? V.getOperand(0) : DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(VT));
} else if (V.getOpcode() == ISD::VECTOR_SHUFFLE) {
SDOperand Idx = PermMask.getOperand(i);
if (Idx.getOpcode() == ISD::UNDEF)
return DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(VT));
return getShuffleScalarElt(V.Val,cast<ConstantSDNode>(Idx)->getValue(),DAG);
}
return SDOperand();
}
/// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the
/// node is a GlobalAddress + an offset.
static bool isGAPlusOffset(SDNode *N, GlobalValue* &GA, int64_t &Offset) {
unsigned Opc = N->getOpcode();
Evan Cheng
committed
if (Opc == X86ISD::Wrapper) {
if (dyn_cast<GlobalAddressSDNode>(N->getOperand(0))) {
GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal();
return true;
}
} else if (Opc == ISD::ADD) {
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
SDOperand N1 = N->getOperand(0);
SDOperand N2 = N->getOperand(1);
if (isGAPlusOffset(N1.Val, GA, Offset)) {
ConstantSDNode *V = dyn_cast<ConstantSDNode>(N2);
if (V) {
Offset += V->getSignExtended();
return true;
}
} else if (isGAPlusOffset(N2.Val, GA, Offset)) {
ConstantSDNode *V = dyn_cast<ConstantSDNode>(N1);
if (V) {
Offset += V->getSignExtended();
return true;
}
}
}
return false;
}
/// isConsecutiveLoad - Returns true if N is loading from an address of Base
/// + Dist * Size.
static bool isConsecutiveLoad(SDNode *N, SDNode *Base, int Dist, int Size,
MachineFrameInfo *MFI) {
if (N->getOperand(0).Val != Base->getOperand(0).Val)
return false;
SDOperand Loc = N->getOperand(1);
SDOperand BaseLoc = Base->getOperand(1);
if (Loc.getOpcode() == ISD::FrameIndex) {
if (BaseLoc.getOpcode() != ISD::FrameIndex)
return false;
int FI = dyn_cast<FrameIndexSDNode>(Loc)->getIndex();
int BFI = dyn_cast<FrameIndexSDNode>(BaseLoc)->getIndex();
int FS = MFI->getObjectSize(FI);
int BFS = MFI->getObjectSize(BFI);
if (FS != BFS || FS != Size) return false;
return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Size);
} else {
GlobalValue *GV1 = NULL;
GlobalValue *GV2 = NULL;
int64_t Offset1 = 0;
int64_t Offset2 = 0;
bool isGA1 = isGAPlusOffset(Loc.Val, GV1, Offset1);
bool isGA2 = isGAPlusOffset(BaseLoc.Val, GV2, Offset2);
if (isGA1 && isGA2 && GV1 == GV2)
return Offset1 == (Offset2 + Dist*Size);
}
return false;
}
static bool isBaseAlignment16(SDNode *Base, MachineFrameInfo *MFI,
const X86Subtarget *Subtarget) {
GlobalValue *GV;
int64_t Offset;
if (isGAPlusOffset(Base, GV, Offset))
return (GV->getAlignment() >= 16 && (Offset % 16) == 0);
else {
assert(Base->getOpcode() == ISD::FrameIndex && "Unexpected base node!");
int BFI = dyn_cast<FrameIndexSDNode>(Base)->getIndex();
if (BFI < 0)
// Fixed objects do not specify alignment, however the offsets are known.
return ((Subtarget->getStackAlignment() % 16) == 0 &&
(MFI->getObjectOffset(BFI) % 16) == 0);
else
return MFI->getObjectAlignment(BFI) >= 16;
}
return false;
}