X86InstrInfo.td

               "#ATOMNAND8 PSEUDO!", 
               [(set GR8:$dst, (atomic_load_nand_8 addr:$ptr, GR8:$val))]>;
}

let Constraints = "$val1 = $dst1, $val2 = $dst2", 
                  Defs = [EFLAGS, EAX, EBX, ECX, EDX],
                  Uses = [EAX, EBX, ECX, EDX],
                  mayLoad = 1, mayStore = 1,
                  usesCustomInserter = 1 in {
def ATOMAND6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMAND6432 PSEUDO!", []>;
def ATOMOR6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMOR6432 PSEUDO!", []>;
def ATOMXOR6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMXOR6432 PSEUDO!", []>;
def ATOMNAND6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMNAND6432 PSEUDO!", []>;
def ATOMADD6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMADD6432 PSEUDO!", []>;
def ATOMSUB6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMSUB6432 PSEUDO!", []>;
def ATOMSWAP6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMSWAP6432 PSEUDO!", []>;
}

// Segmentation support instructions.

def LAR16rm : I<0x02, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src), 
                "lar{w}\t{$src, $dst|$dst, $src}", []>, TB, OpSize;
def LAR16rr : I<0x02, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src),
                "lar{w}\t{$src, $dst|$dst, $src}", []>, TB, OpSize;

// i16mem operand in LAR32rm and GR32 operand in LAR32rr is not a typo.
def LAR32rm : I<0x02, MRMSrcMem, (outs GR32:$dst), (ins i16mem:$src), 
                "lar{l}\t{$src, $dst|$dst, $src}", []>, TB;
def LAR32rr : I<0x02, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src),
                "lar{l}\t{$src, $dst|$dst, $src}", []>, TB;

def LSL16rm : I<0x03, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src),
                "lsl{w}\t{$src, $dst|$dst, $src}", []>, TB, OpSize; 
def LSL16rr : I<0x03, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src),
                "lsl{w}\t{$src, $dst|$dst, $src}", []>, TB, OpSize;
def LSL32rm : I<0x03, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src),
                "lsl{l}\t{$src, $dst|$dst, $src}", []>, TB; 
def LSL32rr : I<0x03, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src),
                "lsl{l}\t{$src, $dst|$dst, $src}", []>, TB;
                
def INVLPG : I<0x01, RawFrm, (outs), (ins), "invlpg", []>, TB;

def STRr : I<0x00, MRM1r, (outs GR16:$dst), (ins),
             "str{w}\t{$dst}", []>, TB;
def STRm : I<0x00, MRM1m, (outs i16mem:$dst), (ins),
             "str{w}\t{$dst}", []>, TB;
def LTRr : I<0x00, MRM3r, (outs), (ins GR16:$src),
             "ltr{w}\t{$src}", []>, TB;
def LTRm : I<0x00, MRM3m, (outs), (ins i16mem:$src),
             "ltr{w}\t{$src}", []>, TB;
             
def PUSHFS16 : I<0xa0, RawFrm, (outs), (ins),
                 "push{w}\t%fs", []>, OpSize, TB;
def PUSHFS32 : I<0xa0, RawFrm, (outs), (ins),
                 "push{l}\t%fs", []>, TB;
def PUSHGS16 : I<0xa8, RawFrm, (outs), (ins),
                 "push{w}\t%gs", []>, OpSize, TB;
def PUSHGS32 : I<0xa8, RawFrm, (outs), (ins),
                 "push{l}\t%gs", []>, TB;

def POPFS16 : I<0xa1, RawFrm, (outs), (ins),
                "pop{w}\t%fs", []>, OpSize, TB;
def POPFS32 : I<0xa1, RawFrm, (outs), (ins),
                "pop{l}\t%fs", []>, TB;
def POPGS16 : I<0xa9, RawFrm, (outs), (ins),
                "pop{w}\t%gs", []>, OpSize, TB;
def POPGS32 : I<0xa9, RawFrm, (outs), (ins),
                "pop{l}\t%gs", []>, TB;

def LDS16rm : I<0xc5, MRMSrcMem, (outs GR16:$dst), (ins opaque32mem:$src),
                "lds{w}\t{$src, $dst|$dst, $src}", []>, OpSize;
def LDS32rm : I<0xc5, MRMSrcMem, (outs GR32:$dst), (ins opaque48mem:$src),
                "lds{l}\t{$src, $dst|$dst, $src}", []>;
def LSS16rm : I<0xb2, MRMSrcMem, (outs GR16:$dst), (ins opaque32mem:$src),
                "lss{w}\t{$src, $dst|$dst, $src}", []>, TB, OpSize;
def LSS32rm : I<0xb2, MRMSrcMem, (outs GR32:$dst), (ins opaque48mem:$src),
                "lss{l}\t{$src, $dst|$dst, $src}", []>, TB;
def LES16rm : I<0xc4, MRMSrcMem, (outs GR16:$dst), (ins opaque32mem:$src),
                "les{w}\t{$src, $dst|$dst, $src}", []>, OpSize;
def LES32rm : I<0xc4, MRMSrcMem, (outs GR32:$dst), (ins opaque48mem:$src),
                "les{l}\t{$src, $dst|$dst, $src}", []>;
def LFS16rm : I<0xb4, MRMSrcMem, (outs GR16:$dst), (ins opaque32mem:$src),
                "lfs{w}\t{$src, $dst|$dst, $src}", []>, TB, OpSize;
def LFS32rm : I<0xb4, MRMSrcMem, (outs GR32:$dst), (ins opaque48mem:$src),
                "lfs{l}\t{$src, $dst|$dst, $src}", []>, TB;
def LGS16rm : I<0xb5, MRMSrcMem, (outs GR16:$dst), (ins opaque32mem:$src),
                "lgs{w}\t{$src, $dst|$dst, $src}", []>, TB, OpSize;
def LGS32rm : I<0xb5, MRMSrcMem, (outs GR32:$dst), (ins opaque48mem:$src),
                "lgs{l}\t{$src, $dst|$dst, $src}", []>, TB;

def VERRr : I<0x00, MRM4r, (outs), (ins GR16:$seg),
              "verr\t$seg", []>, TB;
def VERRm : I<0x00, MRM4m, (outs), (ins i16mem:$seg),
              "verr\t$seg", []>, TB;
def VERWr : I<0x00, MRM5r, (outs), (ins GR16:$seg),
              "verw\t$seg", []>, TB;
def VERWm : I<0x00, MRM5m, (outs), (ins i16mem:$seg),
              "verw\t$seg", []>, TB;

// Descriptor-table support instructions

def SGDTm : I<0x01, MRM0m, (outs opaque48mem:$dst), (ins),
              "sgdt\t$dst", []>, TB;
def SIDTm : I<0x01, MRM1m, (outs opaque48mem:$dst), (ins),
              "sidt\t$dst", []>, TB;
def SLDT16r : I<0x00, MRM0r, (outs GR16:$dst), (ins),
                "sldt{w}\t$dst", []>, TB;
def SLDT16m : I<0x00, MRM0m, (outs i16mem:$dst), (ins),
                "sldt{w}\t$dst", []>, TB;
def LGDTm : I<0x01, MRM2m, (outs), (ins opaque48mem:$src),
              "lgdt\t$src", []>, TB;
def LIDTm : I<0x01, MRM3m, (outs), (ins opaque48mem:$src),
              "lidt\t$src", []>, TB;
def LLDT16r : I<0x00, MRM2r, (outs), (ins GR16:$src),
                "lldt{w}\t$src", []>, TB;
def LLDT16m : I<0x00, MRM2m, (outs), (ins i16mem:$src),
                "lldt{w}\t$src", []>, TB;
                
// String manipulation instructions

def LODSB : I<0xAC, RawFrm, (outs), (ins), "lodsb", []>;
def LODSW : I<0xAD, RawFrm, (outs), (ins), "lodsw", []>, OpSize;
def LODSD : I<0xAD, RawFrm, (outs), (ins), "lods{l|d}", []>;

def OUTSB : I<0x6E, RawFrm, (outs), (ins), "outsb", []>;
def OUTSW : I<0x6F, RawFrm, (outs), (ins), "outsw", []>, OpSize;
def OUTSD : I<0x6F, RawFrm, (outs), (ins), "outs{l|d}", []>;

// CPU flow control instructions

def HLT : I<0xF4, RawFrm, (outs), (ins), "hlt", []>;
def RSM : I<0xAA, RawFrm, (outs), (ins), "rsm", []>, TB;

// FPU control instructions

def FNINIT : I<0xE3, RawFrm, (outs), (ins), "fninit", []>, DB;

// Flag instructions

def CLC : I<0xF8, RawFrm, (outs), (ins), "clc", []>;
def STC : I<0xF9, RawFrm, (outs), (ins), "stc", []>;
def CLI : I<0xFA, RawFrm, (outs), (ins), "cli", []>;
def STI : I<0xFB, RawFrm, (outs), (ins), "sti", []>;
def CLD : I<0xFC, RawFrm, (outs), (ins), "cld", []>;
def STD : I<0xFD, RawFrm, (outs), (ins), "std", []>;
def CMC : I<0xF5, RawFrm, (outs), (ins), "cmc", []>;

def CLTS : I<0x06, RawFrm, (outs), (ins), "clts", []>, TB;

// Table lookup instructions

def XLAT : I<0xD7, RawFrm, (outs), (ins), "xlatb", []>;

// Specialized register support

def WRMSR : I<0x30, RawFrm, (outs), (ins), "wrmsr", []>, TB;
def RDMSR : I<0x32, RawFrm, (outs), (ins), "rdmsr", []>, TB;
def RDPMC : I<0x33, RawFrm, (outs), (ins), "rdpmc", []>, TB;

def SMSW16r : I<0x01, MRM4r, (outs GR16:$dst), (ins), 
                "smsw{w}\t$dst", []>, OpSize, TB;
def SMSW32r : I<0x01, MRM4r, (outs GR32:$dst), (ins), 
                "smsw{l}\t$dst", []>, TB;
// For memory operands, there is only a 16-bit form
def SMSW16m : I<0x01, MRM4m, (outs i16mem:$dst), (ins),
                "smsw{w}\t$dst", []>, TB;

def LMSW16r : I<0x01, MRM6r, (outs), (ins GR16:$src),
                "lmsw{w}\t$src", []>, TB;
def LMSW16m : I<0x01, MRM6m, (outs), (ins i16mem:$src),
                "lmsw{w}\t$src", []>, TB;
                
def CPUID : I<0xA2, RawFrm, (outs), (ins), "cpuid", []>, TB;

// Cache instructions

def INVD : I<0x08, RawFrm, (outs), (ins), "invd", []>, TB;
def WBINVD : I<0x09, RawFrm, (outs), (ins), "wbinvd", []>, TB;

// VMX instructions

// 66 0F 38 80
def INVEPT : I<0x38, RawFrm, (outs), (ins), "invept", []>, OpSize, TB;
// 66 0F 38 81
def INVVPID : I<0x38, RawFrm, (outs), (ins), "invvpid", []>, OpSize, TB;
// 0F 01 C1
def VMCALL : I<0x01, RawFrm, (outs), (ins), "vmcall", []>, TB;
def VMCLEARm : I<0xC7, MRM6m, (outs), (ins i64mem:$vmcs),
  "vmclear\t$vmcs", []>, OpSize, TB;
// 0F 01 C2
def VMLAUNCH : I<0x01, RawFrm, (outs), (ins), "vmlaunch", []>, TB;
// 0F 01 C3
def VMRESUME : I<0x01, RawFrm, (outs), (ins), "vmresume", []>, TB;
def VMPTRLDm : I<0xC7, MRM6m, (outs), (ins i64mem:$vmcs),
  "vmptrld\t$vmcs", []>, TB;
def VMPTRSTm : I<0xC7, MRM7m, (outs i64mem:$vmcs), (ins),
  "vmptrst\t$vmcs", []>, TB;
def VMREAD64rm : I<0x78, MRMDestMem, (outs i64mem:$dst), (ins GR64:$src),
  "vmread{q}\t{$src, $dst|$dst, $src}", []>, TB;
def VMREAD64rr : I<0x78, MRMDestReg, (outs GR64:$dst), (ins GR64:$src),
  "vmread{q}\t{$src, $dst|$dst, $src}", []>, TB;
def VMREAD32rm : I<0x78, MRMDestMem, (outs i32mem:$dst), (ins GR32:$src),
  "vmread{l}\t{$src, $dst|$dst, $src}", []>, TB;
def VMREAD32rr : I<0x78, MRMDestReg, (outs GR32:$dst), (ins GR32:$src),
  "vmread{l}\t{$src, $dst|$dst, $src}", []>, TB;
def VMWRITE64rm : I<0x79, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src),
  "vmwrite{q}\t{$src, $dst|$dst, $src}", []>, TB;
def VMWRITE64rr : I<0x79, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src),
  "vmwrite{q}\t{$src, $dst|$dst, $src}", []>, TB;
def VMWRITE32rm : I<0x79, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src),
  "vmwrite{l}\t{$src, $dst|$dst, $src}", []>, TB;
def VMWRITE32rr : I<0x79, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src),
  "vmwrite{l}\t{$src, $dst|$dst, $src}", []>, TB;
// 0F 01 C4
def VMXOFF : I<0x01, RawFrm, (outs), (ins), "vmxoff", []>, OpSize;
def VMXON : I<0xC7, MRM6m, (outs), (ins i64mem:$vmxon),
  "vmxon\t{$vmxon}", []>, XD;

//===----------------------------------------------------------------------===//
// Non-Instruction Patterns
//===----------------------------------------------------------------------===//

// ConstantPool GlobalAddress, ExternalSymbol, and JumpTable
def : Pat<(i32 (X86Wrapper tconstpool  :$dst)), (MOV32ri tconstpool  :$dst)>;
def : Pat<(i32 (X86Wrapper tjumptable  :$dst)), (MOV32ri tjumptable  :$dst)>;
def : Pat<(i32 (X86Wrapper tglobaltlsaddr:$dst)),(MOV32ri tglobaltlsaddr:$dst)>;
def : Pat<(i32 (X86Wrapper tglobaladdr :$dst)), (MOV32ri tglobaladdr :$dst)>;
def : Pat<(i32 (X86Wrapper texternalsym:$dst)), (MOV32ri texternalsym:$dst)>;
def : Pat<(i32 (X86Wrapper tblockaddress:$dst)), (MOV32ri tblockaddress:$dst)>;

def : Pat<(add GR32:$src1, (X86Wrapper tconstpool:$src2)),
          (ADD32ri GR32:$src1, tconstpool:$src2)>;
def : Pat<(add GR32:$src1, (X86Wrapper tjumptable:$src2)),
          (ADD32ri GR32:$src1, tjumptable:$src2)>;
def : Pat<(add GR32:$src1, (X86Wrapper tglobaladdr :$src2)),
          (ADD32ri GR32:$src1, tglobaladdr:$src2)>;
def : Pat<(add GR32:$src1, (X86Wrapper texternalsym:$src2)),
          (ADD32ri GR32:$src1, texternalsym:$src2)>;
def : Pat<(add GR32:$src1, (X86Wrapper tblockaddress:$src2)),
          (ADD32ri GR32:$src1, tblockaddress:$src2)>;

def : Pat<(store (i32 (X86Wrapper tglobaladdr:$src)), addr:$dst),
          (MOV32mi addr:$dst, tglobaladdr:$src)>;
def : Pat<(store (i32 (X86Wrapper texternalsym:$src)), addr:$dst),
          (MOV32mi addr:$dst, texternalsym:$src)>;
def : Pat<(store (i32 (X86Wrapper tblockaddress:$src)), addr:$dst),
          (MOV32mi addr:$dst, tblockaddress:$src)>;

// Calls
// tailcall stuff
def : Pat<(X86tcret GR32:$dst, imm:$off),
          (TCRETURNri GR32:$dst, imm:$off)>;

def : Pat<(X86tcret (i32 tglobaladdr:$dst), imm:$off),
          (TCRETURNdi texternalsym:$dst, imm:$off)>;

def : Pat<(X86tcret (i32 texternalsym:$dst), imm:$off),
          (TCRETURNdi texternalsym:$dst, imm:$off)>;

// Normal calls, with various flavors of addresses.
def : Pat<(X86call (i32 tglobaladdr:$dst)),
          (CALLpcrel32 tglobaladdr:$dst)>;
def : Pat<(X86call (i32 texternalsym:$dst)),
          (CALLpcrel32 texternalsym:$dst)>;
def : Pat<(X86call (i32 imm:$dst)),
          (CALLpcrel32 imm:$dst)>, Requires<[CallImmAddr]>;

// X86 specific add which produces a flag.
def : Pat<(addc GR32:$src1, GR32:$src2),
          (ADD32rr GR32:$src1, GR32:$src2)>;
def : Pat<(addc GR32:$src1, (load addr:$src2)),
          (ADD32rm GR32:$src1, addr:$src2)>;
def : Pat<(addc GR32:$src1, imm:$src2),
          (ADD32ri GR32:$src1, imm:$src2)>;
def : Pat<(addc GR32:$src1, i32immSExt8:$src2),
          (ADD32ri8 GR32:$src1, i32immSExt8:$src2)>;

def : Pat<(subc GR32:$src1, GR32:$src2),
          (SUB32rr GR32:$src1, GR32:$src2)>;
def : Pat<(subc GR32:$src1, (load addr:$src2)),
          (SUB32rm GR32:$src1, addr:$src2)>;
def : Pat<(subc GR32:$src1, imm:$src2),
          (SUB32ri GR32:$src1, imm:$src2)>;
def : Pat<(subc GR32:$src1, i32immSExt8:$src2),
          (SUB32ri8 GR32:$src1, i32immSExt8:$src2)>;

// Comparisons.

// TEST R,R is smaller than CMP R,0
def : Pat<(parallel (X86cmp GR8:$src1, 0), (implicit EFLAGS)),
          (TEST8rr GR8:$src1, GR8:$src1)>;
def : Pat<(parallel (X86cmp GR16:$src1, 0), (implicit EFLAGS)),
          (TEST16rr GR16:$src1, GR16:$src1)>;
def : Pat<(parallel (X86cmp GR32:$src1, 0), (implicit EFLAGS)),
          (TEST32rr GR32:$src1, GR32:$src1)>;

// Conditional moves with folded loads with operands swapped and conditions
// inverted.
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_B, EFLAGS),
          (CMOVAE16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_B, EFLAGS),
          (CMOVAE32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_AE, EFLAGS),
          (CMOVB16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_AE, EFLAGS),
          (CMOVB32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_E, EFLAGS),
          (CMOVNE16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_E, EFLAGS),
          (CMOVNE32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_NE, EFLAGS),
          (CMOVE16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_NE, EFLAGS),
          (CMOVE32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_BE, EFLAGS),
          (CMOVA16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_BE, EFLAGS),
          (CMOVA32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_A, EFLAGS),
          (CMOVBE16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_A, EFLAGS),
          (CMOVBE32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_L, EFLAGS),
          (CMOVGE16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_L, EFLAGS),
          (CMOVGE32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_GE, EFLAGS),
          (CMOVL16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_GE, EFLAGS),
          (CMOVL32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_LE, EFLAGS),
          (CMOVG16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_LE, EFLAGS),
          (CMOVG32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_G, EFLAGS),
          (CMOVLE16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_G, EFLAGS),
          (CMOVLE32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_P, EFLAGS),
          (CMOVNP16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_P, EFLAGS),
          (CMOVNP32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_NP, EFLAGS),
          (CMOVP16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_NP, EFLAGS),
          (CMOVP32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_S, EFLAGS),
          (CMOVNS16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_S, EFLAGS),
          (CMOVNS32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_NS, EFLAGS),
          (CMOVS16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_NS, EFLAGS),
          (CMOVS32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_O, EFLAGS),
          (CMOVNO16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_O, EFLAGS),
          (CMOVNO32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_NO, EFLAGS),
          (CMOVO16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_NO, EFLAGS),
          (CMOVO32rm GR32:$src2, addr:$src1)>;

// zextload bool -> zextload byte
def : Pat<(zextloadi8i1  addr:$src), (MOV8rm     addr:$src)>;
def : Pat<(zextloadi16i1 addr:$src), (MOVZX16rm8 addr:$src)>;
def : Pat<(zextloadi32i1 addr:$src), (MOVZX32rm8 addr:$src)>;

// extload bool -> extload byte
def : Pat<(extloadi8i1 addr:$src),   (MOV8rm      addr:$src)>;
def : Pat<(extloadi16i1 addr:$src),  (MOVZX16rm8  addr:$src)>;
def : Pat<(extloadi32i1 addr:$src),  (MOVZX32rm8  addr:$src)>;
def : Pat<(extloadi16i8 addr:$src),  (MOVZX16rm8  addr:$src)>;
def : Pat<(extloadi32i8 addr:$src),  (MOVZX32rm8  addr:$src)>;
def : Pat<(extloadi32i16 addr:$src), (MOVZX32rm16 addr:$src)>;

// anyext. Define these to do an explicit zero-extend to
// avoid partial-register updates.
def : Pat<(i16 (anyext GR8 :$src)), (MOVZX16rr8  GR8 :$src)>;
def : Pat<(i32 (anyext GR8 :$src)), (MOVZX32rr8  GR8 :$src)>;
def : Pat<(i32 (anyext GR16:$src)), (MOVZX32rr16 GR16:$src)>;

// (and (i32 load), 255) -> (zextload i8)
def : Pat<(i32 (and (nvloadi32 addr:$src), (i32 255))),
          (MOVZX32rm8 addr:$src)>;
def : Pat<(i32 (and (nvloadi32 addr:$src), (i32 65535))),
          (MOVZX32rm16 addr:$src)>;

//===----------------------------------------------------------------------===//
// Some peepholes
//===----------------------------------------------------------------------===//

// Odd encoding trick: -128 fits into an 8-bit immediate field while
// +128 doesn't, so in this special case use a sub instead of an add.
def : Pat<(add GR16:$src1, 128),
          (SUB16ri8 GR16:$src1, -128)>;
def : Pat<(store (add (loadi16 addr:$dst), 128), addr:$dst),
          (SUB16mi8 addr:$dst, -128)>;
def : Pat<(add GR32:$src1, 128),
          (SUB32ri8 GR32:$src1, -128)>;
def : Pat<(store (add (loadi32 addr:$dst), 128), addr:$dst),
          (SUB32mi8 addr:$dst, -128)>;

// r & (2^16-1) ==> movz
def : Pat<(and GR32:$src1, 0xffff),
          (MOVZX32rr16 (EXTRACT_SUBREG GR32:$src1, x86_subreg_16bit))>;
// r & (2^8-1) ==> movz
def : Pat<(and GR32:$src1, 0xff),
          (MOVZX32rr8 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR32:$src1, 
                                                             GR32_ABCD)),
                                      x86_subreg_8bit))>,
      Requires<[In32BitMode]>;
// r & (2^8-1) ==> movz
def : Pat<(and GR16:$src1, 0xff),
          (MOVZX16rr8 (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src1, 
                                                             GR16_ABCD)),
                                      x86_subreg_8bit))>,
      Requires<[In32BitMode]>;

// sext_inreg patterns
def : Pat<(sext_inreg GR32:$src, i16),
          (MOVSX32rr16 (EXTRACT_SUBREG GR32:$src, x86_subreg_16bit))>;
def : Pat<(sext_inreg GR32:$src, i8),
          (MOVSX32rr8 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR32:$src, 
                                                             GR32_ABCD)),
                                      x86_subreg_8bit))>,
      Requires<[In32BitMode]>;
def : Pat<(sext_inreg GR16:$src, i8),
          (MOVSX16rr8 (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src, 
                                                             GR16_ABCD)),
                                      x86_subreg_8bit))>,
      Requires<[In32BitMode]>;

// trunc patterns
def : Pat<(i16 (trunc GR32:$src)),
          (EXTRACT_SUBREG GR32:$src, x86_subreg_16bit)>;
def : Pat<(i8 (trunc GR32:$src)),
          (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR32:$src, GR32_ABCD)),
                          x86_subreg_8bit)>,
      Requires<[In32BitMode]>;
def : Pat<(i8 (trunc GR16:$src)),
          (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src, GR16_ABCD)),
                          x86_subreg_8bit)>,
      Requires<[In32BitMode]>;

// h-register tricks
def : Pat<(i8 (trunc (srl_su GR16:$src, (i8 8)))),
          (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR16:$src, GR16_ABCD)),
                          x86_subreg_8bit_hi)>,
      Requires<[In32BitMode]>;
def : Pat<(i8 (trunc (srl_su GR32:$src, (i8 8)))),
          (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR32:$src, GR32_ABCD)),
                          x86_subreg_8bit_hi)>,
      Requires<[In32BitMode]>;
def : Pat<(srl_su GR16:$src, (i8 8)),
          (EXTRACT_SUBREG
            (MOVZX32rr8
              (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src, GR16_ABCD)),
                              x86_subreg_8bit_hi)),
            x86_subreg_16bit)>,
      Requires<[In32BitMode]>;
def : Pat<(i32 (zext (srl_su GR16:$src, (i8 8)))),
          (MOVZX32rr8 (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src, 
                                                             GR16_ABCD)),
                                      x86_subreg_8bit_hi))>,
      Requires<[In32BitMode]>;
def : Pat<(i32 (anyext (srl_su GR16:$src, (i8 8)))),
          (MOVZX32rr8 (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src, 
                                                             GR16_ABCD)),
                                      x86_subreg_8bit_hi))>,
      Requires<[In32BitMode]>;
def : Pat<(and (srl_su GR32:$src, (i8 8)), (i32 255)),
          (MOVZX32rr8 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR32:$src, 
                                                             GR32_ABCD)),
                                      x86_subreg_8bit_hi))>,
      Requires<[In32BitMode]>;

// (shl x, 1) ==> (add x, x)
def : Pat<(shl GR8 :$src1, (i8 1)), (ADD8rr  GR8 :$src1, GR8 :$src1)>;
def : Pat<(shl GR16:$src1, (i8 1)), (ADD16rr GR16:$src1, GR16:$src1)>;
def : Pat<(shl GR32:$src1, (i8 1)), (ADD32rr GR32:$src1, GR32:$src1)>;

// (shl x (and y, 31)) ==> (shl x, y)
def : Pat<(shl GR8:$src1, (and CL:$amt, 31)),
          (SHL8rCL GR8:$src1)>;
def : Pat<(shl GR16:$src1, (and CL:$amt, 31)),
          (SHL16rCL GR16:$src1)>;
def : Pat<(shl GR32:$src1, (and CL:$amt, 31)),
          (SHL32rCL GR32:$src1)>;
def : Pat<(store (shl (loadi8 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SHL8mCL addr:$dst)>;
def : Pat<(store (shl (loadi16 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SHL16mCL addr:$dst)>;
def : Pat<(store (shl (loadi32 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SHL32mCL addr:$dst)>;

def : Pat<(srl GR8:$src1, (and CL:$amt, 31)),
          (SHR8rCL GR8:$src1)>;
def : Pat<(srl GR16:$src1, (and CL:$amt, 31)),
          (SHR16rCL GR16:$src1)>;
def : Pat<(srl GR32:$src1, (and CL:$amt, 31)),
          (SHR32rCL GR32:$src1)>;
def : Pat<(store (srl (loadi8 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SHR8mCL addr:$dst)>;
def : Pat<(store (srl (loadi16 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SHR16mCL addr:$dst)>;
def : Pat<(store (srl (loadi32 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SHR32mCL addr:$dst)>;

def : Pat<(sra GR8:$src1, (and CL:$amt, 31)),
          (SAR8rCL GR8:$src1)>;
def : Pat<(sra GR16:$src1, (and CL:$amt, 31)),
          (SAR16rCL GR16:$src1)>;
def : Pat<(sra GR32:$src1, (and CL:$amt, 31)),
          (SAR32rCL GR32:$src1)>;
def : Pat<(store (sra (loadi8 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SAR8mCL addr:$dst)>;
def : Pat<(store (sra (loadi16 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SAR16mCL addr:$dst)>;
def : Pat<(store (sra (loadi32 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SAR32mCL addr:$dst)>;

// (or (x >> c) | (y << (32 - c))) ==> (shrd32 x, y, c)
def : Pat<(or (srl GR32:$src1, CL:$amt),
              (shl GR32:$src2, (sub 32, CL:$amt))),
          (SHRD32rrCL GR32:$src1, GR32:$src2)>;

def : Pat<(store (or (srl (loadi32 addr:$dst), CL:$amt),
                     (shl GR32:$src2, (sub 32, CL:$amt))), addr:$dst),
          (SHRD32mrCL addr:$dst, GR32:$src2)>;

def : Pat<(or (srl GR32:$src1, (i8 (trunc ECX:$amt))),
              (shl GR32:$src2, (i8 (trunc (sub 32, ECX:$amt))))),
          (SHRD32rrCL GR32:$src1, GR32:$src2)>;

def : Pat<(store (or (srl (loadi32 addr:$dst), (i8 (trunc ECX:$amt))),
                     (shl GR32:$src2, (i8 (trunc (sub 32, ECX:$amt))))),
                 addr:$dst),
          (SHRD32mrCL addr:$dst, GR32:$src2)>;

def : Pat<(shrd GR32:$src1, (i8 imm:$amt1), GR32:$src2, (i8 imm:$amt2)),
          (SHRD32rri8 GR32:$src1, GR32:$src2, (i8 imm:$amt1))>;

def : Pat<(store (shrd (loadi32 addr:$dst), (i8 imm:$amt1),
                       GR32:$src2, (i8 imm:$amt2)), addr:$dst),
          (SHRD32mri8 addr:$dst, GR32:$src2, (i8 imm:$amt1))>;

// (or (x << c) | (y >> (32 - c))) ==> (shld32 x, y, c)
def : Pat<(or (shl GR32:$src1, CL:$amt),
              (srl GR32:$src2, (sub 32, CL:$amt))),
          (SHLD32rrCL GR32:$src1, GR32:$src2)>;

def : Pat<(store (or (shl (loadi32 addr:$dst), CL:$amt),
                     (srl GR32:$src2, (sub 32, CL:$amt))), addr:$dst),
          (SHLD32mrCL addr:$dst, GR32:$src2)>;

def : Pat<(or (shl GR32:$src1, (i8 (trunc ECX:$amt))),
              (srl GR32:$src2, (i8 (trunc (sub 32, ECX:$amt))))),
          (SHLD32rrCL GR32:$src1, GR32:$src2)>;

def : Pat<(store (or (shl (loadi32 addr:$dst), (i8 (trunc ECX:$amt))),
                     (srl GR32:$src2, (i8 (trunc (sub 32, ECX:$amt))))),
                 addr:$dst),
          (SHLD32mrCL addr:$dst, GR32:$src2)>;

def : Pat<(shld GR32:$src1, (i8 imm:$amt1), GR32:$src2, (i8 imm:$amt2)),
          (SHLD32rri8 GR32:$src1, GR32:$src2, (i8 imm:$amt1))>;

def : Pat<(store (shld (loadi32 addr:$dst), (i8 imm:$amt1),
                       GR32:$src2, (i8 imm:$amt2)), addr:$dst),
          (SHLD32mri8 addr:$dst, GR32:$src2, (i8 imm:$amt1))>;

// (or (x >> c) | (y << (16 - c))) ==> (shrd16 x, y, c)
def : Pat<(or (srl GR16:$src1, CL:$amt),
              (shl GR16:$src2, (sub 16, CL:$amt))),
          (SHRD16rrCL GR16:$src1, GR16:$src2)>;

def : Pat<(store (or (srl (loadi16 addr:$dst), CL:$amt),
                     (shl GR16:$src2, (sub 16, CL:$amt))), addr:$dst),
          (SHRD16mrCL addr:$dst, GR16:$src2)>;

def : Pat<(or (srl GR16:$src1, (i8 (trunc CX:$amt))),
              (shl GR16:$src2, (i8 (trunc (sub 16, CX:$amt))))),
          (SHRD16rrCL GR16:$src1, GR16:$src2)>;

def : Pat<(store (or (srl (loadi16 addr:$dst), (i8 (trunc CX:$amt))),
                     (shl GR16:$src2, (i8 (trunc (sub 16, CX:$amt))))),
                 addr:$dst),
          (SHRD16mrCL addr:$dst, GR16:$src2)>;

def : Pat<(shrd GR16:$src1, (i8 imm:$amt1), GR16:$src2, (i8 imm:$amt2)),
          (SHRD16rri8 GR16:$src1, GR16:$src2, (i8 imm:$amt1))>;

def : Pat<(store (shrd (loadi16 addr:$dst), (i8 imm:$amt1),
                       GR16:$src2, (i8 imm:$amt2)), addr:$dst),
          (SHRD16mri8 addr:$dst, GR16:$src2, (i8 imm:$amt1))>;

// (or (x << c) | (y >> (16 - c))) ==> (shld16 x, y, c)
def : Pat<(or (shl GR16:$src1, CL:$amt),
              (srl GR16:$src2, (sub 16, CL:$amt))),
          (SHLD16rrCL GR16:$src1, GR16:$src2)>;

def : Pat<(store (or (shl (loadi16 addr:$dst), CL:$amt),
                     (srl GR16:$src2, (sub 16, CL:$amt))), addr:$dst),
          (SHLD16mrCL addr:$dst, GR16:$src2)>;

def : Pat<(or (shl GR16:$src1, (i8 (trunc CX:$amt))),
              (srl GR16:$src2, (i8 (trunc (sub 16, CX:$amt))))),
          (SHLD16rrCL GR16:$src1, GR16:$src2)>;

def : Pat<(store (or (shl (loadi16 addr:$dst), (i8 (trunc CX:$amt))),
                     (srl GR16:$src2, (i8 (trunc (sub 16, CX:$amt))))),
                 addr:$dst),
          (SHLD16mrCL addr:$dst, GR16:$src2)>;

def : Pat<(shld GR16:$src1, (i8 imm:$amt1), GR16:$src2, (i8 imm:$amt2)),
          (SHLD16rri8 GR16:$src1, GR16:$src2, (i8 imm:$amt1))>;

def : Pat<(store (shld (loadi16 addr:$dst), (i8 imm:$amt1),
                       GR16:$src2, (i8 imm:$amt2)), addr:$dst),
          (SHLD16mri8 addr:$dst, GR16:$src2, (i8 imm:$amt1))>;

// (anyext (setcc_carry)) -> (setcc_carry)
def : Pat<(i16 (anyext (i8 (X86setcc_c X86_COND_B, EFLAGS)))),
          (SETB_C16r)>;
def : Pat<(i32 (anyext (i8 (X86setcc_c X86_COND_B, EFLAGS)))),
          (SETB_C32r)>;

//===----------------------------------------------------------------------===//
// EFLAGS-defining Patterns
//===----------------------------------------------------------------------===//

// Register-Register Addition with EFLAGS result
def : Pat<(parallel (X86add_flag GR8:$src1, GR8:$src2),
                    (implicit EFLAGS)),
          (ADD8rr GR8:$src1, GR8:$src2)>;
def : Pat<(parallel (X86add_flag GR16:$src1, GR16:$src2),
                    (implicit EFLAGS)),
          (ADD16rr GR16:$src1, GR16:$src2)>;
def : Pat<(parallel (X86add_flag GR32:$src1, GR32:$src2),
                    (implicit EFLAGS)),
          (ADD32rr GR32:$src1, GR32:$src2)>;

// Register-Memory Addition with EFLAGS result
def : Pat<(parallel (X86add_flag GR8:$src1, (loadi8 addr:$src2)),
                    (implicit EFLAGS)),
          (ADD8rm GR8:$src1, addr:$src2)>;
def : Pat<(parallel (X86add_flag GR16:$src1, (loadi16 addr:$src2)),
                    (implicit EFLAGS)),
          (ADD16rm GR16:$src1, addr:$src2)>;
def : Pat<(parallel (X86add_flag GR32:$src1, (loadi32 addr:$src2)),
                    (implicit EFLAGS)),
          (ADD32rm GR32:$src1, addr:$src2)>;

// Register-Integer Addition with EFLAGS result
def : Pat<(parallel (X86add_flag GR8:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (ADD8ri GR8:$src1, imm:$src2)>;
def : Pat<(parallel (X86add_flag GR16:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (ADD16ri GR16:$src1, imm:$src2)>;
def : Pat<(parallel (X86add_flag GR32:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (ADD32ri GR32:$src1, imm:$src2)>;
def : Pat<(parallel (X86add_flag GR16:$src1, i16immSExt8:$src2),
                    (implicit EFLAGS)),
          (ADD16ri8 GR16:$src1, i16immSExt8:$src2)>;
def : Pat<(parallel (X86add_flag GR32:$src1, i32immSExt8:$src2),
                    (implicit EFLAGS)),
          (ADD32ri8 GR32:$src1, i32immSExt8:$src2)>;

// Memory-Register Addition with EFLAGS result
def : Pat<(parallel (store (X86add_flag (loadi8 addr:$dst), GR8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD8mr addr:$dst, GR8:$src2)>;
def : Pat<(parallel (store (X86add_flag (loadi16 addr:$dst), GR16:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD16mr addr:$dst, GR16:$src2)>;
def : Pat<(parallel (store (X86add_flag (loadi32 addr:$dst), GR32:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD32mr addr:$dst, GR32:$src2)>;

// Memory-Integer Addition with EFLAGS result
def : Pat<(parallel (store (X86add_flag (loadi8 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD8mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86add_flag (loadi16 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD16mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86add_flag (loadi32 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD32mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86add_flag (loadi16 addr:$dst), i16immSExt8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD16mi8 addr:$dst, i16immSExt8:$src2)>;
def : Pat<(parallel (store (X86add_flag (loadi32 addr:$dst), i32immSExt8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD32mi8 addr:$dst, i32immSExt8:$src2)>;

// Register-Register Subtraction with EFLAGS result
def : Pat<(parallel (X86sub_flag GR8:$src1, GR8:$src2),
                    (implicit EFLAGS)),
          (SUB8rr GR8:$src1, GR8:$src2)>;
def : Pat<(parallel (X86sub_flag GR16:$src1, GR16:$src2),
                    (implicit EFLAGS)),
          (SUB16rr GR16:$src1, GR16:$src2)>;
def : Pat<(parallel (X86sub_flag GR32:$src1, GR32:$src2),
                    (implicit EFLAGS)),
          (SUB32rr GR32:$src1, GR32:$src2)>;

// Register-Memory Subtraction with EFLAGS result
def : Pat<(parallel (X86sub_flag GR8:$src1, (loadi8 addr:$src2)),
                    (implicit EFLAGS)),
          (SUB8rm GR8:$src1, addr:$src2)>;
def : Pat<(parallel (X86sub_flag GR16:$src1, (loadi16 addr:$src2)),
                    (implicit EFLAGS)),
          (SUB16rm GR16:$src1, addr:$src2)>;
def : Pat<(parallel (X86sub_flag GR32:$src1, (loadi32 addr:$src2)),
                    (implicit EFLAGS)),
          (SUB32rm GR32:$src1, addr:$src2)>;

// Register-Integer Subtraction with EFLAGS result
def : Pat<(parallel (X86sub_flag GR8:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (SUB8ri GR8:$src1, imm:$src2)>;
def : Pat<(parallel (X86sub_flag GR16:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (SUB16ri GR16:$src1, imm:$src2)>;
def : Pat<(parallel (X86sub_flag GR32:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (SUB32ri GR32:$src1, imm:$src2)>;
def : Pat<(parallel (X86sub_flag GR16:$src1, i16immSExt8:$src2),
                    (implicit EFLAGS)),
          (SUB16ri8 GR16:$src1, i16immSExt8:$src2)>;
def : Pat<(parallel (X86sub_flag GR32:$src1, i32immSExt8:$src2),
                    (implicit EFLAGS)),
          (SUB32ri8 GR32:$src1, i32immSExt8:$src2)>;

// Memory-Register Subtraction with EFLAGS result
def : Pat<(parallel (store (X86sub_flag (loadi8 addr:$dst), GR8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB8mr addr:$dst, GR8:$src2)>;
def : Pat<(parallel (store (X86sub_flag (loadi16 addr:$dst), GR16:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB16mr addr:$dst, GR16:$src2)>;
def : Pat<(parallel (store (X86sub_flag (loadi32 addr:$dst), GR32:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB32mr addr:$dst, GR32:$src2)>;

// Memory-Integer Subtraction with EFLAGS result
def : Pat<(parallel (store (X86sub_flag (loadi8 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB8mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86sub_flag (loadi16 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB16mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86sub_flag (loadi32 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB32mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86sub_flag (loadi16 addr:$dst), i16immSExt8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB16mi8 addr:$dst, i16immSExt8:$src2)>;
def : Pat<(parallel (store (X86sub_flag (loadi32 addr:$dst), i32immSExt8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB32mi8 addr:$dst, i32immSExt8:$src2)>;


// Register-Register Signed Integer Multiply with EFLAGS result
def : Pat<(parallel (X86smul_flag GR16:$src1, GR16:$src2),
                    (implicit EFLAGS)),
          (IMUL16rr GR16:$src1, GR16:$src2)>;
def : Pat<(parallel (X86smul_flag GR32:$src1, GR32:$src2),
                    (implicit EFLAGS)),
          (IMUL32rr GR32:$src1, GR32:$src2)>;

// Register-Memory Signed Integer Multiply with EFLAGS result
def : Pat<(parallel (X86smul_flag GR16:$src1, (loadi16 addr:$src2)),
                    (implicit EFLAGS)),
          (IMUL16rm GR16:$src1, addr:$src2)>;
def : Pat<(parallel (X86smul_flag GR32:$src1, (loadi32 addr:$src2)),
                    (implicit EFLAGS)),
          (IMUL32rm GR32:$src1, addr:$src2)>;

// Register-Integer Signed Integer Multiply with EFLAGS result
def : Pat<(parallel (X86smul_flag GR16:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (IMUL16rri GR16:$src1, imm:$src2)>;
def : Pat<(parallel (X86smul_flag GR32:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (IMUL32rri GR32:$src1, imm:$src2)>;
def : Pat<(parallel (X86smul_flag GR16:$src1, i16immSExt8:$src2),
                    (implicit EFLAGS)),
          (IMUL16rri8 GR16:$src1, i16immSExt8:$src2)>;
def : Pat<(parallel (X86smul_flag GR32:$src1, i32immSExt8:$src2),
                    (implicit EFLAGS)),
          (IMUL32rri8 GR32:$src1, i32immSExt8:$src2)>;

// Memory-Integer Signed Integer Multiply with EFLAGS result
def : Pat<(parallel (X86smul_flag (loadi16 addr:$src1), imm:$src2),
                    (implicit EFLAGS)),
          (IMUL16rmi addr:$src1, imm:$src2)>;
def : Pat<(parallel (X86smul_flag (loadi32 addr:$src1), imm:$src2),
                    (implicit EFLAGS)),
          (IMUL32rmi addr:$src1, imm:$src2)>;
def : Pat<(parallel (X86smul_flag (loadi16 addr:$src1), i16immSExt8:$src2),
                    (implicit EFLAGS)),
          (IMUL16rmi8 addr:$src1, i16immSExt8:$src2)>;
def : Pat<(parallel (X86smul_flag (loadi32 addr:$src1), i32immSExt8:$src2),
                    (implicit EFLAGS)),
          (IMUL32rmi8 addr:$src1, i32immSExt8:$src2)>;

// Optimize multiply by 2 with EFLAGS result.
let AddedComplexity = 2 in {
def : Pat<(parallel (X86smul_flag GR16:$src1, 2),
                    (implicit EFLAGS)),
          (ADD16rr GR16:$src1, GR16:$src1)>;

def : Pat<(parallel (X86smul_flag GR32:$src1, 2),
                    (implicit EFLAGS)),
          (ADD32rr GR32:$src1, GR32:$src1)>;
}

// INC and DEC with EFLAGS result. Note that these do not set CF.
def : Pat<(parallel (X86inc_flag GR8:$src), (implicit EFLAGS)),
          (INC8r GR8:$src)>;
def : Pat<(parallel (store (i8 (X86inc_flag (loadi8 addr:$dst))), addr:$dst),
                    (implicit EFLAGS)),
          (INC8m addr:$dst)>;
def : Pat<(parallel (X86dec_flag GR8:$src), (implicit EFLAGS)),
          (DEC8r GR8:$src)>;
def : Pat<(parallel (store (i8 (X86dec_flag (loadi8 addr:$dst))), addr:$dst),
                    (implicit EFLAGS)),
          (DEC8m addr:$dst)>;

def : Pat<(parallel (X86inc_flag GR16:$src), (implicit EFLAGS)),
          (INC16r GR16:$src)>, Requires<[In32BitMode]>;
def : Pat<(parallel (store (i16 (X86inc_flag (loadi16 addr:$dst))), addr:$dst),
                    (implicit EFLAGS)),
          (INC16m addr:$dst)>, Requires<[In32BitMode]>;
def : Pat<(parallel (X86dec_flag GR16:$src), (implicit EFLAGS)),
          (DEC16r GR16:$src)>, Requires<[In32BitMode]>;
def : Pat<(parallel (store (i16 (X86dec_flag (loadi16 addr:$dst))), addr:$dst),
                    (implicit EFLAGS)),
          (DEC16m addr:$dst)>, Requires<[In32BitMode]>;

def : Pat<(parallel (X86inc_flag GR32:$src), (implicit EFLAGS)),
          (INC32r GR32:$src)>, Requires<[In32BitMode]>;
def : Pat<(parallel (store (i32 (X86inc_flag (loadi32 addr:$dst))), addr:$dst),
                    (implicit EFLAGS)),
          (INC32m addr:$dst)>, Requires<[In32BitMode]>;
def : Pat<(parallel (X86dec_flag GR32:$src), (implicit EFLAGS)),
          (DEC32r GR32:$src)>, Requires<[In32BitMode]>;
def : Pat<(parallel (store (i32 (X86dec_flag (loadi32 addr:$dst))), addr:$dst),
                    (implicit EFLAGS)),
          (DEC32m addr:$dst)>, Requires<[In32BitMode]>;

// Register-Register Or with EFLAGS result
def : Pat<(parallel (X86or_flag GR8:$src1, GR8:$src2),
                    (implicit EFLAGS)),
          (OR8rr GR8:$src1, GR8:$src2)>;
def : Pat<(parallel (X86or_flag GR16:$src1, GR16:$src2),
                    (implicit EFLAGS)),
          (OR16rr GR16:$src1, GR16:$src2)>;
def : Pat<(parallel (X86or_flag GR32:$src1, GR32:$src2),
                    (implicit EFLAGS)),
          (OR32rr GR32:$src1, GR32:$src2)>;

// Register-Memory Or with EFLAGS result
def : Pat<(parallel (X86or_flag GR8:$src1, (loadi8 addr:$src2)),
                    (implicit EFLAGS)),
          (OR8rm GR8:$src1, addr:$src2)>;
def : Pat<(parallel (X86or_flag GR16:$src1, (loadi16 addr:$src2)),
                    (implicit EFLAGS)),
          (OR16rm GR16:$src1, addr:$src2)>;
def : Pat<(parallel (X86or_flag GR32:$src1, (loadi32 addr:$src2)),
                    (implicit EFLAGS)),
          (OR32rm GR32:$src1, addr:$src2)>;

// Register-Integer Or with EFLAGS result
def : Pat<(parallel (X86or_flag GR8:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (OR8ri GR8:$src1, imm:$src2)>;
def : Pat<(parallel (X86or_flag GR16:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (OR16ri GR16:$src1, imm:$src2)>;
def : Pat<(parallel (X86or_flag GR32:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (OR32ri GR32:$src1, imm:$src2)>;
def : Pat<(parallel (X86or_flag GR16:$src1, i16immSExt8:$src2),
                    (implicit EFLAGS)),
          (OR16ri8 GR16:$src1, i16immSExt8:$src2)>;
def : Pat<(parallel (X86or_flag GR32:$src1, i32immSExt8:$src2),
                    (implicit EFLAGS)),
          (OR32ri8 GR32:$src1, i32immSExt8:$src2)>;

// Memory-Register Or with EFLAGS result
def : Pat<(parallel (store (X86or_flag (loadi8 addr:$dst), GR8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (OR8mr addr:$dst, GR8:$src2)>;
def : Pat<(parallel (store (X86or_flag (loadi16 addr:$dst), GR16:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (OR16mr addr:$dst, GR16:$src2)>;
def : Pat<(parallel (store (X86or_flag (loadi32 addr:$dst), GR32:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (OR32mr addr:$dst, GR32:$src2)>;

// Memory-Integer Or with EFLAGS result
def : Pat<(parallel (store (X86or_flag (loadi8 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (OR8mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86or_flag (loadi16 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (OR16mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86or_flag (loadi32 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (OR32mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86or_flag (loadi16 addr:$dst), i16immSExt8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (OR16mi8 addr:$dst, i16immSExt8:$src2)>;
def : Pat<(parallel (store (X86or_flag (loadi32 addr:$dst), i32immSExt8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (OR32mi8 addr:$dst, i32immSExt8:$src2)>;

// Register-Register XOr with EFLAGS result
def : Pat<(parallel (X86xor_flag GR8:$src1, GR8:$src2),
                    (implicit EFLAGS)),
          (XOR8rr GR8:$src1, GR8:$src2)>;
def : Pat<(parallel (X86xor_flag GR16:$src1, GR16:$src2),
                    (implicit EFLAGS)),
          (XOR16rr GR16:$src1, GR16:$src2)>;
def : Pat<(parallel (X86xor_flag GR32:$src1, GR32:$src2),
                    (implicit EFLAGS)),
          (XOR32rr GR32:$src1, GR32:$src2)>;

// Register-Memory XOr with EFLAGS result
def : Pat<(parallel (X86xor_flag GR8:$src1, (loadi8 addr:$src2)),
                    (implicit EFLAGS)),
          (XOR8rm GR8:$src1, addr:$src2)>;
def : Pat<(parallel (X86xor_flag GR16:$src1, (loadi16 addr:$src2)),
                    (implicit EFLAGS)),
          (XOR16rm GR16:$src1, addr:$src2)>;
def : Pat<(parallel (X86xor_flag GR32:$src1, (loadi32 addr:$src2)),
                    (implicit EFLAGS)),
          (XOR32rm GR32:$src1, addr:$src2)>;

// Register-Integer XOr with EFLAGS result
def : Pat<(parallel (X86xor_flag GR8:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (XOR8ri GR8:$src1, imm:$src2)>;
def : Pat<(parallel (X86xor_flag GR16:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (XOR16ri GR16:$src1, imm:$src2)>;
def : Pat<(parallel (X86xor_flag GR32:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (XOR32ri GR32:$src1, imm:$src2)>;
def : Pat<(parallel (X86xor_flag GR16:$src1, i16immSExt8:$src2),
                    (implicit EFLAGS)),
          (XOR16ri8 GR16:$src1, i16immSExt8:$src2)>;
def : Pat<(parallel (X86xor_flag GR32:$src1, i32immSExt8:$src2),
                    (implicit EFLAGS)),
          (XOR32ri8 GR32:$src1, i32immSExt8:$src2)>;