X86InstrInfo.td

// DWARF Pseudo Instructions
//

def DWARF_LOC   : I<0, Pseudo, (outs),
                    (ins i32imm:$line, i32imm:$col, i32imm:$file),
                    ".loc\t${file:debug} ${line:debug} ${col:debug}",
                    [(dwarf_loc (i32 imm:$line), (i32 imm:$col),
                      (i32 imm:$file))]>;

//===----------------------------------------------------------------------===//
// EH Pseudo Instructions
//
let isTerminator = 1, isReturn = 1, isBarrier = 1,
    hasCtrlDep = 1 in {
def EH_RETURN   : I<0xC3, RawFrm, (outs), (ins GR32:$addr),
                    "ret\t#eh_return, addr: $addr",
                    [(X86ehret GR32:$addr)]>;

}

//===----------------------------------------------------------------------===//
// Atomic support
//

// Atomic swap. These are just normal xchg instructions. But since a memory
// operand is referenced, the atomicity is ensured.
let Constraints = "$val = $dst" in {
def XCHG32rm : I<0x87, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$ptr, GR32:$val),
               "xchg{l}\t{$val, $ptr|$ptr, $val}", 
               [(set GR32:$dst, (atomic_swap_32 addr:$ptr, GR32:$val))]>;
def XCHG16rm : I<0x87, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$ptr, GR16:$val),
               "xchg{w}\t{$val, $ptr|$ptr, $val}", 
               [(set GR16:$dst, (atomic_swap_16 addr:$ptr, GR16:$val))]>, 
                OpSize;
def XCHG8rm  : I<0x86, MRMSrcMem, (outs GR8:$dst), (ins i8mem:$ptr, GR8:$val),
               "xchg{b}\t{$val, $ptr|$ptr, $val}", 
               [(set GR8:$dst, (atomic_swap_8 addr:$ptr, GR8:$val))]>;
}

// Atomic compare and swap.
let Defs = [EAX, EFLAGS], Uses = [EAX] in {
def LCMPXCHG32 : I<0xB1, MRMDestMem, (outs), (ins i32mem:$ptr, GR32:$swap),
               "lock\n\t"
               "cmpxchg{l}\t{$swap, $ptr|$ptr, $swap}",
               [(X86cas addr:$ptr, GR32:$swap, 4)]>, TB, LOCK;
}
let Defs = [EAX, EDX, EFLAGS], Uses = [EAX, EBX, ECX, EDX] in {
def LCMPXCHG8B : I<0xC7, MRM1m, (outs), (ins i32mem:$ptr),
               "lock\n\t"
               "cmpxchg8b\t$ptr",
               [(X86cas8 addr:$ptr)]>, TB, LOCK;
}

let Defs = [AX, EFLAGS], Uses = [AX] in {
def LCMPXCHG16 : I<0xB1, MRMDestMem, (outs), (ins i16mem:$ptr, GR16:$swap),
               "lock\n\t"
               "cmpxchg{w}\t{$swap, $ptr|$ptr, $swap}",
               [(X86cas addr:$ptr, GR16:$swap, 2)]>, TB, OpSize, LOCK;
}
let Defs = [AL, EFLAGS], Uses = [AL] in {
def LCMPXCHG8 : I<0xB0, MRMDestMem, (outs), (ins i8mem:$ptr, GR8:$swap),
               "lock\n\t"
               "cmpxchg{b}\t{$swap, $ptr|$ptr, $swap}",
               [(X86cas addr:$ptr, GR8:$swap, 1)]>, TB, LOCK;
}

// Atomic exchange and add
let Constraints = "$val = $dst", Defs = [EFLAGS] in {
def LXADD32 : I<0xC1, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$ptr, GR32:$val),
               "lock\n\t"
               "xadd{l}\t{$val, $ptr|$ptr, $val}",
               [(set GR32:$dst, (atomic_load_add_32 addr:$ptr, GR32:$val))]>,
                TB, LOCK;
def LXADD16 : I<0xC1, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$ptr, GR16:$val),
               "lock\n\t"
               "xadd{w}\t{$val, $ptr|$ptr, $val}",
               [(set GR16:$dst, (atomic_load_add_16 addr:$ptr, GR16:$val))]>,
                TB, OpSize, LOCK;
def LXADD8  : I<0xC0, MRMSrcMem, (outs GR8:$dst), (ins i8mem:$ptr, GR8:$val),
               "lock\n\t"
               "xadd{b}\t{$val, $ptr|$ptr, $val}",
               [(set GR8:$dst, (atomic_load_add_8 addr:$ptr, GR8:$val))]>,
                TB, LOCK;
}

// Atomic exchange, and, or, xor
let Constraints = "$val = $dst", Defs = [EFLAGS],
                  usesCustomDAGSchedInserter = 1 in {
def ATOMAND32 : I<0, Pseudo, (outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
               "#ATOMAND32 PSEUDO!", 
               [(set GR32:$dst, (atomic_load_and_32 addr:$ptr, GR32:$val))]>;
def ATOMOR32 : I<0, Pseudo, (outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
               "#ATOMOR32 PSEUDO!", 
               [(set GR32:$dst, (atomic_load_or_32 addr:$ptr, GR32:$val))]>;
def ATOMXOR32 : I<0, Pseudo,(outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
               "#ATOMXOR32 PSEUDO!", 
               [(set GR32:$dst, (atomic_load_xor_32 addr:$ptr, GR32:$val))]>;
def ATOMNAND32 : I<0, Pseudo,(outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
               "#ATOMNAND32 PSEUDO!", 
               [(set GR32:$dst, (atomic_load_nand_32 addr:$ptr, GR32:$val))]>;
def ATOMMIN32: I<0, Pseudo, (outs GR32:$dst), (ins i32mem:$ptr, GR32:$val),
               "#ATOMMIN32 PSEUDO!", 
               [(set GR32:$dst, (atomic_load_min_32 addr:$ptr, GR32:$val))]>;
def ATOMMAX32: I<0, Pseudo, (outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
               "#ATOMMAX32 PSEUDO!", 
               [(set GR32:$dst, (atomic_load_max_32 addr:$ptr, GR32:$val))]>;
def ATOMUMIN32: I<0, Pseudo, (outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
               "#ATOMUMIN32 PSEUDO!", 
               [(set GR32:$dst, (atomic_load_umin_32 addr:$ptr, GR32:$val))]>;
def ATOMUMAX32: I<0, Pseudo, (outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
               "#ATOMUMAX32 PSEUDO!", 
               [(set GR32:$dst, (atomic_load_umax_32 addr:$ptr, GR32:$val))]>;

def ATOMAND16 : I<0, Pseudo, (outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
               "#ATOMAND16 PSEUDO!", 
               [(set GR16:$dst, (atomic_load_and_16 addr:$ptr, GR16:$val))]>;
def ATOMOR16 : I<0, Pseudo, (outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
               "#ATOMOR16 PSEUDO!", 
               [(set GR16:$dst, (atomic_load_or_16 addr:$ptr, GR16:$val))]>;
def ATOMXOR16 : I<0, Pseudo,(outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
               "#ATOMXOR16 PSEUDO!", 
               [(set GR16:$dst, (atomic_load_xor_16 addr:$ptr, GR16:$val))]>;
def ATOMNAND16 : I<0, Pseudo,(outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
               "#ATOMNAND16 PSEUDO!", 
               [(set GR16:$dst, (atomic_load_nand_16 addr:$ptr, GR16:$val))]>;
def ATOMMIN16: I<0, Pseudo, (outs GR16:$dst), (ins i16mem:$ptr, GR16:$val),
               "#ATOMMIN16 PSEUDO!", 
               [(set GR16:$dst, (atomic_load_min_16 addr:$ptr, GR16:$val))]>;
def ATOMMAX16: I<0, Pseudo, (outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
               "#ATOMMAX16 PSEUDO!", 
               [(set GR16:$dst, (atomic_load_max_16 addr:$ptr, GR16:$val))]>;
def ATOMUMIN16: I<0, Pseudo, (outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
               "#ATOMUMIN16 PSEUDO!", 
               [(set GR16:$dst, (atomic_load_umin_16 addr:$ptr, GR16:$val))]>;
def ATOMUMAX16: I<0, Pseudo, (outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
               "#ATOMUMAX16 PSEUDO!", 
               [(set GR16:$dst, (atomic_load_umax_16 addr:$ptr, GR16:$val))]>;

def ATOMAND8 : I<0, Pseudo, (outs GR8:$dst),(ins i8mem:$ptr, GR8:$val),
               "#ATOMAND8 PSEUDO!", 
               [(set GR8:$dst, (atomic_load_and_8 addr:$ptr, GR8:$val))]>;
def ATOMOR8 : I<0, Pseudo, (outs GR8:$dst),(ins i8mem:$ptr, GR8:$val),
               "#ATOMOR8 PSEUDO!", 
               [(set GR8:$dst, (atomic_load_or_8 addr:$ptr, GR8:$val))]>;
def ATOMXOR8 : I<0, Pseudo,(outs GR8:$dst),(ins i8mem:$ptr, GR8:$val),
               "#ATOMXOR8 PSEUDO!", 
               [(set GR8:$dst, (atomic_load_xor_8 addr:$ptr, GR8:$val))]>;
def ATOMNAND8 : I<0, Pseudo,(outs GR8:$dst),(ins i8mem:$ptr, GR8:$val),
               "#ATOMNAND8 PSEUDO!", 
               [(set GR8:$dst, (atomic_load_nand_8 addr:$ptr, GR8:$val))]>;
}

let Constraints = "$val1 = $dst1, $val2 = $dst2", 
                  Defs = [EFLAGS, EAX, EBX, ECX, EDX],
                  Uses = [EAX, EBX, ECX, EDX],
                  mayLoad = 1, mayStore = 1,
                  usesCustomDAGSchedInserter = 1 in {
def ATOMAND6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMAND6432 PSEUDO!", []>;
def ATOMOR6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMOR6432 PSEUDO!", []>;
def ATOMXOR6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMXOR6432 PSEUDO!", []>;
def ATOMNAND6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMNAND6432 PSEUDO!", []>;
def ATOMADD6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMADD6432 PSEUDO!", []>;
def ATOMSUB6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMSUB6432 PSEUDO!", []>;
def ATOMSWAP6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
               "#ATOMSWAP6432 PSEUDO!", []>;
}

//===----------------------------------------------------------------------===//
// Non-Instruction Patterns
//===----------------------------------------------------------------------===//

// ConstantPool GlobalAddress, ExternalSymbol, and JumpTable
def : Pat<(i32 (X86Wrapper tconstpool  :$dst)), (MOV32ri tconstpool  :$dst)>;
def : Pat<(i32 (X86Wrapper tjumptable  :$dst)), (MOV32ri tjumptable  :$dst)>;
def : Pat<(i32 (X86Wrapper tglobaltlsaddr:$dst)),(MOV32ri tglobaltlsaddr:$dst)>;
def : Pat<(i32 (X86Wrapper tglobaladdr :$dst)), (MOV32ri tglobaladdr :$dst)>;
def : Pat<(i32 (X86Wrapper texternalsym:$dst)), (MOV32ri texternalsym:$dst)>;

def : Pat<(add GR32:$src1, (X86Wrapper tconstpool:$src2)),
          (ADD32ri GR32:$src1, tconstpool:$src2)>;
def : Pat<(add GR32:$src1, (X86Wrapper tjumptable:$src2)),
          (ADD32ri GR32:$src1, tjumptable:$src2)>;
def : Pat<(add GR32:$src1, (X86Wrapper tglobaladdr :$src2)),
          (ADD32ri GR32:$src1, tglobaladdr:$src2)>;
def : Pat<(add GR32:$src1, (X86Wrapper texternalsym:$src2)),
          (ADD32ri GR32:$src1, texternalsym:$src2)>;

def : Pat<(store (i32 (X86Wrapper tglobaladdr:$src)), addr:$dst),
          (MOV32mi addr:$dst, tglobaladdr:$src)>;
def : Pat<(store (i32 (X86Wrapper texternalsym:$src)), addr:$dst),
          (MOV32mi addr:$dst, texternalsym:$src)>;

// Calls
// tailcall stuff
def : Pat<(X86tailcall GR32:$dst),
          (TAILCALL)>;

def : Pat<(X86tailcall (i32 tglobaladdr:$dst)),
          (TAILCALL)>;
def : Pat<(X86tailcall (i32 texternalsym:$dst)),
          (TAILCALL)>;

def : Pat<(X86tcret GR32:$dst, imm:$off),
          (TCRETURNri GR32:$dst, imm:$off)>;

def : Pat<(X86tcret (i32 tglobaladdr:$dst), imm:$off),
          (TCRETURNdi texternalsym:$dst, imm:$off)>;

def : Pat<(X86tcret (i32 texternalsym:$dst), imm:$off),
          (TCRETURNdi texternalsym:$dst, imm:$off)>;

def : Pat<(X86call (i32 tglobaladdr:$dst)),
          (CALLpcrel32 tglobaladdr:$dst)>;
def : Pat<(X86call (i32 texternalsym:$dst)),
          (CALLpcrel32 texternalsym:$dst)>;

// X86 specific add which produces a flag.
def : Pat<(addc GR32:$src1, GR32:$src2),
          (ADD32rr GR32:$src1, GR32:$src2)>;
def : Pat<(addc GR32:$src1, (load addr:$src2)),
          (ADD32rm GR32:$src1, addr:$src2)>;
def : Pat<(addc GR32:$src1, imm:$src2),
          (ADD32ri GR32:$src1, imm:$src2)>;
def : Pat<(addc GR32:$src1, i32immSExt8:$src2),
          (ADD32ri8 GR32:$src1, i32immSExt8:$src2)>;

def : Pat<(subc GR32:$src1, GR32:$src2),
          (SUB32rr GR32:$src1, GR32:$src2)>;
def : Pat<(subc GR32:$src1, (load addr:$src2)),
          (SUB32rm GR32:$src1, addr:$src2)>;
def : Pat<(subc GR32:$src1, imm:$src2),
          (SUB32ri GR32:$src1, imm:$src2)>;
def : Pat<(subc GR32:$src1, i32immSExt8:$src2),
          (SUB32ri8 GR32:$src1, i32immSExt8:$src2)>;

// Comparisons.

// TEST R,R is smaller than CMP R,0
def : Pat<(parallel (X86cmp GR8:$src1, 0), (implicit EFLAGS)),
          (TEST8rr GR8:$src1, GR8:$src1)>;
def : Pat<(parallel (X86cmp GR16:$src1, 0), (implicit EFLAGS)),
          (TEST16rr GR16:$src1, GR16:$src1)>;
def : Pat<(parallel (X86cmp GR32:$src1, 0), (implicit EFLAGS)),
          (TEST32rr GR32:$src1, GR32:$src1)>;

// Conditional moves with folded loads with operands swapped and conditions
// inverted.
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_B, EFLAGS),
          (CMOVAE16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_B, EFLAGS),
          (CMOVAE32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_AE, EFLAGS),
          (CMOVB16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_AE, EFLAGS),
          (CMOVB32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_E, EFLAGS),
          (CMOVNE16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_E, EFLAGS),
          (CMOVNE32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_NE, EFLAGS),
          (CMOVE16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_NE, EFLAGS),
          (CMOVE32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_BE, EFLAGS),
          (CMOVA16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_BE, EFLAGS),
          (CMOVA32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_A, EFLAGS),
          (CMOVBE16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_A, EFLAGS),
          (CMOVBE32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_L, EFLAGS),
          (CMOVGE16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_L, EFLAGS),
          (CMOVGE32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_GE, EFLAGS),
          (CMOVL16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_GE, EFLAGS),
          (CMOVL32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_LE, EFLAGS),
          (CMOVG16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_LE, EFLAGS),
          (CMOVG32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_G, EFLAGS),
          (CMOVLE16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_G, EFLAGS),
          (CMOVLE32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_P, EFLAGS),
          (CMOVNP16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_P, EFLAGS),
          (CMOVNP32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_NP, EFLAGS),
          (CMOVP16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_NP, EFLAGS),
          (CMOVP32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_S, EFLAGS),
          (CMOVNS16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_S, EFLAGS),
          (CMOVNS32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_NS, EFLAGS),
          (CMOVS16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_NS, EFLAGS),
          (CMOVS32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_O, EFLAGS),
          (CMOVNO16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_O, EFLAGS),
          (CMOVNO32rm GR32:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, X86_COND_NO, EFLAGS),
          (CMOVO16rm GR16:$src2, addr:$src1)>;
def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, X86_COND_NO, EFLAGS),
          (CMOVO32rm GR32:$src2, addr:$src1)>;

// zextload bool -> zextload byte
def : Pat<(zextloadi8i1  addr:$src), (MOV8rm     addr:$src)>;
def : Pat<(zextloadi16i1 addr:$src), (MOVZX16rm8 addr:$src)>;
def : Pat<(zextloadi32i1 addr:$src), (MOVZX32rm8 addr:$src)>;

// extload bool -> extload byte
def : Pat<(extloadi8i1 addr:$src),   (MOV8rm      addr:$src)>;
def : Pat<(extloadi16i1 addr:$src),  (MOVZX16rm8  addr:$src)>,
         Requires<[In32BitMode]>;
def : Pat<(extloadi32i1 addr:$src),  (MOVZX32rm8  addr:$src)>;
def : Pat<(extloadi16i8 addr:$src),  (MOVZX16rm8  addr:$src)>,
         Requires<[In32BitMode]>;
def : Pat<(extloadi32i8 addr:$src),  (MOVZX32rm8  addr:$src)>;
def : Pat<(extloadi32i16 addr:$src), (MOVZX32rm16 addr:$src)>;

// anyext
def : Pat<(i16 (anyext GR8 :$src)), (MOVZX16rr8  GR8 :$src)>,
         Requires<[In32BitMode]>;
def : Pat<(i32 (anyext GR8 :$src)), (MOVZX32rr8  GR8 :$src)>,
         Requires<[In32BitMode]>;
def : Pat<(i32 (anyext GR16:$src)),
          (INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR16:$src, x86_subreg_16bit)>;

// (and (i32 load), 255) -> (zextload i8)
def : Pat<(i32 (and (nvloadi32 addr:$src), (i32 255))),
          (MOVZX32rm8 addr:$src)>;
def : Pat<(i32 (and (nvloadi32 addr:$src), (i32 65535))),
          (MOVZX32rm16 addr:$src)>;

//===----------------------------------------------------------------------===//
// Some peepholes
//===----------------------------------------------------------------------===//

// Odd encoding trick: -128 fits into an 8-bit immediate field while
// +128 doesn't, so in this special case use a sub instead of an add.
def : Pat<(add GR16:$src1, 128),
          (SUB16ri8 GR16:$src1, -128)>;
def : Pat<(store (add (loadi16 addr:$dst), 128), addr:$dst),
          (SUB16mi8 addr:$dst, -128)>;
def : Pat<(add GR32:$src1, 128),
          (SUB32ri8 GR32:$src1, -128)>;
def : Pat<(store (add (loadi32 addr:$dst), 128), addr:$dst),
          (SUB32mi8 addr:$dst, -128)>;

// r & (2^16-1) ==> movz
def : Pat<(and GR32:$src1, 0xffff),
          (MOVZX32rr16 (EXTRACT_SUBREG GR32:$src1, x86_subreg_16bit))>;
// r & (2^8-1) ==> movz
def : Pat<(and GR32:$src1, 0xff),
          (MOVZX32rr8 (EXTRACT_SUBREG (COPY_TO_REGCLASS GR32:$src1, GR32_),
                                      x86_subreg_8bit))>,
      Requires<[In32BitMode]>;
// r & (2^8-1) ==> movz
def : Pat<(and GR16:$src1, 0xff),
          (MOVZX16rr8 (EXTRACT_SUBREG (COPY_TO_REGCLASS GR16:$src1, GR16_),
                                      x86_subreg_8bit))>,
      Requires<[In32BitMode]>;

// sext_inreg patterns
def : Pat<(sext_inreg GR32:$src, i16),
          (MOVSX32rr16 (EXTRACT_SUBREG GR32:$src, x86_subreg_16bit))>;
def : Pat<(sext_inreg GR32:$src, i8),
          (MOVSX32rr8 (EXTRACT_SUBREG (COPY_TO_REGCLASS GR32:$src, GR32_),
                                      x86_subreg_8bit))>,
      Requires<[In32BitMode]>;
def : Pat<(sext_inreg GR16:$src, i8),
          (MOVSX16rr8 (EXTRACT_SUBREG (COPY_TO_REGCLASS GR16:$src, GR16_),
                                      x86_subreg_8bit))>,
      Requires<[In32BitMode]>;

// trunc patterns
def : Pat<(i16 (trunc GR32:$src)),
          (EXTRACT_SUBREG GR32:$src, x86_subreg_16bit)>;
def : Pat<(i8 (trunc GR32:$src)),
          (EXTRACT_SUBREG (COPY_TO_REGCLASS GR32:$src, GR32_),
                          x86_subreg_8bit)>,
      Requires<[In32BitMode]>;
def : Pat<(i8 (trunc GR16:$src)),
          (EXTRACT_SUBREG (COPY_TO_REGCLASS GR16:$src, GR16_),
                          x86_subreg_8bit)>,
      Requires<[In32BitMode]>;

// h-register tricks
def : Pat<(i8 (trunc (srl_su GR16:$src, (i8 8)))),
          (EXTRACT_SUBREG (COPY_TO_REGCLASS GR16:$src, GR16_),
                          x86_subreg_8bit_hi)>,
      Requires<[In32BitMode]>;
def : Pat<(i8 (trunc (srl_su GR32:$src, (i8 8)))),
          (EXTRACT_SUBREG (COPY_TO_REGCLASS GR32:$src, GR32_),
                          x86_subreg_8bit_hi)>,
      Requires<[In32BitMode]>;
def : Pat<(srl_su GR16:$src, (i8 8)),
          (EXTRACT_SUBREG
            (MOVZX32rr8
              (EXTRACT_SUBREG (COPY_TO_REGCLASS GR16:$src, GR16_),
                              x86_subreg_8bit_hi)),
            x86_subreg_16bit)>,
      Requires<[In32BitMode]>;
def : Pat<(and (srl_su GR32:$src, (i8 8)), (i32 255)),
          (MOVZX32rr8 (EXTRACT_SUBREG (COPY_TO_REGCLASS GR32:$src, GR32_),
                                      x86_subreg_8bit_hi))>,
      Requires<[In32BitMode]>;

// (shl x, 1) ==> (add x, x)
def : Pat<(shl GR8 :$src1, (i8 1)), (ADD8rr  GR8 :$src1, GR8 :$src1)>;
def : Pat<(shl GR16:$src1, (i8 1)), (ADD16rr GR16:$src1, GR16:$src1)>;
def : Pat<(shl GR32:$src1, (i8 1)), (ADD32rr GR32:$src1, GR32:$src1)>;

// (shl x (and y, 31)) ==> (shl x, y)
def : Pat<(shl GR8:$src1, (and CL:$amt, 31)),
          (SHL8rCL GR8:$src1)>;
def : Pat<(shl GR16:$src1, (and CL:$amt, 31)),
          (SHL16rCL GR16:$src1)>;
def : Pat<(shl GR32:$src1, (and CL:$amt, 31)),
          (SHL32rCL GR32:$src1)>;
def : Pat<(store (shl (loadi8 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SHL8mCL addr:$dst)>;
def : Pat<(store (shl (loadi16 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SHL16mCL addr:$dst)>;
def : Pat<(store (shl (loadi32 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SHL32mCL addr:$dst)>;

def : Pat<(srl GR8:$src1, (and CL:$amt, 31)),
          (SHR8rCL GR8:$src1)>;
def : Pat<(srl GR16:$src1, (and CL:$amt, 31)),
          (SHR16rCL GR16:$src1)>;
def : Pat<(srl GR32:$src1, (and CL:$amt, 31)),
          (SHR32rCL GR32:$src1)>;
def : Pat<(store (srl (loadi8 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SHR8mCL addr:$dst)>;
def : Pat<(store (srl (loadi16 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SHR16mCL addr:$dst)>;
def : Pat<(store (srl (loadi32 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SHR32mCL addr:$dst)>;

def : Pat<(sra GR8:$src1, (and CL:$amt, 31)),
          (SAR8rCL GR8:$src1)>;
def : Pat<(sra GR16:$src1, (and CL:$amt, 31)),
          (SAR16rCL GR16:$src1)>;
def : Pat<(sra GR32:$src1, (and CL:$amt, 31)),
          (SAR32rCL GR32:$src1)>;
def : Pat<(store (sra (loadi8 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SAR8mCL addr:$dst)>;
def : Pat<(store (sra (loadi16 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SAR16mCL addr:$dst)>;
def : Pat<(store (sra (loadi32 addr:$dst), (and CL:$amt, 31)), addr:$dst),
          (SAR32mCL addr:$dst)>;

// (or (x >> c) | (y << (32 - c))) ==> (shrd32 x, y, c)
def : Pat<(or (srl GR32:$src1, CL:$amt),
              (shl GR32:$src2, (sub 32, CL:$amt))),
          (SHRD32rrCL GR32:$src1, GR32:$src2)>;

def : Pat<(store (or (srl (loadi32 addr:$dst), CL:$amt),
                     (shl GR32:$src2, (sub 32, CL:$amt))), addr:$dst),
          (SHRD32mrCL addr:$dst, GR32:$src2)>;

def : Pat<(or (srl GR32:$src1, (i8 (trunc ECX:$amt))),
              (shl GR32:$src2, (i8 (trunc (sub 32, ECX:$amt))))),
          (SHRD32rrCL GR32:$src1, GR32:$src2)>;

def : Pat<(store (or (srl (loadi32 addr:$dst), (i8 (trunc ECX:$amt))),
                     (shl GR32:$src2, (i8 (trunc (sub 32, ECX:$amt))))),
                 addr:$dst),
          (SHRD32mrCL addr:$dst, GR32:$src2)>;

def : Pat<(shrd GR32:$src1, (i8 imm:$amt1), GR32:$src2, (i8 imm:$amt2)),
          (SHRD32rri8 GR32:$src1, GR32:$src2, (i8 imm:$amt1))>;

def : Pat<(store (shrd (loadi32 addr:$dst), (i8 imm:$amt1),
                       GR32:$src2, (i8 imm:$amt2)), addr:$dst),
          (SHRD32mri8 addr:$dst, GR32:$src2, (i8 imm:$amt1))>;

// (or (x << c) | (y >> (32 - c))) ==> (shld32 x, y, c)
def : Pat<(or (shl GR32:$src1, CL:$amt),
              (srl GR32:$src2, (sub 32, CL:$amt))),
          (SHLD32rrCL GR32:$src1, GR32:$src2)>;

def : Pat<(store (or (shl (loadi32 addr:$dst), CL:$amt),
                     (srl GR32:$src2, (sub 32, CL:$amt))), addr:$dst),
          (SHLD32mrCL addr:$dst, GR32:$src2)>;

def : Pat<(or (shl GR32:$src1, (i8 (trunc ECX:$amt))),
              (srl GR32:$src2, (i8 (trunc (sub 32, ECX:$amt))))),
          (SHLD32rrCL GR32:$src1, GR32:$src2)>;

def : Pat<(store (or (shl (loadi32 addr:$dst), (i8 (trunc ECX:$amt))),
                     (srl GR32:$src2, (i8 (trunc (sub 32, ECX:$amt))))),
                 addr:$dst),
          (SHLD32mrCL addr:$dst, GR32:$src2)>;

def : Pat<(shld GR32:$src1, (i8 imm:$amt1), GR32:$src2, (i8 imm:$amt2)),
          (SHLD32rri8 GR32:$src1, GR32:$src2, (i8 imm:$amt1))>;

def : Pat<(store (shld (loadi32 addr:$dst), (i8 imm:$amt1),
                       GR32:$src2, (i8 imm:$amt2)), addr:$dst),
          (SHLD32mri8 addr:$dst, GR32:$src2, (i8 imm:$amt1))>;

// (or (x >> c) | (y << (16 - c))) ==> (shrd16 x, y, c)
def : Pat<(or (srl GR16:$src1, CL:$amt),
              (shl GR16:$src2, (sub 16, CL:$amt))),
          (SHRD16rrCL GR16:$src1, GR16:$src2)>;

def : Pat<(store (or (srl (loadi16 addr:$dst), CL:$amt),
                     (shl GR16:$src2, (sub 16, CL:$amt))), addr:$dst),
          (SHRD16mrCL addr:$dst, GR16:$src2)>;

def : Pat<(or (srl GR16:$src1, (i8 (trunc CX:$amt))),
              (shl GR16:$src2, (i8 (trunc (sub 16, CX:$amt))))),
          (SHRD16rrCL GR16:$src1, GR16:$src2)>;

def : Pat<(store (or (srl (loadi16 addr:$dst), (i8 (trunc CX:$amt))),
                     (shl GR16:$src2, (i8 (trunc (sub 16, CX:$amt))))),
                 addr:$dst),
          (SHRD16mrCL addr:$dst, GR16:$src2)>;

def : Pat<(shrd GR16:$src1, (i8 imm:$amt1), GR16:$src2, (i8 imm:$amt2)),
          (SHRD16rri8 GR16:$src1, GR16:$src2, (i8 imm:$amt1))>;

def : Pat<(store (shrd (loadi16 addr:$dst), (i8 imm:$amt1),
                       GR16:$src2, (i8 imm:$amt2)), addr:$dst),
          (SHRD16mri8 addr:$dst, GR16:$src2, (i8 imm:$amt1))>;

// (or (x << c) | (y >> (16 - c))) ==> (shld16 x, y, c)
def : Pat<(or (shl GR16:$src1, CL:$amt),
              (srl GR16:$src2, (sub 16, CL:$amt))),
          (SHLD16rrCL GR16:$src1, GR16:$src2)>;

def : Pat<(store (or (shl (loadi16 addr:$dst), CL:$amt),
                     (srl GR16:$src2, (sub 16, CL:$amt))), addr:$dst),
          (SHLD16mrCL addr:$dst, GR16:$src2)>;

def : Pat<(or (shl GR16:$src1, (i8 (trunc CX:$amt))),
              (srl GR16:$src2, (i8 (trunc (sub 16, CX:$amt))))),
          (SHLD16rrCL GR16:$src1, GR16:$src2)>;

def : Pat<(store (or (shl (loadi16 addr:$dst), (i8 (trunc CX:$amt))),
                     (srl GR16:$src2, (i8 (trunc (sub 16, CX:$amt))))),
                 addr:$dst),
          (SHLD16mrCL addr:$dst, GR16:$src2)>;

def : Pat<(shld GR16:$src1, (i8 imm:$amt1), GR16:$src2, (i8 imm:$amt2)),
          (SHLD16rri8 GR16:$src1, GR16:$src2, (i8 imm:$amt1))>;

def : Pat<(store (shld (loadi16 addr:$dst), (i8 imm:$amt1),
                       GR16:$src2, (i8 imm:$amt2)), addr:$dst),
          (SHLD16mri8 addr:$dst, GR16:$src2, (i8 imm:$amt1))>;

//===----------------------------------------------------------------------===//
// EFLAGS-defining Patterns
//===----------------------------------------------------------------------===//

// Register-Register Addition with EFLAGS result
def : Pat<(parallel (X86add_flag GR8:$src1, GR8:$src2),
                    (implicit EFLAGS)),
          (ADD8rr GR8:$src1, GR8:$src2)>;

// Register-Register Addition with EFLAGS result
def : Pat<(parallel (X86add_flag GR16:$src1, GR16:$src2),
                    (implicit EFLAGS)),
          (ADD16rr GR16:$src1, GR16:$src2)>;
def : Pat<(parallel (X86add_flag GR32:$src1, GR32:$src2),
                    (implicit EFLAGS)),
          (ADD32rr GR32:$src1, GR32:$src2)>;

// Register-Memory Addition with EFLAGS result
def : Pat<(parallel (X86add_flag GR8:$src1, (loadi8 addr:$src2)),
                    (implicit EFLAGS)),
          (ADD8rm GR8:$src1, addr:$src2)>;
def : Pat<(parallel (X86add_flag GR16:$src1, (loadi16 addr:$src2)),
                    (implicit EFLAGS)),
          (ADD16rm GR16:$src1, addr:$src2)>;
def : Pat<(parallel (X86add_flag GR32:$src1, (loadi32 addr:$src2)),
                    (implicit EFLAGS)),
          (ADD32rm GR32:$src1, addr:$src2)>;

// Register-Integer Addition with EFLAGS result
def : Pat<(parallel (X86add_flag GR8:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (ADD8ri GR8:$src1, imm:$src2)>;

// Register-Integer Addition with EFLAGS result
def : Pat<(parallel (X86add_flag GR16:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (ADD16ri GR16:$src1, imm:$src2)>;
def : Pat<(parallel (X86add_flag GR32:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (ADD32ri GR32:$src1, imm:$src2)>;
def : Pat<(parallel (X86add_flag GR16:$src1, i16immSExt8:$src2),
                    (implicit EFLAGS)),
          (ADD16ri8 GR16:$src1, i16immSExt8:$src2)>;
def : Pat<(parallel (X86add_flag GR32:$src1, i32immSExt8:$src2),
                    (implicit EFLAGS)),
          (ADD32ri8 GR32:$src1, i32immSExt8:$src2)>;

// Memory-Register Addition with EFLAGS result
def : Pat<(parallel (store (X86add_flag (loadi8 addr:$dst), GR8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD8mr addr:$dst, GR8:$src2)>;
def : Pat<(parallel (store (X86add_flag (loadi16 addr:$dst), GR16:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD16mr addr:$dst, GR16:$src2)>;
def : Pat<(parallel (store (X86add_flag (loadi32 addr:$dst), GR32:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD32mr addr:$dst, GR32:$src2)>;
def : Pat<(parallel (store (X86add_flag (loadi8 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD8mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86add_flag (loadi16 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD16mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86add_flag (loadi32 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD32mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86add_flag (loadi16 addr:$dst), i16immSExt8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD16mi8 addr:$dst, i16immSExt8:$src2)>;
def : Pat<(parallel (store (X86add_flag (loadi32 addr:$dst), i32immSExt8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (ADD32mi8 addr:$dst, i32immSExt8:$src2)>;

// Register-Register Subtraction with EFLAGS result
def : Pat<(parallel (X86sub_flag GR8:$src1, GR8:$src2),
                    (implicit EFLAGS)),
          (SUB8rr GR8:$src1, GR8:$src2)>;
def : Pat<(parallel (X86sub_flag GR16:$src1, GR16:$src2),
                    (implicit EFLAGS)),
          (SUB16rr GR16:$src1, GR16:$src2)>;
def : Pat<(parallel (X86sub_flag GR32:$src1, GR32:$src2),
                    (implicit EFLAGS)),
          (SUB32rr GR32:$src1, GR32:$src2)>;

// Register-Memory Subtraction with EFLAGS result
def : Pat<(parallel (X86sub_flag GR8:$src1, (loadi8 addr:$src2)),
                    (implicit EFLAGS)),
          (SUB8rm GR8:$src1, addr:$src2)>;
def : Pat<(parallel (X86sub_flag GR16:$src1, (loadi16 addr:$src2)),
                    (implicit EFLAGS)),
          (SUB16rm GR16:$src1, addr:$src2)>;
def : Pat<(parallel (X86sub_flag GR32:$src1, (loadi32 addr:$src2)),
                    (implicit EFLAGS)),
          (SUB32rm GR32:$src1, addr:$src2)>;

// Register-Integer Subtraction with EFLAGS result
def : Pat<(parallel (X86sub_flag GR8:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (SUB8ri GR8:$src1, imm:$src2)>;
def : Pat<(parallel (X86sub_flag GR16:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (SUB16ri GR16:$src1, imm:$src2)>;
def : Pat<(parallel (X86sub_flag GR32:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (SUB32ri GR32:$src1, imm:$src2)>;
def : Pat<(parallel (X86sub_flag GR16:$src1, i16immSExt8:$src2),
                    (implicit EFLAGS)),
          (SUB16ri8 GR16:$src1, i16immSExt8:$src2)>;
def : Pat<(parallel (X86sub_flag GR32:$src1, i32immSExt8:$src2),
                    (implicit EFLAGS)),
          (SUB32ri8 GR32:$src1, i32immSExt8:$src2)>;

// Memory-Register Subtraction with EFLAGS result
def : Pat<(parallel (store (X86sub_flag (loadi8 addr:$dst), GR8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB8mr addr:$dst, GR8:$src2)>;
def : Pat<(parallel (store (X86sub_flag (loadi16 addr:$dst), GR16:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB16mr addr:$dst, GR16:$src2)>;
def : Pat<(parallel (store (X86sub_flag (loadi32 addr:$dst), GR32:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB32mr addr:$dst, GR32:$src2)>;

// Memory-Integer Subtraction with EFLAGS result
def : Pat<(parallel (store (X86sub_flag (loadi8 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB8mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86sub_flag (loadi16 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB16mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86sub_flag (loadi32 addr:$dst), imm:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB32mi addr:$dst, imm:$src2)>;
def : Pat<(parallel (store (X86sub_flag (loadi16 addr:$dst), i16immSExt8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB16mi8 addr:$dst, i16immSExt8:$src2)>;
def : Pat<(parallel (store (X86sub_flag (loadi32 addr:$dst), i32immSExt8:$src2),
                           addr:$dst),
                    (implicit EFLAGS)),
          (SUB32mi8 addr:$dst, i32immSExt8:$src2)>;


// Register-Register Signed Integer Multiply with EFLAGS result
def : Pat<(parallel (X86smul_flag GR16:$src1, GR16:$src2),
                    (implicit EFLAGS)),
          (IMUL16rr GR16:$src1, GR16:$src2)>;
def : Pat<(parallel (X86smul_flag GR32:$src1, GR32:$src2),
                    (implicit EFLAGS)),
          (IMUL32rr GR32:$src1, GR32:$src2)>;

// Register-Memory Signed Integer Multiply with EFLAGS result
def : Pat<(parallel (X86smul_flag GR16:$src1, (loadi16 addr:$src2)),
                    (implicit EFLAGS)),
          (IMUL16rm GR16:$src1, addr:$src2)>;
def : Pat<(parallel (X86smul_flag GR32:$src1, (loadi32 addr:$src2)),
                    (implicit EFLAGS)),
          (IMUL32rm GR32:$src1, addr:$src2)>;

// Register-Integer Signed Integer Multiply with EFLAGS result
def : Pat<(parallel (X86smul_flag GR16:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (IMUL16rri GR16:$src1, imm:$src2)>;
def : Pat<(parallel (X86smul_flag GR32:$src1, imm:$src2),
                    (implicit EFLAGS)),
          (IMUL32rri GR32:$src1, imm:$src2)>;
def : Pat<(parallel (X86smul_flag GR16:$src1, i16immSExt8:$src2),
                    (implicit EFLAGS)),
          (IMUL16rri8 GR16:$src1, i16immSExt8:$src2)>;
def : Pat<(parallel (X86smul_flag GR32:$src1, i32immSExt8:$src2),
                    (implicit EFLAGS)),
          (IMUL32rri8 GR32:$src1, i32immSExt8:$src2)>;

// Memory-Integer Signed Integer Multiply with EFLAGS result
def : Pat<(parallel (X86smul_flag (loadi16 addr:$src1), imm:$src2),
                    (implicit EFLAGS)),
          (IMUL16rmi addr:$src1, imm:$src2)>;
def : Pat<(parallel (X86smul_flag (loadi32 addr:$src1), imm:$src2),
                    (implicit EFLAGS)),
          (IMUL32rmi addr:$src1, imm:$src2)>;
def : Pat<(parallel (X86smul_flag (loadi16 addr:$src1), i16immSExt8:$src2),
                    (implicit EFLAGS)),
          (IMUL16rmi8 addr:$src1, i16immSExt8:$src2)>;
def : Pat<(parallel (X86smul_flag (loadi32 addr:$src1), i32immSExt8:$src2),
                    (implicit EFLAGS)),
          (IMUL32rmi8 addr:$src1, i32immSExt8:$src2)>;

// Optimize multiply by 2 with EFLAGS result.
let AddedComplexity = 2 in {
def : Pat<(parallel (X86smul_flag GR16:$src1, 2),
                    (implicit EFLAGS)),
          (ADD16rr GR16:$src1, GR16:$src1)>;

def : Pat<(parallel (X86smul_flag GR32:$src1, 2),
                    (implicit EFLAGS)),
          (ADD32rr GR32:$src1, GR32:$src1)>;
}

// INC and DEC with EFLAGS result. Note that these do not set CF.
def : Pat<(parallel (X86inc_flag GR8:$src), (implicit EFLAGS)),
          (INC8r GR8:$src)>;
def : Pat<(parallel (store (i8 (X86inc_flag (loadi8 addr:$dst))), addr:$dst),
                    (implicit EFLAGS)),
          (INC8m addr:$dst)>;
def : Pat<(parallel (X86dec_flag GR8:$src), (implicit EFLAGS)),
          (DEC8r GR8:$src)>;
def : Pat<(parallel (store (i8 (X86dec_flag (loadi8 addr:$dst))), addr:$dst),
                    (implicit EFLAGS)),
          (DEC8m addr:$dst)>;

def : Pat<(parallel (X86inc_flag GR16:$src), (implicit EFLAGS)),
          (INC16r GR16:$src)>, Requires<[In32BitMode]>;
def : Pat<(parallel (store (i16 (X86inc_flag (loadi16 addr:$dst))), addr:$dst),
                    (implicit EFLAGS)),
          (INC16m addr:$dst)>, Requires<[In32BitMode]>;
def : Pat<(parallel (X86dec_flag GR16:$src), (implicit EFLAGS)),
          (DEC16r GR16:$src)>, Requires<[In32BitMode]>;
def : Pat<(parallel (store (i16 (X86dec_flag (loadi16 addr:$dst))), addr:$dst),
                    (implicit EFLAGS)),
          (DEC16m addr:$dst)>, Requires<[In32BitMode]>;

def : Pat<(parallel (X86inc_flag GR32:$src), (implicit EFLAGS)),
          (INC32r GR32:$src)>, Requires<[In32BitMode]>;
def : Pat<(parallel (store (i32 (X86inc_flag (loadi32 addr:$dst))), addr:$dst),
                    (implicit EFLAGS)),
          (INC32m addr:$dst)>, Requires<[In32BitMode]>;
def : Pat<(parallel (X86dec_flag GR32:$src), (implicit EFLAGS)),
          (DEC32r GR32:$src)>, Requires<[In32BitMode]>;
def : Pat<(parallel (store (i32 (X86dec_flag (loadi32 addr:$dst))), addr:$dst),
                    (implicit EFLAGS)),
          (DEC32m addr:$dst)>, Requires<[In32BitMode]>;

//===----------------------------------------------------------------------===//
// Floating Point Stack Support
//===----------------------------------------------------------------------===//

include "X86InstrFPStack.td"

//===----------------------------------------------------------------------===//
// X86-64 Support
//===----------------------------------------------------------------------===//

include "X86Instr64bit.td"

//===----------------------------------------------------------------------===//
// XMM Floating point support (requires SSE / SSE2)
//===----------------------------------------------------------------------===//

include "X86InstrSSE.td"

//===----------------------------------------------------------------------===//
// MMX and XMM Packed Integer support (requires MMX, SSE, and SSE2)
//===----------------------------------------------------------------------===//

include "X86InstrMMX.td"